package workbench import ( "context" "errors" "fmt" "io" "net" "regexp" "strconv" "strings" "time" apperrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "gorm.io/gorm" ) var httpStatusPattern = regexp.MustCompile(`(?i)(?:status(?: code)?[ :]+)([45][0-9]{2})\b`) var diagnosticSecretPatterns = []*regexp.Regexp{ regexp.MustCompile(`(?i)(bearer\s+)[^\s,;]+`), regexp.MustCompile(`(?i)((?:api[_-]?key|access[_-]?token|authorization|password|secret)\s*[=:]\s*)[^\s,;]+`), regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{8,}`), } type phaseTimeoutError struct { phase string elapsed time.Duration cause error } func (e *phaseTimeoutError) Error() string { return e.phase + " timeout" } func (e *phaseTimeoutError) Unwrap() error { return e.cause } func phaseFailure(err error, phase string, started time.Time) error { if err == nil { return nil } var existing *phaseTimeoutError if errors.As(err, &existing) { return err } var network net.Error if errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &network) && network.Timeout()) { return &phaseTimeoutError{phase: phase, elapsed: time.Since(started), cause: err} } return err } func redactDiagnosticMessage(message string) string { for _, pattern := range diagnosticSecretPatterns { message = pattern.ReplaceAllString(message, "[已隐藏]") } message = strings.Join(strings.Fields(message), " ") if runes := []rune(message); len(runes) > 600 { message = string(runes[:600]) + "..." } return message } // Diagnostic values are derived from error types and known status markers. // Provider response bodies can echo prompts or credentials and are never logged here. func jobErrorDetails(err error) (string, string) { var phase *phaseTimeoutError if errors.As(err, &phase) { label := "AI 模型响应" if phase.phase == "retrieval" { label = "知识库检索" } else if phase.phase == "reading" { label = "资料摘要读取" } return phase.phase + "_timeout", fmt.Sprintf("%s超时(已等待 %d 秒),原内容已保留;可按任务编号查看具体阶段后重试", label, max(1, int(phase.elapsed.Seconds()))) } var structured interface{ StructuredOutputDiagnostic() JSON } if errors.As(err, &structured) { return "structured_output_invalid", redactDiagnosticMessage(err.Error()) } var appError *apperrors.AppError if errors.As(err, &appError) && appError.HTTPCode >= 400 && appError.HTTPCode < 500 { return fmt.Sprintf("application_%d", appError.Code), redactDiagnosticMessage(appError.Message) } switch { case errors.Is(err, context.DeadlineExceeded): return "timeout", "任务处理超过时限,原内容已保留;请按任务编号查看具体阶段后重试" case errors.Is(err, context.Canceled): return "cancelled", "任务执行已中断,已保留原内容,请重试" case errors.Is(err, ErrForbidden): return "forbidden", redactDiagnosticMessage(err.Error()) case errors.Is(err, ErrConflict): return "conflict", redactDiagnosticMessage(err.Error()) case errors.Is(err, ErrInvalid): return "invalid", redactDiagnosticMessage(err.Error()) case errors.Is(err, gorm.ErrRecordNotFound): return "resource_unavailable", "任务引用的资源不存在或已不可访问,请检查 Agent、导图和画布配置" case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF): return "provider_disconnected", "模型服务连接中断或返回不完整,已保留原内容,请重试" } var netErr net.Error if errors.As(err, &netErr) { if netErr.Timeout() { return "network_timeout", "连接模型或知识服务超时,已保留原内容,请重试" } return "network_error", "无法连接模型或知识服务,请检查服务地址及网络后重试" } if match := httpStatusPattern.FindStringSubmatch(err.Error()); len(match) == 2 { status, _ := strconv.Atoi(match[1]) message := "模型服务返回错误" switch status { case 401: message = "模型服务认证失败,请检查模型密钥" case 403: message = "模型服务拒绝访问,请检查密钥的模型权限" case 404: message = "模型或服务接口不存在,请检查模型名称及服务地址" case 429: message = "模型服务限流或额度不足,请检查额度并稍后重试" default: if status >= 500 { message = "模型服务暂时异常,请稍后重试" } } return fmt.Sprintf("provider_http_%d", status), fmt.Sprintf("%s(HTTP %d),原内容已保留", message, status) } return "internal_error", "系统处理失败,已记录错误;请联系管理员按发生时间查看服务日志" } func jobErrorMetadata(err error) JSON { class, message := jobErrorDetails(err) errorTypes := []string{} for current := err; current != nil && len(errorTypes) < 8; current = errors.Unwrap(current) { errorTypes = append(errorTypes, fmt.Sprintf("%T", current)) } metadata := JSON{"error_class": class, "error_message": message, "error_type": strings.Join(errorTypes, " -> ")} var phase *phaseTimeoutError if errors.As(err, &phase) { metadata["phase"], metadata["elapsed_ms"] = phase.phase, phase.elapsed.Milliseconds() } var appError *apperrors.AppError if errors.As(err, &appError) { metadata["application_code"], metadata["application_http_status"] = appError.Code, appError.HTTPCode } var structured interface{ StructuredOutputDiagnostic() JSON } if errors.As(err, &structured) { metadata["structured_output"] = structured.StructuredOutputDiagnostic() } return metadata } func logJobFailure(ctx context.Context, job *Job, stage string, err error) { fields := logger.Fields(jobErrorMetadata(err)) fields["stage"] = stage if job != nil { fields["job_id"], fields["resource_id"] = job.ID, job.ResourceID fields["tenant_id"], fields["owner_id"] = job.TenantID, job.OwnerID fields["kind"], fields["action"] = job.Kind, job.Action } logger.GetLogger(ctx).WithFields(fields).Error("工作台任务处理失败") } func (s *Service) recordJobFailure(ctx context.Context, job *Job, stage string, err error) { if job == nil || job.TenantID == 0 { return } ctx = context.WithValue(ctx, types.TenantIDContextKey, job.TenantID) ctx = context.WithValue(ctx, types.UserIDContextKey, job.OwnerID) metadata := jobErrorMetadata(err) metadata["source"], metadata["job_id"], metadata["stage"] = "background_job", job.ID, stage metadata["kind"], metadata["action"], metadata["job_status"] = job.Kind, job.Action, job.Status var stored Job if queryErr := s.db.WithContext(ctx).Select("status", "progress", "stage", "generation").Where("id=? AND tenant_id=?", job.ID, job.TenantID).Take(&stored).Error; queryErr == nil { metadata["progress"], metadata["job_stage"], metadata["job_status"] = stored.Progress, stored.Stage, stored.Status for _, key := range []string{"model_id", "agent_id", "attempt"} { if value, exists := stored.Generation[key]; exists { metadata[key] = value } } if phase, exists := stored.Generation["phase"]; exists { metadata["generation_phase"] = phase if _, diagnosed := metadata["phase"]; !diagnosed { metadata["phase"] = phase } } } level := "error" if class := metadata["error_class"].(string); class == "forbidden" || class == "conflict" || class == "invalid" || class == "cancelled" || class == "resource_unavailable" || strings.HasPrefix(class, "application_") { level = "warn" } row := OperationLog{Kind: "system", Level: level, Action: actionName(job.Action), ResourceID: job.ResourceID, TraceID: job.ID, Message: actionName(job.Action) + "失败:" + metadata["error_message"].(string), Metadata: metadata} if recordErr := s.recordOperation(ctx, row); recordErr != nil { logJobFailure(ctx, job, "persist_failure", recordErr) } } func (s *Service) recoverExpiredJobs(ctx context.Context, now time.Time) error { var jobs []Job if err := s.db.WithContext(ctx).Where("status = ? AND (lease_until IS NULL OR lease_until <= ?)", "running", now).Order("updated_at ASC").Limit(100).Find(&jobs).Error; err != nil { return err } for i := range jobs { job := &jobs[i] result := s.db.WithContext(ctx).Model(&Job{}).Where("id = ? AND tenant_id = ? AND status = ? AND (lease_until IS NULL OR lease_until <= ?)", job.ID, job.TenantID, "running", now). Updates(map[string]any{"status": "failed", "stage": "执行中断", "error": "任务执行中断,请检查结果后重试", "updated_at": now}) if result.Error != nil { return result.Error } if result.RowsAffected == 1 { s.recordJobFailure(ctx, job, "lease_recovery", context.Canceled) } } return nil }