WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/workbench/diagnostics.go

@@ -0,0 +1,213 @@
+package workbench
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	apperrors "github.com/Tencent/WeKnora/internal/errors"
+	"github.com/Tencent/WeKnora/internal/logger"
+	"github.com/Tencent/WeKnora/internal/types"
+	"gorm.io/gorm"
+)
+
+var httpStatusPattern = regexp.MustCompile(`(?i)(?:status(?: code)?[ :]+)([45][0-9]{2})\b`)
+var diagnosticSecretPatterns = []*regexp.Regexp{
+	regexp.MustCompile(`(?i)(bearer\s+)[^\s,;]+`),
+	regexp.MustCompile(`(?i)((?:api[_-]?key|access[_-]?token|authorization|password|secret)\s*[=:]\s*)[^\s,;]+`),
+	regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{8,}`),
+}
+
+type phaseTimeoutError struct {
+	phase   string
+	elapsed time.Duration
+	cause   error
+}
+
+func (e *phaseTimeoutError) Error() string { return e.phase + " timeout" }
+func (e *phaseTimeoutError) Unwrap() error { return e.cause }
+
+func phaseFailure(err error, phase string, started time.Time) error {
+	if err == nil {
+		return nil
+	}
+	var existing *phaseTimeoutError
+	if errors.As(err, &existing) {
+		return err
+	}
+	var network net.Error
+	if errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &network) && network.Timeout()) {
+		return &phaseTimeoutError{phase: phase, elapsed: time.Since(started), cause: err}
+	}
+	return err
+}
+
+func redactDiagnosticMessage(message string) string {
+	for _, pattern := range diagnosticSecretPatterns {
+		message = pattern.ReplaceAllString(message, "[已隐藏]")
+	}
+	message = strings.Join(strings.Fields(message), " ")
+	if runes := []rune(message); len(runes) > 600 {
+		message = string(runes[:600]) + "..."
+	}
+	return message
+}
+
+// Diagnostic values are derived from error types and known status markers.
+// Provider response bodies can echo prompts or credentials and are never logged here.
+func jobErrorDetails(err error) (string, string) {
+	var phase *phaseTimeoutError
+	if errors.As(err, &phase) {
+		label := "AI 模型响应"
+		if phase.phase == "retrieval" {
+			label = "知识库检索"
+		} else if phase.phase == "reading" {
+			label = "资料摘要读取"
+		}
+		return phase.phase + "_timeout", fmt.Sprintf("%s超时（已等待 %d 秒），原内容已保留；可按任务编号查看具体阶段后重试", label, max(1, int(phase.elapsed.Seconds())))
+	}
+	var structured interface{ StructuredOutputDiagnostic() JSON }
+	if errors.As(err, &structured) {
+		return "structured_output_invalid", redactDiagnosticMessage(err.Error())
+	}
+	var appError *apperrors.AppError
+	if errors.As(err, &appError) && appError.HTTPCode >= 400 && appError.HTTPCode < 500 {
+		return fmt.Sprintf("application_%d", appError.Code), redactDiagnosticMessage(appError.Message)
+	}
+	switch {
+	case errors.Is(err, context.DeadlineExceeded):
+		return "timeout", "任务处理超过时限，原内容已保留；请按任务编号查看具体阶段后重试"
+	case errors.Is(err, context.Canceled):
+		return "cancelled", "任务执行已中断，已保留原内容，请重试"
+	case errors.Is(err, ErrForbidden):
+		return "forbidden", redactDiagnosticMessage(err.Error())
+	case errors.Is(err, ErrConflict):
+		return "conflict", redactDiagnosticMessage(err.Error())
+	case errors.Is(err, ErrInvalid):
+		return "invalid", redactDiagnosticMessage(err.Error())
+	case errors.Is(err, gorm.ErrRecordNotFound):
+		return "resource_unavailable", "任务引用的资源不存在或已不可访问，请检查 Agent、导图和画布配置"
+	case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF):
+		return "provider_disconnected", "模型服务连接中断或返回不完整，已保留原内容，请重试"
+	}
+	var netErr net.Error
+	if errors.As(err, &netErr) {
+		if netErr.Timeout() {
+			return "network_timeout", "连接模型或知识服务超时，已保留原内容，请重试"
+		}
+		return "network_error", "无法连接模型或知识服务，请检查服务地址及网络后重试"
+	}
+	if match := httpStatusPattern.FindStringSubmatch(err.Error()); len(match) == 2 {
+		status, _ := strconv.Atoi(match[1])
+		message := "模型服务返回错误"
+		switch status {
+		case 401:
+			message = "模型服务认证失败，请检查模型密钥"
+		case 403:
+			message = "模型服务拒绝访问，请检查密钥的模型权限"
+		case 404:
+			message = "模型或服务接口不存在，请检查模型名称及服务地址"
+		case 429:
+			message = "模型服务限流或额度不足，请检查额度并稍后重试"
+		default:
+			if status >= 500 {
+				message = "模型服务暂时异常，请稍后重试"
+			}
+		}
+		return fmt.Sprintf("provider_http_%d", status), fmt.Sprintf("%s（HTTP %d），原内容已保留", message, status)
+	}
+	return "internal_error", "系统处理失败，已记录错误；请联系管理员按发生时间查看服务日志"
+}
+
+func jobErrorMetadata(err error) JSON {
+	class, message := jobErrorDetails(err)
+	errorTypes := []string{}
+	for current := err; current != nil && len(errorTypes) < 8; current = errors.Unwrap(current) {
+		errorTypes = append(errorTypes, fmt.Sprintf("%T", current))
+	}
+	metadata := JSON{"error_class": class, "error_message": message, "error_type": strings.Join(errorTypes, " -> ")}
+	var phase *phaseTimeoutError
+	if errors.As(err, &phase) {
+		metadata["phase"], metadata["elapsed_ms"] = phase.phase, phase.elapsed.Milliseconds()
+	}
+	var appError *apperrors.AppError
+	if errors.As(err, &appError) {
+		metadata["application_code"], metadata["application_http_status"] = appError.Code, appError.HTTPCode
+	}
+	var structured interface{ StructuredOutputDiagnostic() JSON }
+	if errors.As(err, &structured) {
+		metadata["structured_output"] = structured.StructuredOutputDiagnostic()
+	}
+	return metadata
+}
+
+func logJobFailure(ctx context.Context, job *Job, stage string, err error) {
+	fields := logger.Fields(jobErrorMetadata(err))
+	fields["stage"] = stage
+	if job != nil {
+		fields["job_id"], fields["resource_id"] = job.ID, job.ResourceID
+		fields["tenant_id"], fields["owner_id"] = job.TenantID, job.OwnerID
+		fields["kind"], fields["action"] = job.Kind, job.Action
+	}
+	logger.GetLogger(ctx).WithFields(fields).Error("工作台任务处理失败")
+}
+
+func (s *Service) recordJobFailure(ctx context.Context, job *Job, stage string, err error) {
+	if job == nil || job.TenantID == 0 {
+		return
+	}
+	ctx = context.WithValue(ctx, types.TenantIDContextKey, job.TenantID)
+	ctx = context.WithValue(ctx, types.UserIDContextKey, job.OwnerID)
+	metadata := jobErrorMetadata(err)
+	metadata["source"], metadata["job_id"], metadata["stage"] = "background_job", job.ID, stage
+	metadata["kind"], metadata["action"], metadata["job_status"] = job.Kind, job.Action, job.Status
+	var stored Job
+	if queryErr := s.db.WithContext(ctx).Select("status", "progress", "stage", "generation").Where("id=? AND tenant_id=?", job.ID, job.TenantID).Take(&stored).Error; queryErr == nil {
+		metadata["progress"], metadata["job_stage"], metadata["job_status"] = stored.Progress, stored.Stage, stored.Status
+		for _, key := range []string{"model_id", "agent_id", "attempt"} {
+			if value, exists := stored.Generation[key]; exists {
+				metadata[key] = value
+			}
+		}
+		if phase, exists := stored.Generation["phase"]; exists {
+			metadata["generation_phase"] = phase
+			if _, diagnosed := metadata["phase"]; !diagnosed {
+				metadata["phase"] = phase
+			}
+		}
+	}
+	level := "error"
+	if class := metadata["error_class"].(string); class == "forbidden" || class == "conflict" || class == "invalid" || class == "cancelled" || class == "resource_unavailable" || strings.HasPrefix(class, "application_") {
+		level = "warn"
+	}
+	row := OperationLog{Kind: "system", Level: level, Action: actionName(job.Action), ResourceID: job.ResourceID, TraceID: job.ID,
+		Message: actionName(job.Action) + "失败：" + metadata["error_message"].(string), Metadata: metadata}
+	if recordErr := s.recordOperation(ctx, row); recordErr != nil {
+		logJobFailure(ctx, job, "persist_failure", recordErr)
+	}
+}
+
+func (s *Service) recoverExpiredJobs(ctx context.Context, now time.Time) error {
+	var jobs []Job
+	if err := s.db.WithContext(ctx).Where("status = ? AND (lease_until IS NULL OR lease_until <= ?)", "running", now).Order("updated_at ASC").Limit(100).Find(&jobs).Error; err != nil {
+		return err
+	}
+	for i := range jobs {
+		job := &jobs[i]
+		result := s.db.WithContext(ctx).Model(&Job{}).Where("id = ? AND tenant_id = ? AND status = ? AND (lease_until IS NULL OR lease_until <= ?)", job.ID, job.TenantID, "running", now).
+			Updates(map[string]any{"status": "failed", "stage": "执行中断", "error": "任务执行中断，请检查结果后重试", "updated_at": now})
+		if result.Error != nil {
+			return result.Error
+		}
+		if result.RowsAffected == 1 {
+			s.recordJobFailure(ctx, job, "lease_recovery", context.Canceled)
+		}
+	}
+	return nil
+}