xinmi/internal/workbench/diagnostics.go

package workbench

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net"
	"regexp"
	"strconv"
	"strings"
	"time"

	apperrors "github.com/Tencent/WeKnora/internal/errors"
	"github.com/Tencent/WeKnora/internal/logger"
	"github.com/Tencent/WeKnora/internal/types"
	"gorm.io/gorm"
)

var httpStatusPattern = regexp.MustCompile(`(?i)(?:status(?: code)?[ :]+)([45][0-9]{2})\b`)
var diagnosticSecretPatterns = []*regexp.Regexp{
	regexp.MustCompile(`(?i)(bearer\s+)[^\s,;]+`),
	regexp.MustCompile(`(?i)((?:api[_-]?key|access[_-]?token|authorization|password|secret)\s*[=:]\s*)[^\s,;]+`),
	regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{8,}`),
}

type phaseTimeoutError struct {
	phase   string
	elapsed time.Duration
	cause   error
}

func (e *phaseTimeoutError) Error() string { return e.phase + " timeout" }
func (e *phaseTimeoutError) Unwrap() error { return e.cause }

func phaseFailure(err error, phase string, started time.Time) error {
	if err == nil {
		return nil
	}
	var existing *phaseTimeoutError
	if errors.As(err, &existing) {
		return err
	}
	var network net.Error
	if errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &network) && network.Timeout()) {
		return &phaseTimeoutError{phase: phase, elapsed: time.Since(started), cause: err}
	}
	return err
}

func redactDiagnosticMessage(message string) string {
	for _, pattern := range diagnosticSecretPatterns {
		message = pattern.ReplaceAllString(message, "[已隐藏]")
	}
	message = strings.Join(strings.Fields(message), " ")
	if runes := []rune(message); len(runes) > 600 {
		message = string(runes[:600]) + "..."
	}
	return message
}

// Diagnostic values are derived from error types and known status markers.
// Provider response bodies can echo prompts or credentials and are never logged here.
func jobErrorDetails(err error) (string, string) {
	var phase *phaseTimeoutError
	if errors.As(err, &phase) {
		label := "AI 模型响应"
		if phase.phase == "retrieval" {
			label = "知识库检索"
		} else if phase.phase == "reading" {
			label = "资料摘要读取"
		}
		return phase.phase + "_timeout", fmt.Sprintf("%s超时（已等待 %d 秒），原内容已保留；可按任务编号查看具体阶段后重试", label, max(1, int(phase.elapsed.Seconds())))
	}
	var structured interface{ StructuredOutputDiagnostic() JSON }
	if errors.As(err, &structured) {
		return "structured_output_invalid", redactDiagnosticMessage(err.Error())
	}
	var appError *apperrors.AppError
	if errors.As(err, &appError) && appError.HTTPCode >= 400 && appError.HTTPCode < 500 {
		return fmt.Sprintf("application_%d", appError.Code), redactDiagnosticMessage(appError.Message)
	}
	switch {
	case errors.Is(err, context.DeadlineExceeded):
		return "timeout", "任务处理超过时限，原内容已保留；请按任务编号查看具体阶段后重试"
	case errors.Is(err, context.Canceled):
		return "cancelled", "任务执行已中断，已保留原内容，请重试"
	case errors.Is(err, ErrForbidden):
		return "forbidden", redactDiagnosticMessage(err.Error())
	case errors.Is(err, ErrConflict):
		return "conflict", redactDiagnosticMessage(err.Error())
	case errors.Is(err, ErrInvalid):
		return "invalid", redactDiagnosticMessage(err.Error())
	case errors.Is(err, gorm.ErrRecordNotFound):
		return "resource_unavailable", "任务引用的资源不存在或已不可访问，请检查 Agent、导图和画布配置"
	case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF):
		return "provider_disconnected", "模型服务连接中断或返回不完整，已保留原内容，请重试"
	}
	var netErr net.Error
	if errors.As(err, &netErr) {
		if netErr.Timeout() {
			return "network_timeout", "连接模型或知识服务超时，已保留原内容，请重试"
		}
		return "network_error", "无法连接模型或知识服务，请检查服务地址及网络后重试"
	}
	if match := httpStatusPattern.FindStringSubmatch(err.Error()); len(match) == 2 {
		status, _ := strconv.Atoi(match[1])
		message := "模型服务返回错误"
		switch status {
		case 401:
			message = "模型服务认证失败，请检查模型密钥"
		case 403:
			message = "模型服务拒绝访问，请检查密钥的模型权限"
		case 404:
			message = "模型或服务接口不存在，请检查模型名称及服务地址"
		case 429:
			message = "模型服务限流或额度不足，请检查额度并稍后重试"
		default:
			if status >= 500 {
				message = "模型服务暂时异常，请稍后重试"
			}
		}
		return fmt.Sprintf("provider_http_%d", status), fmt.Sprintf("%s（HTTP %d），原内容已保留", message, status)
	}
	return "internal_error", "系统处理失败，已记录错误；请联系管理员按发生时间查看服务日志"
}

func jobErrorMetadata(err error) JSON {
	class, message := jobErrorDetails(err)
	errorTypes := []string{}
	for current := err; current != nil && len(errorTypes) < 8; current = errors.Unwrap(current) {
		errorTypes = append(errorTypes, fmt.Sprintf("%T", current))
	}
	metadata := JSON{"error_class": class, "error_message": message, "error_type": strings.Join(errorTypes, " -> ")}
	var phase *phaseTimeoutError
	if errors.As(err, &phase) {
		metadata["phase"], metadata["elapsed_ms"] = phase.phase, phase.elapsed.Milliseconds()
	}
	var appError *apperrors.AppError
	if errors.As(err, &appError) {
		metadata["application_code"], metadata["application_http_status"] = appError.Code, appError.HTTPCode
	}
	var structured interface{ StructuredOutputDiagnostic() JSON }
	if errors.As(err, &structured) {
		metadata["structured_output"] = structured.StructuredOutputDiagnostic()
	}
	return metadata
}

func logJobFailure(ctx context.Context, job *Job, stage string, err error) {
	fields := logger.Fields(jobErrorMetadata(err))
	fields["stage"] = stage
	if job != nil {
		fields["job_id"], fields["resource_id"] = job.ID, job.ResourceID
		fields["tenant_id"], fields["owner_id"] = job.TenantID, job.OwnerID
		fields["kind"], fields["action"] = job.Kind, job.Action
	}
	logger.GetLogger(ctx).WithFields(fields).Error("工作台任务处理失败")
}

func (s *Service) recordJobFailure(ctx context.Context, job *Job, stage string, err error) {
	if job == nil || job.TenantID == 0 {
		return
	}
	ctx = context.WithValue(ctx, types.TenantIDContextKey, job.TenantID)
	ctx = context.WithValue(ctx, types.UserIDContextKey, job.OwnerID)
	metadata := jobErrorMetadata(err)
	metadata["source"], metadata["job_id"], metadata["stage"] = "background_job", job.ID, stage
	metadata["kind"], metadata["action"], metadata["job_status"] = job.Kind, job.Action, job.Status
	var stored Job
	if queryErr := s.db.WithContext(ctx).Select("status", "progress", "stage", "generation").Where("id=? AND tenant_id=?", job.ID, job.TenantID).Take(&stored).Error; queryErr == nil {
		metadata["progress"], metadata["job_stage"], metadata["job_status"] = stored.Progress, stored.Stage, stored.Status
		for _, key := range []string{"model_id", "agent_id", "attempt"} {
			if value, exists := stored.Generation[key]; exists {
				metadata[key] = value
			}
		}
		if phase, exists := stored.Generation["phase"]; exists {
			metadata["generation_phase"] = phase
			if _, diagnosed := metadata["phase"]; !diagnosed {
				metadata["phase"] = phase
			}
		}
	}
	level := "error"
	if class := metadata["error_class"].(string); class == "forbidden" || class == "conflict" || class == "invalid" || class == "cancelled" || class == "resource_unavailable" || strings.HasPrefix(class, "application_") {
		level = "warn"
	}
	row := OperationLog{Kind: "system", Level: level, Action: actionName(job.Action), ResourceID: job.ResourceID, TraceID: job.ID,
		Message: actionName(job.Action) + "失败：" + metadata["error_message"].(string), Metadata: metadata}
	if recordErr := s.recordOperation(ctx, row); recordErr != nil {
		logJobFailure(ctx, job, "persist_failure", recordErr)
	}
}

func (s *Service) recoverExpiredJobs(ctx context.Context, now time.Time) error {
	var jobs []Job
	if err := s.db.WithContext(ctx).Where("status = ? AND (lease_until IS NULL OR lease_until <= ?)", "running", now).Order("updated_at ASC").Limit(100).Find(&jobs).Error; err != nil {
		return err
	}
	for i := range jobs {
		job := &jobs[i]
		result := s.db.WithContext(ctx).Model(&Job{}).Where("id = ? AND tenant_id = ? AND status = ? AND (lease_until IS NULL OR lease_until <= ?)", job.ID, job.TenantID, "running", now).
			Updates(map[string]any{"status": "failed", "stage": "执行中断", "error": "任务执行中断，请检查结果后重试", "updated_at": now})
		if result.Error != nil {
			return result.Error
		}
		if result.RowsAffected == 1 {
			s.recordJobFailure(ctx, job, "lease_recovery", context.Canceled)
		}
	}
	return nil
}