214 lines
8.5 KiB
Go
214 lines
8.5 KiB
Go
|
package workbench
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"context"
|
|||
|
|
"errors"
|
|||
|
|
"fmt"
|
|||
|
|
"io"
|
|||
|
|
"net"
|
|||
|
|
"regexp"
|
|||
|
|
"strconv"
|
|||
|
|
"strings"
|
|||
|
|
"time"
|
|||
|
|
|
|||
|
|
apperrors "github.com/Tencent/WeKnora/internal/errors"
|
|||
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|||
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|||
|
|
"gorm.io/gorm"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
var httpStatusPattern = regexp.MustCompile(`(?i)(?:status(?: code)?[ :]+)([45][0-9]{2})\b`)
|
|||
|
|
var diagnosticSecretPatterns = []*regexp.Regexp{
|
|||
|
|
regexp.MustCompile(`(?i)(bearer\s+)[^\s,;]+`),
|
|||
|
|
regexp.MustCompile(`(?i)((?:api[_-]?key|access[_-]?token|authorization|password|secret)\s*[=:]\s*)[^\s,;]+`),
|
|||
|
|
regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{8,}`),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
type phaseTimeoutError struct {
|
|||
|
|
phase string
|
|||
|
|
elapsed time.Duration
|
|||
|
|
cause error
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (e *phaseTimeoutError) Error() string { return e.phase + " timeout" }
|
|||
|
|
func (e *phaseTimeoutError) Unwrap() error { return e.cause }
|
|||
|
|
|
|||
|
|
func phaseFailure(err error, phase string, started time.Time) error {
|
|||
|
|
if err == nil {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
var existing *phaseTimeoutError
|
|||
|
|
if errors.As(err, &existing) {
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
var network net.Error
|
|||
|
|
if errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &network) && network.Timeout()) {
|
|||
|
|
return &phaseTimeoutError{phase: phase, elapsed: time.Since(started), cause: err}
|
|||
|
|
}
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func redactDiagnosticMessage(message string) string {
|
|||
|
|
for _, pattern := range diagnosticSecretPatterns {
|
|||
|
|
message = pattern.ReplaceAllString(message, "[已隐藏]")
|
|||
|
|
}
|
|||
|
|
message = strings.Join(strings.Fields(message), " ")
|
|||
|
|
if runes := []rune(message); len(runes) > 600 {
|
|||
|
|
message = string(runes[:600]) + "..."
|
|||
|
|
}
|
|||
|
|
return message
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Diagnostic values are derived from error types and known status markers.
|
|||
|
|
// Provider response bodies can echo prompts or credentials and are never logged here.
|
|||
|
|
func jobErrorDetails(err error) (string, string) {
|
|||
|
|
var phase *phaseTimeoutError
|
|||
|
|
if errors.As(err, &phase) {
|
|||
|
|
label := "AI 模型响应"
|
|||
|
|
if phase.phase == "retrieval" {
|
|||
|
|
label = "知识库检索"
|
|||
|
|
} else if phase.phase == "reading" {
|
|||
|
|
label = "资料摘要读取"
|
|||
|
|
}
|
|||
|
|
return phase.phase + "_timeout", fmt.Sprintf("%s超时（已等待 %d 秒），原内容已保留；可按任务编号查看具体阶段后重试", label, max(1, int(phase.elapsed.Seconds())))
|
|||
|
|
}
|
|||
|
|
var structured interface{ StructuredOutputDiagnostic() JSON }
|
|||
|
|
if errors.As(err, &structured) {
|
|||
|
|
return "structured_output_invalid", redactDiagnosticMessage(err.Error())
|
|||
|
|
}
|
|||
|
|
var appError *apperrors.AppError
|
|||
|
|
if errors.As(err, &appError) && appError.HTTPCode >= 400 && appError.HTTPCode < 500 {
|
|||
|
|
return fmt.Sprintf("application_%d", appError.Code), redactDiagnosticMessage(appError.Message)
|
|||
|
|
}
|
|||
|
|
switch {
|
|||
|
|
case errors.Is(err, context.DeadlineExceeded):
|
|||
|
|
return "timeout", "任务处理超过时限，原内容已保留；请按任务编号查看具体阶段后重试"
|
|||
|
|
case errors.Is(err, context.Canceled):
|
|||
|
|
return "cancelled", "任务执行已中断，已保留原内容，请重试"
|
|||
|
|
case errors.Is(err, ErrForbidden):
|
|||
|
|
return "forbidden", redactDiagnosticMessage(err.Error())
|
|||
|
|
case errors.Is(err, ErrConflict):
|
|||
|
|
return "conflict", redactDiagnosticMessage(err.Error())
|
|||
|
|
case errors.Is(err, ErrInvalid):
|
|||
|
|
return "invalid", redactDiagnosticMessage(err.Error())
|
|||
|
|
case errors.Is(err, gorm.ErrRecordNotFound):
|
|||
|
|
return "resource_unavailable", "任务引用的资源不存在或已不可访问，请检查 Agent、导图和画布配置"
|
|||
|
|
case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF):
|
|||
|
|
return "provider_disconnected", "模型服务连接中断或返回不完整，已保留原内容，请重试"
|
|||
|
|
}
|
|||
|
|
var netErr net.Error
|
|||
|
|
if errors.As(err, &netErr) {
|
|||
|
|
if netErr.Timeout() {
|
|||
|
|
return "network_timeout", "连接模型或知识服务超时，已保留原内容，请重试"
|
|||
|
|
}
|
|||
|
|
return "network_error", "无法连接模型或知识服务，请检查服务地址及网络后重试"
|
|||
|
|
}
|
|||
|
|
if match := httpStatusPattern.FindStringSubmatch(err.Error()); len(match) == 2 {
|
|||
|
|
status, _ := strconv.Atoi(match[1])
|
|||
|
|
message := "模型服务返回错误"
|
|||
|
|
switch status {
|
|||
|
|
case 401:
|
|||
|
|
message = "模型服务认证失败，请检查模型密钥"
|
|||
|
|
case 403:
|
|||
|
|
message = "模型服务拒绝访问，请检查密钥的模型权限"
|
|||
|
|
case 404:
|
|||
|
|
message = "模型或服务接口不存在，请检查模型名称及服务地址"
|
|||
|
|
case 429:
|
|||
|
|
message = "模型服务限流或额度不足，请检查额度并稍后重试"
|
|||
|
|
default:
|
|||
|
|
if status >= 500 {
|
|||
|
|
message = "模型服务暂时异常，请稍后重试"
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return fmt.Sprintf("provider_http_%d", status), fmt.Sprintf("%s（HTTP %d），原内容已保留", message, status)
|
|||
|
|
}
|
|||
|
|
return "internal_error", "系统处理失败，已记录错误；请联系管理员按发生时间查看服务日志"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func jobErrorMetadata(err error) JSON {
|
|||
|
|
class, message := jobErrorDetails(err)
|
|||
|
|
errorTypes := []string{}
|
|||
|
|
for current := err; current != nil && len(errorTypes) < 8; current = errors.Unwrap(current) {
|
|||
|
|
errorTypes = append(errorTypes, fmt.Sprintf("%T", current))
|
|||
|
|
}
|
|||
|
|
metadata := JSON{"error_class": class, "error_message": message, "error_type": strings.Join(errorTypes, " -> ")}
|
|||
|
|
var phase *phaseTimeoutError
|
|||
|
|
if errors.As(err, &phase) {
|
|||
|
|
metadata["phase"], metadata["elapsed_ms"] = phase.phase, phase.elapsed.Milliseconds()
|
|||
|
|
}
|
|||
|
|
var appError *apperrors.AppError
|
|||
|
|
if errors.As(err, &appError) {
|
|||
|
|
metadata["application_code"], metadata["application_http_status"] = appError.Code, appError.HTTPCode
|
|||
|
|
}
|
|||
|
|
var structured interface{ StructuredOutputDiagnostic() JSON }
|
|||
|
|
if errors.As(err, &structured) {
|
|||
|
|
metadata["structured_output"] = structured.StructuredOutputDiagnostic()
|
|||
|
|
}
|
|||
|
|
return metadata
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func logJobFailure(ctx context.Context, job *Job, stage string, err error) {
|
|||
|
|
fields := logger.Fields(jobErrorMetadata(err))
|
|||
|
|
fields["stage"] = stage
|
|||
|
|
if job != nil {
|
|||
|
|
fields["job_id"], fields["resource_id"] = job.ID, job.ResourceID
|
|||
|
|
fields["tenant_id"], fields["owner_id"] = job.TenantID, job.OwnerID
|
|||
|
|
fields["kind"], fields["action"] = job.Kind, job.Action
|
|||
|
|
}
|
|||
|
|
logger.GetLogger(ctx).WithFields(fields).Error("工作台任务处理失败")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Service) recordJobFailure(ctx context.Context, job *Job, stage string, err error) {
|
|||
|
|
if job == nil || job.TenantID == 0 {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
ctx = context.WithValue(ctx, types.TenantIDContextKey, job.TenantID)
|
|||
|
|
ctx = context.WithValue(ctx, types.UserIDContextKey, job.OwnerID)
|
|||
|
|
metadata := jobErrorMetadata(err)
|
|||
|
|
metadata["source"], metadata["job_id"], metadata["stage"] = "background_job", job.ID, stage
|
|||
|
|
metadata["kind"], metadata["action"], metadata["job_status"] = job.Kind, job.Action, job.Status
|
|||
|
|
var stored Job
|
|||
|
|
if queryErr := s.db.WithContext(ctx).Select("status", "progress", "stage", "generation").Where("id=? AND tenant_id=?", job.ID, job.TenantID).Take(&stored).Error; queryErr == nil {
|
|||
|
|
metadata["progress"], metadata["job_stage"], metadata["job_status"] = stored.Progress, stored.Stage, stored.Status
|
|||
|
|
for _, key := range []string{"model_id", "agent_id", "attempt"} {
|
|||
|
|
if value, exists := stored.Generation[key]; exists {
|
|||
|
|
metadata[key] = value
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if phase, exists := stored.Generation["phase"]; exists {
|
|||
|
|
metadata["generation_phase"] = phase
|
|||
|
|
if _, diagnosed := metadata["phase"]; !diagnosed {
|
|||
|
|
metadata["phase"] = phase
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
level := "error"
|
|||
|
|
if class := metadata["error_class"].(string); class == "forbidden" || class == "conflict" || class == "invalid" || class == "cancelled" || class == "resource_unavailable" || strings.HasPrefix(class, "application_") {
|
|||
|
|
level = "warn"
|
|||
|
|
}
|
|||
|
|
row := OperationLog{Kind: "system", Level: level, Action: actionName(job.Action), ResourceID: job.ResourceID, TraceID: job.ID,
|
|||
|
|
Message: actionName(job.Action) + "失败：" + metadata["error_message"].(string), Metadata: metadata}
|
|||
|
|
if recordErr := s.recordOperation(ctx, row); recordErr != nil {
|
|||
|
|
logJobFailure(ctx, job, "persist_failure", recordErr)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Service) recoverExpiredJobs(ctx context.Context, now time.Time) error {
|
|||
|
|
var jobs []Job
|
|||
|
|
if err := s.db.WithContext(ctx).Where("status = ? AND (lease_until IS NULL OR lease_until <= ?)", "running", now).Order("updated_at ASC").Limit(100).Find(&jobs).Error; err != nil {
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
for i := range jobs {
|
|||
|
|
job := &jobs[i]
|
|||
|
|
result := s.db.WithContext(ctx).Model(&Job{}).Where("id = ? AND tenant_id = ? AND status = ? AND (lease_until IS NULL OR lease_until <= ?)", job.ID, job.TenantID, "running", now).
|
|||
|
|
Updates(map[string]any{"status": "failed", "stage": "执行中断", "error": "任务执行中断，请检查结果后重试", "updated_at": now})
|
|||
|
|
if result.Error != nil {
|
|||
|
|
return result.Error
|
|||
|
|
}
|
|||
|
|
if result.RowsAffected == 1 {
|
|||
|
|
s.recordJobFailure(ctx, job, "lease_recovery", context.Canceled)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return nil
|
|||
|
|
}
|