WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/workbench/diagnostics_test.go

@@ -0,0 +1,186 @@
+package workbench
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestJobErrorDetailsNeverExposeProviderBody(t *testing.T) {
+	for _, code := range []int{400, 401, 403, 404, 429, 500, 524} {
+		err := fmt.Errorf("create chat completion: error, status code: %d, secret-api-key and private prompt", code)
+		class, message := jobErrorDetails(err)
+		if class != fmt.Sprintf("provider_http_%d", code) || !strings.Contains(message, fmt.Sprintf("HTTP %d", code)) {
+			t.Fatalf("unexpected diagnostic %s %s", class, message)
+		}
+		if strings.Contains(message, "secret") || strings.Contains(message, "private") {
+			t.Fatal("provider response leaked")
+		}
+	}
+	for _, tc := range []struct {
+		err   error
+		class string
+	}{
+		{context.DeadlineExceeded, "timeout"}, {context.Canceled, "cancelled"}, {io.ErrUnexpectedEOF, "provider_disconnected"}, {errors.New("raw private-prompt"), "internal_error"},
+	} {
+		class, message := jobErrorDetails(fmt.Errorf("call failed: %w", tc.err))
+		if class != tc.class || strings.Contains(message, "private-prompt") {
+			t.Fatalf("unexpected diagnostic %s %s", class, message)
+		}
+	}
+}
+
+func TestFailedJobKeepsProgressAndDoesNotCallDBFaultForbidden(t *testing.T) {
+	for _, dbFault := range []bool{false, true} {
+		s := sqliteWorkbench(t)
+		if !dbFault {
+			if err := s.db.AutoMigrate(&types.TenantMember{}); err != nil {
+				t.Fatal(err)
+			}
+		}
+		until := time.Now().Add(time.Minute)
+		job := Job{ID: "diagnostic-job", TenantID: 1, OwnerID: "owner", ResourceID: "resource", Kind: "training-maps", Status: "running", Progress: 5, LeaseToken: "lease", LeaseUntil: &until}
+		if err := s.db.Create(&job).Error; err != nil {
+			t.Fatal(err)
+		}
+		if err := s.db.Model(&Job{}).Where("id = ?", job.ID).Update("progress", 45).Error; err != nil {
+			t.Fatal(err)
+		}
+		s.run(context.Background(), &job)
+		var stored Job
+		if err := s.db.First(&stored, "id = ?", job.ID).Error; err != nil {
+			t.Fatal(err)
+		}
+		if stored.Status != "failed" || stored.Progress != 45 {
+			t.Fatalf("lost failure state/progress: %+v", stored)
+		}
+		if dbFault && stored.Error == ErrForbidden.Error() {
+			t.Fatal("database failure reported as missing permission")
+		}
+		if !dbFault && stored.Error != ErrForbidden.Error() {
+			t.Fatal("missing member was allowed")
+		}
+		var logged OperationLog
+		if err := s.db.Where("kind = ? AND trace_id = ?", "system", job.ID).Take(&logged).Error; err != nil {
+			t.Fatal("background failure was not persisted:", err)
+		}
+		if logged.TenantID != 1 || logged.OwnerID != "owner" || logged.ResourceID != "resource" || logged.Metadata["source"] != "background_job" {
+			t.Fatalf("failure attribution lost: %+v", logged)
+		}
+		if dbFault && logged.Level != "error" || !dbFault && logged.Level != "warn" {
+			t.Fatalf("wrong failure classification: %+v", logged)
+		}
+	}
+}
+
+func TestJobFailurePersistsSafeStructuredDiagnostic(t *testing.T) {
+	s := sqliteWorkbench(t)
+	job := Job{ID: "format-job", TenantID: 7, OwnerID: "author", ResourceID: "map", Action: "generate", Kind: "training-maps", Status: "failed", Progress: 70, Generation: JSON{"model_id": "model", "agent_id": "agent", "attempt": 2, "untrusted": "private prompt"}}
+	if err := s.db.Create(&job).Error; err != nil {
+		t.Fatal(err)
+	}
+	_, err := parseJSON[mapDaysOutput](`{"days":"private-body-secret"}`)
+	s.recordJobFailure(context.Background(), &job, "execute", err)
+	var row OperationLog
+	if err := s.db.First(&row).Error; err != nil {
+		t.Fatal(err)
+	}
+	if row.Metadata["error_class"] != "structured_output_invalid" || row.Metadata["structured_output"] == nil || row.Metadata["model_id"] != "model" || row.Level != "error" || row.TenantID != 7 {
+		t.Fatalf("diagnostic was not retained: %+v", row)
+	}
+	for _, secret := range []string{"private-body-secret", "private prompt", "untrusted"} {
+		if strings.Contains(compact(row), secret) {
+			t.Fatalf("private data leaked: %s", secret)
+		}
+	}
+}
+
+func TestRecoverExpiredJobsRecordsEachFailureOnce(t *testing.T) {
+	s := sqliteWorkbench(t)
+	expired := time.Now().Add(-time.Minute)
+	for _, job := range []Job{
+		{ID: "expired", TenantID: 1, OwnerID: "alice", Status: "running", LeaseUntil: &expired},
+		{ID: "cancelled", TenantID: 1, OwnerID: "alice", Status: "cancelled", LeaseUntil: &expired},
+	} {
+		if err := s.db.Create(&job).Error; err != nil {
+			t.Fatal(err)
+		}
+	}
+	for i := 0; i < 2; i++ {
+		if err := s.recoverExpiredJobs(context.Background(), time.Now()); err != nil {
+			t.Fatal(err)
+		}
+	}
+	var rows []OperationLog
+	if err := s.db.Find(&rows).Error; err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) != 1 || rows[0].TraceID != "expired" || rows[0].Metadata["stage"] != "lease_recovery" {
+		t.Fatalf("duplicate or missing lease recovery record: %+v", rows)
+	}
+}
+
+func TestJobDiagnosticRedactsCredentials(t *testing.T) {
+	err := fmt.Errorf("%w: 模型配置 apikey=private-value Authorization: Bearer private-token sk-private12345", ErrInvalid)
+	_, message := jobErrorDetails(err)
+	if strings.Contains(message, "private") || !strings.Contains(message, "模型配置") {
+		t.Fatalf("credential redaction lost useful context or exposed a credential: %s", message)
+	}
+}
+
+func TestTimeoutDiagnosticsIdentifyPhaseWithoutProviderBody(t *testing.T) {
+	for _, phase := range []string{"model", "retrieval", "reading"} {
+		err := phaseFailure(fmt.Errorf("private-provider-body: %w", context.DeadlineExceeded), phase, time.Now().Add(-3*time.Second))
+		if !errors.Is(err, context.DeadlineExceeded) {
+			t.Fatal("timeout no longer unwraps")
+		}
+		metadata := jobErrorMetadata(err)
+		if metadata["error_class"] != phase+"_timeout" || metadata["phase"] != phase || metadata["elapsed_ms"].(int64) < 3000 || strings.Contains(compact(metadata), "private-provider-body") {
+			t.Fatalf("invalid phase diagnostic: %+v", metadata)
+		}
+		if phaseFailure(err, "other", time.Now()) != err {
+			t.Fatal("outer wrapper lost specific timeout phase")
+		}
+	}
+	if phaseFailure(context.Canceled, "model", time.Now()) != context.Canceled {
+		t.Fatal("cancellation was mislabeled timeout")
+	}
+}
+
+func TestJobFailureKeepsTimeoutPhaseDuringStructuredRepair(t *testing.T) {
+	for _, tc := range []struct {
+		name, phase, class string
+		err                error
+	}{
+		{"model timeout", "model", "model_timeout", phaseFailure(context.DeadlineExceeded, "model", time.Now().Add(-time.Second))},
+		{"other failure", "structured_output_repair", "internal_error", errors.New("private provider response")},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			s := sqliteWorkbench(t)
+			job := Job{ID: "repair-job", TenantID: 7, OwnerID: "author", ResourceID: "map", Action: "generate", Kind: "training-maps", Status: "failed", Generation: JSON{"phase": "structured_output_repair", "model_id": "model", "agent_id": "agent", "attempt": 2}}
+			if err := s.db.Create(&job).Error; err != nil {
+				t.Fatal(err)
+			}
+			s.recordJobFailure(context.Background(), &job, "execute", tc.err)
+			var row OperationLog
+			if err := s.db.First(&row).Error; err != nil {
+				t.Fatal(err)
+			}
+			if row.Metadata["phase"] != tc.phase || row.Metadata["error_class"] != tc.class || row.Metadata["generation_phase"] != "structured_output_repair" {
+				t.Fatalf("error phase or generation operation was lost: %+v", row.Metadata)
+			}
+			if row.Metadata["model_id"] != "model" || row.Metadata["agent_id"] != "agent" || row.Metadata["attempt"] != float64(2) {
+				t.Fatalf("generation attribution was lost: %+v", row.Metadata)
+			}
+			if tc.phase == "model" && row.Metadata["elapsed_ms"].(float64) < 1000 {
+				t.Fatalf("timeout elapsed time was lost: %+v", row.Metadata)
+			}
+		})
+	}
+}