WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

2026-09-22 16:19:06 +08:00
commit f7fc992f57
3929 changed files with 1006607 additions and 0 deletions
@@ -0,0 +1,848 @@
+package main
+
+import (
+	"crypto/sha256"
+	"database/sql"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"flag"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/hibiken/asynq"
+	"github.com/mattn/go-sqlite3"
+	"gorm.io/driver/postgres"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+func TestGraphExactTaskCannotReuseAnUnpinnedPayload(t *testing.T) {
+	candidate := graphRecoveryCandidate{ID: "chunk", KnowledgeID: "doc", ContentHash: strings.Repeat("a", 64)}
+	for _, change := range []string{"unchanged", "legacy", "tenant", "chunk", "kb", "document", "hash", "invalid"} {
+		t.Run(change, func(t *testing.T) {
+			task := types.ExtractChunkPayload{TenantID: 1, ChunkID: candidate.ID, ExpectedSource: &types.GraphSourceIdentity{KnowledgeBaseID: "kb", KnowledgeID: candidate.KnowledgeID, ContentSHA256: candidate.ContentHash}}
+			switch change {
+			case "legacy":
+				task.ExpectedSource = nil
+			case "tenant":
+				task.TenantID = 2
+			case "chunk":
+				task.ChunkID = "other"
+			case "kb":
+				task.ExpectedSource.KnowledgeBaseID = "other"
+			case "document":
+				task.ExpectedSource.KnowledgeID = "other"
+			case "hash":
+				task.ExpectedSource.ContentSHA256 = strings.Repeat("b", 64)
+			}
+			payload, _ := json.Marshal(task)
+			if change == "invalid" {
+				payload = []byte("invalid")
+			}
+			if graphTaskMatchesReviewedSource(payload, 1, "kb", candidate) != (change == "unchanged") {
+				t.Fatal("unsafe existing task accepted")
+			}
+		})
+	}
+}
+
+func TestGraphExactScopeValidation(t *testing.T) {
+	const chunk = "e40b74eb-0ac7-4ff6-a107-00ce5adfe610"
+	const knowledge = "9669e1b1-e28e-49a4-ae13-e97e37e21b11"
+	const hash = "4c4d84679328fc6dd3c17add7a91d13669c2d0cc2bc507098ed4fcc1c0dd84b2"
+	for _, tc := range []struct {
+		name, chunk, knowledge, hash string
+		after                        int64
+		limit, outstanding           int
+		valid                        bool
+	}{
+		{"normal batch", "", "", "", 2, 20, 2, true},
+		{"exact", chunk, knowledge, hash, 0, 1, 1, true},
+		{"missing chunk", "", knowledge, hash, 0, 1, 1, false},
+		{"missing knowledge", chunk, "", hash, 0, 1, 1, false},
+		{"missing hash", chunk, knowledge, "", 0, 1, 1, false},
+		{"invalid chunk", "other", knowledge, hash, 0, 1, 1, false},
+		{"nil uuid", "00000000-0000-0000-0000-000000000000", knowledge, hash, 0, 1, 1, false},
+		{"noncanonical chunk", strings.ToUpper(chunk), knowledge, hash, 0, 1, 1, false},
+		{"invalid hash", chunk, knowledge, strings.Repeat("z", 64), 0, 1, 1, false},
+		{"uppercase hash", chunk, knowledge, strings.ToUpper(hash), 0, 1, 1, false},
+		{"short hash", chunk, knowledge, hash[:62], 0, 1, 1, false},
+		{"cursor", chunk, knowledge, hash, 1, 1, 1, false},
+		{"batch", chunk, knowledge, hash, 0, 2, 1, false},
+		{"parallel", chunk, knowledge, hash, 0, 1, 2, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			err := graphValidateExactScope(tc.chunk, tc.knowledge, tc.hash, tc.after, tc.limit, tc.outstanding)
+			if (err == nil) != tc.valid {
+				t.Fatalf("valid=%t err=%v", tc.valid, err)
+			}
+		})
+	}
+}
+
+func TestGraphExactScopeNeverSubstitutesNextCandidate(t *testing.T) {
+	for _, change := range []string{"unchanged", "hash", "document", "disabled", "deleted", "document deleted", "unparsed", "completed", "failed", "other tenant", "other kb", "missing"} {
+		t.Run(change, func(t *testing.T) {
+			db := graphTestDB(t, filepath.Join(t.TempDir(), "exact.db"))
+			wanted := seedGraphChunk(t, db, 1, "kb", "wanted", "reviewed source", 1)
+			seedGraphChunk(t, db, 1, "kb", "next", "next source", 2)
+			switch change {
+			case "hash":
+				db.Exec("UPDATE chunks SET content='changed' WHERE id='wanted'")
+			case "document":
+				wanted.KnowledgeID = "other"
+			case "disabled":
+				db.Exec("UPDATE chunks SET is_enabled=false WHERE id='wanted'")
+			case "deleted":
+				db.Exec("UPDATE chunks SET deleted_at='2026-09-20' WHERE id='wanted'")
+			case "document deleted":
+				db.Exec("UPDATE knowledges SET deleted_at='2026-09-20' WHERE id=?", wanted.KnowledgeID)
+			case "unparsed":
+				db.Exec("UPDATE knowledges SET parse_status='pending' WHERE id=?", wanted.KnowledgeID)
+			case "completed":
+				saveGraphCheckpoint(t, db, 1, "kb", wanted, wanted.ContentHash)
+			case "failed":
+				if err := recordGraphFailure(db, 1, "kb", "grounded-v6", graphRecoveryTaskID(1, "kb", wanted, "grounded-v6"), wanted, "review_required"); err != nil {
+					t.Fatal(err)
+				}
+			case "other tenant":
+				db.Exec("UPDATE chunks SET tenant_id=2 WHERE id='wanted'")
+			case "other kb":
+				db.Exec("UPDATE chunks SET knowledge_base_id='other' WHERE id='wanted'")
+			case "missing":
+				wanted.ID = "absent"
+			}
+			got, err := graphRecoveryCandidates(db, 1, "kb", "grounded-v6", 0, 1, true, false, wanted)
+			if change == "unchanged" {
+				if err != nil || len(got) != 1 || got[0].ID != wanted.ID {
+					t.Fatalf("exact source: %v %v", got, err)
+				}
+			} else if err == nil || len(got) != 0 {
+				t.Fatalf("changed source selected a candidate: %v %v", got, err)
+			}
+		})
+	}
+}
+
+func TestGraphWatcherRequiresVerifiedCoverage(t *testing.T) {
+	for _, test := range []struct {
+		name, report      string
+		complete, blocked bool
+	}{
+		{"all source chunks verified", `{"coverage":{"TextChunks":51,"NativeCompleted":51},"candidate_count":0}`, true, false},
+		{"empty knowledge base", `{"coverage":{"TextChunks":0,"NativeCompleted":0},"candidate_count":0}`, true, false},
+		{"unparsed source excludes remaining candidates", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":0}`, false, true},
+		{"queued work is not complete", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":2,"already_queued":2}`, false, false},
+		{"exhausted retries need attention", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":2,"archived":1}`, false, true},
+		{"old mapping is not extraction", `{"coverage":{"TextChunks":51,"MappedChunks":51,"NativeCompleted":0},"candidate_count":0}`, false, true},
+		{"missing coverage fails closed", `{"candidate_count":0}`, false, true},
+		{"persistent failures survive candidate exclusion", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":0,"failure_count":2}`, false, true},
+		{"failures remain visible while later work progresses", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":1,"failure_count":1}`, false, true},
+		{"stale failure report must not claim complete", `{"coverage":{"TextChunks":51,"NativeCompleted":51},"candidate_count":0,"failure_count":1}`, false, true},
+		{"impossible coverage fails closed", `{"coverage":{"TextChunks":51,"NativeCompleted":52},"candidate_count":1}`, false, true},
+		{"paused queue needs attention", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":2,"queue_paused":true}`, false, true},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var report map[string]any
+			if err := json.Unmarshal([]byte(test.report), &report); err != nil {
+				t.Fatal(err)
+			}
+			complete, blocked := graphBatchStatus(report)
+			if complete != test.complete || blocked != test.blocked {
+				t.Fatalf("got complete=%t blocked=%t; want %t %t", complete, blocked, test.complete, test.blocked)
+			}
+		})
+	}
+}
+
+func TestGraphArchivedReasonDoesNotExposeProviderDetails(t *testing.T) {
+	for _, tc := range []struct{ message, reason string }{
+		{"graph_needs_review: no source-grounded relationships: skip retry for the task", "relation_coverage_review_required"},
+		{"provider rejected secret credential", "archived"},
+		{"read response: context deadline exceeded; credential=secret", "provider_response_timeout"},
+		{"read response: net/http: timeout awaiting response headers", "provider_response_timeout"},
+		{"context deadline exceeded", "task_timeout"},
+		{"context canceled", "task_cancelled"},
+		{"", "archived"},
+	} {
+		if got := graphArchivedReason(tc.message); got != tc.reason {
+			t.Fatalf("got %q; want %q", got, tc.reason)
+		}
+	}
+}
+
+func init() {
+	// The production candidate SQL is exercised unchanged. These deterministic
+	// functions provide PostgreSQL's content-hash operations to the test database.
+	sql.Register("graph_backfill_test", &sqlite3.SQLiteDriver{ConnectHook: func(conn *sqlite3.SQLiteConn) error {
+		if err := conn.RegisterFunc("convert_to", func(content, _ string) string { return content }, true); err != nil {
+			return err
+		}
+		if err := conn.RegisterFunc("sha256", func(content string) []byte { hash := sha256.Sum256([]byte(content)); return hash[:] }, true); err != nil {
+			return err
+		}
+		return conn.RegisterFunc("encode", func(content []byte, _ string) string { return hex.EncodeToString(content) }, true)
+	}})
+}
+
+func graphTestDB(t *testing.T, path string) *gorm.DB {
+	t.Helper()
+	db, err := gorm.Open(sqlite.Dialector{DriverName: "graph_backfill_test", DSN: path}, &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	conn, err := db.DB()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = conn.Close() })
+	if err := db.AutoMigrate(&graphRecoveryFailure{}); err != nil {
+		t.Fatal(err)
+	}
+	for _, statement := range []string{
+		`CREATE TABLE IF NOT EXISTS knowledge_bases(id TEXT,tenant_id INTEGER,deleted_at TEXT)`,
+		`CREATE TABLE IF NOT EXISTS chunks(id TEXT,tenant_id INTEGER,knowledge_base_id TEXT,knowledge_id TEXT,seq_id INTEGER,chunk_index INTEGER,content TEXT,deleted_at TEXT,is_enabled BOOLEAN,chunk_type TEXT)`,
+		`CREATE TABLE IF NOT EXISTS knowledges(id TEXT,tenant_id INTEGER,knowledge_base_id TEXT,deleted_at TEXT,parse_status TEXT)`,
+		`CREATE TABLE IF NOT EXISTS native_graph_records(tenant_id INTEGER,knowledge_base_id TEXT,source_table TEXT,payload JSON)`,
+	} {
+		if err := db.Exec(statement).Error; err != nil {
+			t.Fatal(err)
+		}
+	}
+	return db
+}
+
+func seedGraphChunk(t *testing.T, db *gorm.DB, tenant uint64, kbID, id, content string, seq int) graphRecoveryCandidate {
+	t.Helper()
+	if err := db.Exec(`INSERT INTO knowledge_bases(id,tenant_id) SELECT ?,? WHERE NOT EXISTS(SELECT 1 FROM knowledge_bases WHERE id=? AND tenant_id=?)`, kbID, tenant, kbID, tenant).Error; err != nil {
+		t.Fatal(err)
+	}
+	knowledgeID := "knowledge-" + id
+	if err := db.Exec(`INSERT INTO knowledges(id,tenant_id,knowledge_base_id,parse_status) VALUES(?,?,?,'completed')`, knowledgeID, tenant, kbID).Error; err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Exec(`INSERT INTO chunks(id,tenant_id,knowledge_base_id,knowledge_id,seq_id,chunk_index,content,is_enabled,chunk_type) VALUES(?,?,?,?,?,0,?,true,'text')`, id, tenant, kbID, knowledgeID, seq, content).Error; err != nil {
+		t.Fatal(err)
+	}
+	hash := sha256.Sum256([]byte(content))
+	return graphRecoveryCandidate{ID: id, KnowledgeID: knowledgeID, SeqID: int64(seq), ContentHash: hex.EncodeToString(hash[:])}
+}
+
+type graphTestReader struct {
+	info   *asynq.TaskInfo
+	err    error
+	reads  int
+	onRead func()
+}
+
+func (r *graphTestReader) GetTaskInfo(_, _ string) (*asynq.TaskInfo, error) {
+	r.reads++
+	if r.onRead != nil {
+		r.onRead()
+	}
+	return r.info, r.err
+}
+
+func seedGraphReconciliation(t *testing.T) (*gorm.DB, graphRecoveryCandidate, *graphTestReader) {
+	t.Helper()
+	db := graphTestDB(t, filepath.Join(t.TempDir(), "reconcile.db"))
+	c := seedGraphChunk(t, db, 1, "kb", "source", "reviewed source", 1)
+	id := graphRecoveryTaskID(1, "kb", c, "grounded-v6")
+	if ok, err := reserveGraphDispatch(db, 1, "kb", "grounded-v6", id, c); err != nil || !ok {
+		t.Fatalf("reserve: %t %v", ok, err)
+	}
+	payload, err := json.Marshal(types.ExtractChunkPayload{TenantID: 1, ChunkID: c.ID, ExpectedSource: &types.GraphSourceIdentity{KnowledgeBaseID: "kb", KnowledgeID: c.KnowledgeID, ContentSHA256: c.ContentHash}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return db, c, &graphTestReader{info: &asynq.TaskInfo{ID: id, Queue: types.QueueGraph, Type: types.TypeChunkExtract, Payload: payload, State: asynq.TaskStateArchived, LastErr: "read response: context deadline exceeded; credential=must-not-leak", LastFailedAt: time.Date(2026, 9, 20, 8, 46, 33, 0, time.UTC)}}
+}
+
+func graphReadLedger(t *testing.T, db *gorm.DB, id string) graphRecoveryFailure {
+	t.Helper()
+	var ledger graphRecoveryFailure
+	if err := db.First(&ledger, "task_id=?", id).Error; err != nil {
+		t.Fatal(err)
+	}
+	return ledger
+}
+
+func TestGraphReconciliationRequiresExactNonDispatchMode(t *testing.T) {
+	for _, tc := range []struct {
+		name, chunk, revision, retryReason string
+		retry, valid                       bool
+	}{
+		{name: "exact", chunk: "chunk", revision: "grounded-v6", valid: true},
+		{name: "missing source", revision: "grounded-v6"},
+		{name: "missing revision", chunk: "chunk"},
+		{name: "blank revision", chunk: "chunk", revision: " "},
+		{name: "retry requested", chunk: "chunk", revision: "grounded-v6", retry: true},
+		{name: "retry reason supplied", chunk: "chunk", revision: "grounded-v6", retryReason: "retry attempt"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			err := graphValidateReconciliation(true, tc.chunk, tc.revision, tc.retry, tc.retryReason)
+			if (err == nil) != tc.valid {
+				t.Fatalf("valid=%t err=%v", tc.valid, err)
+			}
+		})
+	}
+}
+
+func TestGraphReconciliationCLIRejectsMixedModesBeforeOpeningDatabase(t *testing.T) {
+	const chunk = "edf40fb3-f09d-4532-a6fa-533408fe19cb"
+	const knowledge = "c3db6397-6b50-4d1e-a667-2e3efe503cc7"
+	const hash = "63bdc854af31ea50dd0360d93a8b2340391e15e5a1c251db714ca1e9533b2e5b"
+	for _, tc := range []struct {
+		name, flag, value, want string
+	}{
+		{"retry", "--retry-failed", "true", "no retry flags"},
+		{"retry reason", "--retry-reason", "previous investigation", "no retry flags"},
+		{"cursor", "--after", "1", "no cursor"},
+		{"batch", "--limit", "2", "limit=1"},
+		{"parallel", "--max-outstanding", "2", "max-outstanding=1"},
+		{"missing hash", "--source-sha256", "", "source SHA-256"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			originalFlags, originalArgs := flag.CommandLine, os.Args
+			defer func() { flag.CommandLine, os.Args = originalFlags, originalArgs }()
+			flag.CommandLine = flag.NewFlagSet("reconcile-test", flag.ContinueOnError)
+			t.Setenv("GRAPH_BACKFILL_REVISION", "grounded-v6")
+			t.Setenv("GRAPH_BACKFILL_DSN", "invalid connection must not be opened")
+			os.Args = []string{"reconcile-test", "--reconcile-only", "--tenant", "10059", "--kb", "16c3e59c-4bb6-41cf-ae0a-6d0941b3d032", "--chunk", chunk, "--knowledge", knowledge, "--source-sha256", hash, "--limit", "1", "--max-outstanding", "1", tc.flag + "=" + tc.value}
+			if err := runGraphRecovery(); err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("expected validation %q before connection, got %v", tc.want, err)
+			}
+		})
+	}
+}
+
+func TestGraphReconciliationSanitizesPostgresCheckpointErrors(t *testing.T) {
+	conn, mock, err := sqlmock.New()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	db, err := gorm.Open(postgres.New(postgres.Config{Conn: conn}), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	mock.ExpectQuery("SELECT .* AS current.* FROM native_graph_backfill_failures").WithArgs("task").WillReturnError(errors.New(`invalid input syntax for type integer: "private-source-value"`))
+	current, complete, err := graphReconcileSourceStatus(db, "task")
+	if err == nil || current || complete || strings.Contains(err.Error(), "private-source-value") {
+		t.Fatalf("unsafe checkpoint query outcome: %t %t %v", current, complete, err)
+	}
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestGraphReconciliationPreviewThenApplyIsBoundedAndIdempotent(t *testing.T) {
+	db, c, reader := seedGraphReconciliation(t)
+	original := graphReadLedger(t, db, reader.info.ID)
+	other := seedGraphChunk(t, db, 1, "kb", "other", "untouched source", 2)
+	otherID := graphRecoveryTaskID(1, "kb", other, "grounded-v6")
+	if _, err := reserveGraphDispatch(db, 1, "kb", "grounded-v6", otherID, other); err != nil {
+		t.Fatal(err)
+	}
+	unrelated := graphReadLedger(t, db, otherID)
+	preview, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, false)
+	if err != nil || !preview.WouldUpdate || preview.LedgerUpdated || !preview.Blocked || preview.Reason != "provider_response_timeout" || preview.ProposedState != "failed" {
+		t.Fatalf("preview: %+v %v", preview, err)
+	}
+	if got := graphReadLedger(t, db, reader.info.ID); !reflect.DeepEqual(got, original) {
+		t.Fatal("preview changed durable state")
+	}
+	result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
+	if err != nil || !result.LedgerUpdated || result.Enqueued != 0 || result.Retried != 0 || result.ModelCalls != 0 {
+		t.Fatalf("apply: %+v %v", result, err)
+	}
+	updated := graphReadLedger(t, db, reader.info.ID)
+	if updated.State != "failed" || updated.Reason != "provider_response_timeout" || !updated.FirstFailedAt.Equal(reader.info.LastFailedAt) || updated.ManualRetries != 0 {
+		t.Fatalf("failure not durably recorded: %+v", updated)
+	}
+	encoded, _ := json.Marshal(result)
+	if strings.Contains(string(encoded), "must-not-leak") {
+		t.Fatal("report exposed provider details")
+	}
+	result, err = reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
+	if err != nil || result.WouldUpdate || result.LedgerUpdated || !reflect.DeepEqual(updated, graphReadLedger(t, db, reader.info.ID)) {
+		t.Fatalf("idempotence: %+v %v", result, err)
+	}
+	if !reflect.DeepEqual(unrelated, graphReadLedger(t, db, otherID)) {
+		t.Fatal("reconciliation changed another source ledger")
+	}
+}
+
+func TestGraphReconciliationRefinesArchivedReasonWithoutLosingHistory(t *testing.T) {
+	db, c, reader := seedGraphReconciliation(t)
+	failedAt := time.Date(2026, 9, 18, 1, 2, 3, 0, time.UTC)
+	if err := db.Model(&graphRecoveryFailure{}).Where("task_id=?", reader.info.ID).Updates(map[string]any{"state": "failed", "reason": "archived", "first_failed_at": failedAt, "manual_retries": 1, "retry_reason": "previous authorized recovery"}).Error; err != nil {
+		t.Fatal(err)
+	}
+	result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
+	after := graphReadLedger(t, db, reader.info.ID)
+	if err != nil || !result.LedgerUpdated || after.Reason != "provider_response_timeout" || !after.FirstFailedAt.Equal(failedAt) || after.ManualRetries != 1 || after.RetryReason != "previous authorized recovery" {
+		t.Fatalf("failure history changed: %+v %+v %v", result, after, err)
+	}
+}
+
+func TestGraphReconciliationRequiresTerminalQueueAndPinnedIdentity(t *testing.T) {
+	for _, change := range []string{"missing task", "missing queue", "nil task", "redis error", "active", "pending", "retry", "payload", "top document", "type", "queue", "task id", "max retry", "source hash", "source disabled", "source deleted", "document deleted", "unparsed", "kb deleted", "tenant", "document identity", "resolved", "superseded"} {
+		t.Run(change, func(t *testing.T) {
+			db, c, reader := seedGraphReconciliation(t)
+			id := reader.info.ID
+			switch change {
+			case "missing task":
+				reader.err = asynq.ErrTaskNotFound
+			case "missing queue":
+				reader.err = asynq.ErrQueueNotFound
+			case "nil task":
+				reader.info = nil
+			case "redis error":
+				reader.err = errors.New("secret connection details")
+			case "active":
+				reader.info.State = asynq.TaskStateActive
+			case "pending":
+				reader.info.State = asynq.TaskStatePending
+			case "retry":
+				reader.info.State = asynq.TaskStateRetry
+			case "payload":
+				reader.info.Payload = []byte(`{"tenant_id":2}`)
+			case "top document":
+				var p types.ExtractChunkPayload
+				_ = json.Unmarshal(reader.info.Payload, &p)
+				p.KnowledgeID = "other-document"
+				reader.info.Payload, _ = json.Marshal(p)
+			case "type":
+				reader.info.Type = "other"
+			case "queue":
+				reader.info.Queue = "other"
+			case "task id":
+				reader.info.ID = "other"
+			case "max retry":
+				reader.info.MaxRetry = 1
+			case "source hash":
+				db.Exec("UPDATE chunks SET content='changed'")
+			case "source disabled":
+				db.Exec("UPDATE chunks SET is_enabled=false")
+			case "source deleted":
+				db.Exec("UPDATE chunks SET deleted_at='2026-09-20'")
+			case "document deleted":
+				db.Exec("UPDATE knowledges SET deleted_at='2026-09-20'")
+			case "unparsed":
+				db.Exec("UPDATE knowledges SET parse_status='processing'")
+			case "kb deleted":
+				db.Exec("UPDATE knowledge_bases SET deleted_at='2026-09-20'")
+			case "tenant":
+				db.Exec("UPDATE chunks SET tenant_id=2")
+			case "document identity":
+				c.KnowledgeID = "other-document"
+			case "resolved", "superseded":
+				db.Model(&graphRecoveryFailure{}).Where("task_id=?", id).Update("state", change)
+			}
+			before := graphReadLedger(t, db, id)
+			result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
+			if (err != nil) != (change == "redis error") || result.LedgerUpdated || result.WouldUpdate || result.Enqueued != 0 || result.Retried != 0 || result.ModelCalls != 0 {
+				t.Fatalf("unsafe reconcile: %+v %v", result, err)
+			}
+			if err != nil && strings.Contains(err.Error(), "secret") {
+				t.Fatal("inspection error exposed connection details")
+			}
+			if change != "active" && change != "pending" && change != "retry" && !result.Blocked {
+				t.Fatalf("unsafe case not blocked: %+v", result)
+			}
+			if !reflect.DeepEqual(before, graphReadLedger(t, db, id)) {
+				t.Fatal("blocked reconciliation changed ledger")
+			}
+		})
+	}
+}
+
+func TestGraphReconciliationRequiresCurrentNonemptyCheckpointAndPreservesHistory(t *testing.T) {
+	for _, checkpoint := range []string{"current", "stale hash", "other tenant", "other kb", "other document", "empty", "none", "missing redis"} {
+		t.Run(checkpoint, func(t *testing.T) {
+			db, c, reader := seedGraphReconciliation(t)
+			reader.info.State = asynq.TaskStateCompleted
+			failedAt := time.Date(2026, 9, 18, 0, 0, 0, 0, time.UTC)
+			if err := db.Model(&graphRecoveryFailure{}).Where("task_id=?", reader.info.ID).Updates(map[string]any{"state": "failed", "reason": "provider_response_timeout", "first_failed_at": failedAt, "manual_retries": 1, "retry_reason": "previous authorized recovery"}).Error; err != nil {
+				t.Fatal(err)
+			}
+			if checkpoint != "none" {
+				hash := c.ContentHash
+				if checkpoint == "stale hash" {
+					hash = "old-hash"
+				}
+				saveGraphCheckpoint(t, db, 1, "kb", c, hash)
+			}
+			switch checkpoint {
+			case "other tenant":
+				db.Exec("UPDATE native_graph_records SET tenant_id=2")
+			case "other kb":
+				db.Exec("UPDATE native_graph_records SET knowledge_base_id='other'")
+			case "other document":
+				db.Exec(`UPDATE native_graph_records SET payload=json_set(payload,'$.native_knowledge_id','other')`)
+			case "empty":
+				db.Exec(`UPDATE native_graph_records SET payload=json_set(payload,'$.relation_count',0)`)
+			case "missing redis":
+				reader.err = asynq.ErrTaskNotFound
+			}
+			before := graphReadLedger(t, db, reader.info.ID)
+			result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
+			if err != nil || result.LedgerUpdated != (checkpoint == "current") || result.Blocked == (checkpoint == "current") {
+				t.Fatalf("checkpoint reconciliation: %+v %v", result, err)
+			}
+			after := graphReadLedger(t, db, reader.info.ID)
+			if checkpoint == "current" && after.State != "resolved" {
+				t.Fatal("verified checkpoint not resolved")
+			}
+			if after.Reason != before.Reason || !after.FirstFailedAt.Equal(before.FirstFailedAt) || after.ManualRetries != before.ManualRetries || after.RetryReason != before.RetryReason {
+				t.Fatal("reconciliation discarded prior failure history")
+			}
+		})
+	}
+}
+
+func TestGraphReconciliationRechecksSourceAfterQueueRead(t *testing.T) {
+	db, c, reader := seedGraphReconciliation(t)
+	before := graphReadLedger(t, db, reader.info.ID)
+	reader.onRead = func() { db.Exec("UPDATE chunks SET content='concurrent edit'") }
+	result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
+	if err != nil || !result.Blocked || result.LedgerUpdated || result.Reason != "source_changed_or_unavailable" || !reflect.DeepEqual(before, graphReadLedger(t, db, reader.info.ID)) {
+		t.Fatalf("source edit lost: %+v %v", result, err)
+	}
+}
+
+func TestGraphReconciliationCannotOverrideManualSemanticRejection(t *testing.T) {
+	db, c, reader := seedGraphReconciliation(t)
+	saveGraphCheckpoint(t, db, 1, "kb", c, c.ContentHash)
+	if err := recordGraphFailure(db, 1, "kb", "grounded-v6", reader.info.ID, c, "semantic_review_rejected_product_scope"); err != nil {
+		t.Fatal(err)
+	}
+	before := graphReadLedger(t, db, reader.info.ID)
+	result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
+	if err != nil || !result.Blocked || result.LedgerUpdated || result.Reason != "manual_semantic_review_required" || !reflect.DeepEqual(before, graphReadLedger(t, db, reader.info.ID)) {
+		t.Fatalf("manual rejection was overridden: %+v %v", result, err)
+	}
+}
+
+func TestGraphReconciliationAtomicUpdateChecksConcurrentChanges(t *testing.T) {
+	for _, change := range []string{"source", "checkpoint", "ledger", "ledger identity"} {
+		t.Run(change, func(t *testing.T) {
+			db, c, reader := seedGraphReconciliation(t)
+			before := graphReadLedger(t, db, reader.info.ID)
+			fired := false
+			if err := db.Callback().Update().Before("gorm:update").Register("test:concurrent_change", func(tx *gorm.DB) {
+				if fired {
+					return
+				}
+				fired = true
+				tx = tx.Session(&gorm.Session{NewDB: true})
+				switch change {
+				case "source":
+					tx.Exec("UPDATE chunks SET content='concurrent edit'")
+				case "checkpoint":
+					saveGraphCheckpoint(t, tx, 1, "kb", c, c.ContentHash)
+				case "ledger":
+					tx.Exec("UPDATE native_graph_backfill_failures SET state='retry_requested', manual_retries=1 WHERE task_id=?", reader.info.ID)
+				case "ledger identity":
+					tx.Exec("UPDATE native_graph_backfill_failures SET knowledge_id='other-document' WHERE task_id=?", reader.info.ID)
+				}
+			}); err != nil {
+				t.Fatal(err)
+			}
+			result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
+			if err != nil || !fired || !result.Blocked || result.LedgerUpdated || result.Reason != "reconciliation_state_changed" {
+				t.Fatalf("concurrent change overwritten: %+v %v", result, err)
+			}
+			after := graphReadLedger(t, db, reader.info.ID)
+			if after.Reason != before.Reason || !after.FirstFailedAt.Equal(before.FirstFailedAt) {
+				t.Fatal("failed compare-and-update mutated failure history")
+			}
+		})
+	}
+}
+
+func saveGraphCheckpoint(t *testing.T, db *gorm.DB, tenant uint64, kbID string, c graphRecoveryCandidate, hash string) {
+	t.Helper()
+	payload, err := json.Marshal(map[string]string{"origin": "native_extraction_v1", "extraction_status": "complete", "relation_count": "1", "native_chunk_id": c.ID, "native_knowledge_id": c.KnowledgeID, "source_content_hash": hash})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Exec(`INSERT INTO native_graph_records(tenant_id,knowledge_base_id,source_table,payload) VALUES(?,?,'knowledgegraphchunkstate',?)`, tenant, kbID, string(payload)).Error; err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestGraphFailureAllowsLaterCandidateAfterRestart(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "recovery.db")
+	db := graphTestDB(t, path)
+	first := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
+	second := seedGraphChunk(t, db, 1, "kb", "second", "second content", 2)
+	taskID := graphRecoveryTaskID(1, "kb", first, "revision")
+	reserved, err := reserveGraphDispatch(db, 1, "kb", "revision", taskID, first)
+	if err != nil || !reserved {
+		t.Fatalf("reserve: %t %v", reserved, err)
+	}
+	if err := recordGraphFailure(db, 1, "kb", "revision", taskID, first, "archived"); err != nil {
+		t.Fatal(err)
+	}
+	sqlDB, _ := db.DB()
+	if err := sqlDB.Close(); err != nil {
+		t.Fatal(err)
+	}
+	db = graphTestDB(t, path)
+	candidates, err := graphRecoveryCandidates(db, 1, "kb", "revision", 0, 1, true, false)
+	if err != nil || len(candidates) != 1 || candidates[0].ID != second.ID {
+		t.Fatalf("blocked behind archived task: %+v %v", candidates, err)
+	}
+	failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", false)
+	if err != nil || len(failures) != 1 || failures[0].Reason != "archived" || failures[0].FirstFailedAt.IsZero() {
+		t.Fatalf("lost failure accounting: %+v %v", failures, err)
+	}
+	// A new extraction revision and another tenant must not inherit this exclusion.
+	candidates, err = graphRecoveryCandidates(db, 1, "kb", "new-revision", 0, 1, true, false)
+	if err != nil || len(candidates) != 1 || candidates[0].ID != first.ID {
+		t.Fatalf("revision scope: %+v %v", candidates, err)
+	}
+	seedGraphChunk(t, db, 2, "kb", "first", "first content", 1)
+	candidates, err = graphRecoveryCandidates(db, 2, "kb", "revision", 0, 1, true, false)
+	if err != nil || len(candidates) != 1 || candidates[0].ID != first.ID {
+		t.Fatalf("tenant scope: %+v %v", candidates, err)
+	}
+}
+
+func TestGraphLegacyEmptyCheckpointIsNotCompletedOrResolved(t *testing.T) {
+	db := graphTestDB(t, filepath.Join(t.TempDir(), "recovery.db"))
+	c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
+	saveGraphCheckpoint(t, db, 1, "kb", c, c.ContentHash)
+	if err := db.Exec(`UPDATE native_graph_records SET payload=json_set(payload,'$.relation_count',0)`).Error; err != nil {
+		t.Fatal(err)
+	}
+	taskID := graphRecoveryTaskID(1, "kb", c, "revision")
+	if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "empty_result"); err != nil {
+		t.Fatal(err)
+	}
+	complete, err := graphCandidateCompleted(db, 1, "kb", c)
+	if err != nil || complete {
+		t.Fatalf("empty checkpoint completed: %t %v", complete, err)
+	}
+	failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", true)
+	if err != nil || len(failures) != 1 {
+		t.Fatalf("empty checkpoint resolved failure: %+v %v", failures, err)
+	}
+	candidates, err := graphRecoveryCandidates(db, 1, "kb", "new-revision", 0, 1, true, false)
+	if err != nil || len(candidates) != 1 || candidates[0].ID != c.ID {
+		t.Fatalf("empty checkpoint skipped rebuild: %+v %v", candidates, err)
+	}
+}
+
+func TestGraphDispatchDoesNotRepeatAfterRedisRecordDisappears(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "recovery.db")
+	db := graphTestDB(t, path)
+	c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
+	taskID := graphRecoveryTaskID(1, "kb", c, "revision")
+	reserved, err := reserveGraphDispatch(db, 1, "kb", "revision", taskID, c)
+	if err != nil || !reserved {
+		t.Fatalf("reserve: %t %v", reserved, err)
+	}
+	sqlDB, _ := db.DB()
+	if err := sqlDB.Close(); err != nil {
+		t.Fatal(err)
+	}
+	db = graphTestDB(t, path)
+	reserved, err = reserveGraphDispatch(db, 1, "kb", "revision", taskID, c)
+	if err != nil || reserved {
+		t.Fatalf("replayed unknown dispatch: %t %v", reserved, err)
+	}
+	failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", false)
+	if err != nil || len(failures) != 1 || failures[0].Reason != "task_missing_after_dispatch" {
+		t.Fatalf("lost missing task: %+v %v", failures, err)
+	}
+}
+
+type graphTestQueue struct {
+	runs int
+	err  error
+}
+
+func (q *graphTestQueue) RunTask(_, _ string) error { q.runs++; return q.err }
+
+func TestGraphManualRetryIsBoundedAndAuditedAcrossRestart(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "recovery.db")
+	db := graphTestDB(t, path)
+	c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
+	taskID := graphRecoveryTaskID(1, "kb", c, "revision")
+	if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
+		t.Fatal(err)
+	}
+	queue := &graphTestQueue{}
+	if retried, err := graphFailureRetry(db, queue, taskID, "too short", 1); err == nil || retried || queue.runs != 0 {
+		t.Fatal("unaudited retry was accepted")
+	}
+	if retried, err := graphFailureRetry(db, queue, taskID, "Provider configuration investigated and fixed", 0); err != nil || retried || queue.runs != 0 {
+		t.Fatal("queue cap was ignored")
+	}
+	retried, err := graphFailureRetry(db, queue, taskID, "Provider configuration investigated and fixed", 1)
+	if err != nil || !retried || queue.runs != 1 {
+		t.Fatalf("first retry: %t %v %+v", retried, err, queue)
+	}
+	sqlDB, _ := db.DB()
+	if err := sqlDB.Close(); err != nil {
+		t.Fatal(err)
+	}
+	db = graphTestDB(t, path)
+	if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
+		t.Fatal(err)
+	}
+	retried, err = graphFailureRetry(db, queue, taskID, "Retry repeatedly after restart", 1)
+	if err != nil || retried || queue.runs != 1 {
+		t.Fatalf("retry was repeated: %t %v %+v", retried, err, queue)
+	}
+	failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", false)
+	if err != nil || len(failures) != 1 || failures[0].ManualRetries != 1 || failures[0].RetryReason != "Provider configuration investigated and fixed" {
+		t.Fatalf("lost retry audit: %+v %v", failures, err)
+	}
+}
+
+func TestGraphRetryDispatchErrorKeepsReservation(t *testing.T) {
+	db := graphTestDB(t, filepath.Join(t.TempDir(), "recovery.db"))
+	c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
+	taskID := graphRecoveryTaskID(1, "kb", c, "revision")
+	if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
+		t.Fatal(err)
+	}
+	queue := &graphTestQueue{err: errors.New("lost response")}
+	if retried, err := graphFailureRetry(db, queue, taskID, "Investigated timeout and fixed provider", 1); err == nil || retried {
+		t.Fatal("dispatch error not propagated")
+	}
+	if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
+		t.Fatal(err)
+	}
+	if retried, err := graphFailureRetry(db, queue, taskID, "Investigated timeout and fixed provider", 1); err != nil || retried || queue.runs != 1 {
+		t.Fatal("ambiguous dispatch was replayed")
+	}
+}
+
+func TestGraphFailureReconciliationRequiresCurrentNativeCheckpoint(t *testing.T) {
+	db := graphTestDB(t, filepath.Join(t.TempDir(), "recovery.db"))
+	c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
+	taskID := graphRecoveryTaskID(1, "kb", c, "revision")
+	if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "completed_without_native_checkpoint"); err != nil {
+		t.Fatal(err)
+	}
+	saveGraphCheckpoint(t, db, 1, "kb", c, "stale-content-hash")
+	complete, err := graphCandidateCompleted(db, 1, "kb", c)
+	if err != nil || complete {
+		t.Fatalf("stale checkpoint was accepted: %t %v", complete, err)
+	}
+	failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", true)
+	if err != nil || len(failures) != 1 {
+		t.Fatalf("prematurely resolved: %+v %v", failures, err)
+	}
+	saveGraphCheckpoint(t, db, 1, "kb", c, c.ContentHash)
+	complete, err = graphCandidateCompleted(db, 1, "kb", c)
+	if err != nil || !complete {
+		t.Fatalf("racing checkpoint not detected: %t %v", complete, err)
+	}
+	failures, err = graphUnresolvedFailures(db, 1, "kb", "revision", true)
+	if err != nil || len(failures) != 0 {
+		t.Fatalf("completed source remained blocked: %+v %v", failures, err)
+	}
+	var historical graphRecoveryFailure
+	if err := db.First(&historical, "task_id=?", taskID).Error; err != nil || historical.State != "resolved" || historical.Reason != "completed_without_native_checkpoint" {
+		t.Fatalf("failure history lost: %+v %v", historical, err)
+	}
+}
+
+func TestGraphChangedSourceSupersedesOldFailure(t *testing.T) {
+	db := graphTestDB(t, filepath.Join(t.TempDir(), "recovery.db"))
+	c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
+	taskID := graphRecoveryTaskID(1, "kb", c, "revision")
+	if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Exec("UPDATE chunks SET content='new content'").Error; err != nil {
+		t.Fatal(err)
+	}
+	failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", true)
+	if err != nil || len(failures) != 0 {
+		t.Fatalf("obsolete failure blocked new content: %+v %v", failures, err)
+	}
+	candidates, err := graphRecoveryCandidates(db, 1, "kb", "revision", 0, 1, true, false)
+	if err != nil || len(candidates) != 1 || candidates[0].ContentHash == c.ContentHash {
+		t.Fatalf("new content unavailable: %+v %v", candidates, err)
+	}
+	var historical graphRecoveryFailure
+	if err := db.First(&historical, "task_id=?", taskID).Error; err != nil || historical.State != "superseded" {
+		t.Fatalf("history not preserved: %+v %v", historical, err)
+	}
+}
+
+func TestGraphQueueWindowCountsEveryOutstandingState(t *testing.T) {
+	for _, test := range []struct {
+		name        string
+		info        *asynq.QueueInfo
+		limit, want int
+	}{
+		{"empty", &asynq.QueueInfo{}, 2, 2},
+		{"pending", &asynq.QueueInfo{Pending: 1}, 2, 1},
+		{"active", &asynq.QueueInfo{Active: 2}, 2, 0},
+		{"scheduled", &asynq.QueueInfo{Scheduled: 2}, 2, 0},
+		{"retry", &asynq.QueueInfo{Retry: 2}, 2, 0},
+		{"aggregating", &asynq.QueueInfo{Aggregating: 2}, 2, 0},
+		{"combined", &asynq.QueueInfo{Active: 1, Pending: 1, Retry: 1}, 4, 1},
+		{"exceeded", &asynq.QueueInfo{Active: 4}, 2, 0},
+		{"terminal records do not block later work", &asynq.QueueInfo{Archived: 100, Completed: 100}, 2, 2},
+		{"paused", &asynq.QueueInfo{Paused: true}, 2, 0},
+		{"unknown", nil, 2, 0},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			if got := graphQueueSlots(test.info, test.limit); got != test.want {
+				t.Fatalf("slots=%d want=%d", got, test.want)
+			}
+		})
+	}
+}
+
+func TestGraphTaskIdentityScopesContentTenantAndRevision(t *testing.T) {
+	c := graphRecoveryCandidate{ID: "chunk", ContentHash: "sha256"}
+	id := graphRecoveryTaskID(1, "kb", c, "r1")
+	if id != graphRecoveryTaskID(1, "kb", c, "r1") {
+		t.Fatal("identity changed across restart")
+	}
+	for _, other := range []string{
+		graphRecoveryTaskID(2, "kb", c, "r1"), graphRecoveryTaskID(1, "other", c, "r1"), graphRecoveryTaskID(1, "kb", c, "r2"),
+		graphRecoveryTaskID(1, "kb", graphRecoveryCandidate{ID: "chunk", ContentHash: "changed"}, "r1"),
+	} {
+		if id == other {
+			t.Fatal("unrelated attempts share task identity")
+		}
+	}
+}
+
+func TestGraphResumeCursorDoesNotSkipUndispatchedOrUnverifiedWork(t *testing.T) {
+	candidates := []graphRecoveryCandidate{{ID: "first", SeqID: 10}, {ID: "second", SeqID: 20}, {ID: "third", SeqID: 30}}
+	for _, test := range []struct {
+		name    string
+		settled map[string]bool
+		want    int64
+	}{
+		{"queued but not yet verified", map[string]bool{}, 5},
+		{"second deferred by queue limit", map[string]bool{"first": true, "third": true}, 10},
+		{"all durably handled", map[string]bool{"first": true, "second": true, "third": true}, 30},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			if got := graphResumeCursor(5, candidates, test.settled); got != test.want {
+				t.Fatalf("cursor=%d want=%d", got, test.want)
+			}
+		})
+	}
+}