WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/application/repository/task_queue_test.go

@@ -0,0 +1,837 @@
+package repository
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/Tencent/WeKnora/internal/types/interfaces"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+// taskPendingOpsTestDDL mirrors the production schema in
+// migrations/versioned/000041_task_queue_and_wiki_indexes.up.sql but uses
+// SQLite-compatible types. INTEGER PRIMARY KEY AUTOINCREMENT preserves
+// the monotonically-increasing ID semantics PeekBatch/cursor pagination
+// rely on. JSONB → TEXT is fine since GORM round-trips json.RawMessage
+// as bytes either way.
+const taskPendingOpsTestDDL = `
+CREATE TABLE IF NOT EXISTS task_pending_ops (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    tenant_id   INTEGER NOT NULL,
+    task_type   VARCHAR(64) NOT NULL,
+    scope       VARCHAR(32) NOT NULL,
+    scope_id    VARCHAR(64) NOT NULL,
+    op          VARCHAR(32) NOT NULL,
+    dedup_key   VARCHAR(128) NOT NULL DEFAULT '',
+    payload     TEXT NOT NULL DEFAULT '{}',
+    fail_count  INTEGER NOT NULL DEFAULT 0,
+    enqueued_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+    claimed_at  DATETIME
+);
+`
+
+const taskDeadLettersTestDDL = `
+CREATE TABLE IF NOT EXISTS task_dead_letters (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    tenant_id   INTEGER NOT NULL,
+    task_type   VARCHAR(64) NOT NULL,
+    scope       VARCHAR(32) NOT NULL,
+    scope_id    VARCHAR(64) NOT NULL,
+    related_id  VARCHAR(64) NOT NULL DEFAULT '',
+    payload     TEXT NOT NULL,
+    last_error  TEXT NOT NULL DEFAULT '',
+    fail_count  INTEGER NOT NULL,
+    failed_at   DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+`
+
+const taskQueueKnowledgeBaseTestDDL = `
+CREATE TABLE IF NOT EXISTS knowledge_bases (
+    id         VARCHAR(64) PRIMARY KEY,
+    tenant_id  INTEGER NOT NULL,
+    deleted_at DATETIME
+);
+`
+
+const taskQueueKnowledgeTestDDL = `
+CREATE TABLE IF NOT EXISTS knowledges (
+    id                     VARCHAR(64) PRIMARY KEY,
+    tenant_id              INTEGER NOT NULL,
+    knowledge_base_id      VARCHAR(64) NOT NULL,
+    parse_status           VARCHAR(32) NOT NULL,
+    pending_subtasks_count INTEGER NOT NULL DEFAULT 0,
+    updated_at             DATETIME,
+    deleted_at             DATETIME
+);
+`
+
+func setupTaskQueueTestDB(t *testing.T) *gorm.DB {
+	t.Helper()
+	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+	require.NoError(t, err)
+	require.NoError(t, db.Exec(taskPendingOpsTestDDL).Error)
+	require.NoError(t, db.Exec(taskDeadLettersTestDDL).Error)
+	return db
+}
+
+func makePendingOp(taskType, scope, scopeID, op, dedup string, payload []byte) *types.TaskPendingOp {
+	return &types.TaskPendingOp{
+		TenantID: 1,
+		TaskType: taskType,
+		Scope:    scope,
+		ScopeID:  scopeID,
+		Op:       op,
+		DedupKey: dedup,
+		Payload:  payload,
+	}
+}
+
+func setupFinalizingPendingOpTest(t *testing.T) (*gorm.DB, interfaces.TaskPendingOpsFinalizingSeeder) {
+	t.Helper()
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	seeder, ok := repo.(interfaces.TaskPendingOpsFinalizingSeeder)
+	require.True(t, ok, "task pending repository must support atomic finalizing handoff")
+	require.NoError(t, db.Exec(taskQueueKnowledgeBaseTestDDL).Error)
+	require.NoError(t, db.Exec(taskQueueKnowledgeTestDDL).Error)
+	require.NoError(t, db.Exec(
+		`INSERT INTO knowledge_bases (id, tenant_id) VALUES ('kb-1', 1)`,
+	).Error)
+	require.NoError(t, db.Exec(
+		`INSERT INTO knowledges (id, tenant_id, knowledge_base_id, parse_status) VALUES ('knowledge-1', 1, 'kb-1', ?)`,
+		types.ParseStatusProcessing,
+	).Error)
+	return db, seeder
+}
+
+func TestTaskPendingOps_SeedKnowledgeFinalizingWithPendingOpCommitsTogether(t *testing.T) {
+	db, seeder := setupFinalizingPendingOpTest(t)
+	op := makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-1", "ingest", "knowledge-1", []byte(`{}`))
+
+	promoted, err := seeder.SeedKnowledgeFinalizingWithPendingOp(
+		context.Background(), "knowledge-1", 3, op,
+	)
+
+	require.NoError(t, err)
+	require.True(t, promoted)
+	var knowledge types.Knowledge
+	require.NoError(t, db.Select("parse_status", "pending_subtasks_count").First(&knowledge, "id = ?", "knowledge-1").Error)
+	assert.Equal(t, types.ParseStatusFinalizing, knowledge.ParseStatus)
+	assert.Equal(t, 3, knowledge.PendingSubtasksCount)
+	var count int64
+	require.NoError(t, db.Model(&types.TaskPendingOp{}).Where("dedup_key = ?", "knowledge-1").Count(&count).Error)
+	assert.Equal(t, int64(1), count)
+}
+
+func TestTaskPendingOps_SeedKnowledgeFinalizingRollsBackWhenPendingOpInsertFails(t *testing.T) {
+	db, seeder := setupFinalizingPendingOpTest(t)
+	require.NoError(t, db.Exec(`DROP TABLE task_pending_ops`).Error)
+	op := makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-1", "ingest", "knowledge-1", []byte(`{}`))
+
+	promoted, err := seeder.SeedKnowledgeFinalizingWithPendingOp(
+		context.Background(), "knowledge-1", 3, op,
+	)
+
+	require.Error(t, err)
+	assert.False(t, promoted)
+	var knowledge types.Knowledge
+	require.NoError(t, db.Select("parse_status", "pending_subtasks_count").First(&knowledge, "id = ?", "knowledge-1").Error)
+	assert.Equal(t, types.ParseStatusProcessing, knowledge.ParseStatus)
+	assert.Zero(t, knowledge.PendingSubtasksCount)
+}
+
+func TestTaskPendingOps_SeedKnowledgeFinalizingSkipsNonProcessingKnowledge(t *testing.T) {
+	db, seeder := setupFinalizingPendingOpTest(t)
+	require.NoError(t, db.Model(&types.Knowledge{}).
+		Where("id = ?", "knowledge-1").
+		Update("parse_status", types.ParseStatusCancelled).Error)
+	op := makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-1", "ingest", "knowledge-1", []byte(`{}`))
+
+	promoted, err := seeder.SeedKnowledgeFinalizingWithPendingOp(
+		context.Background(), "knowledge-1", 3, op,
+	)
+
+	require.NoError(t, err)
+	assert.False(t, promoted)
+	var count int64
+	require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&count).Error)
+	assert.Zero(t, count)
+}
+
+func TestTaskPendingOps_SeedKnowledgeFinalizingSkipsDeletedKnowledgeBase(t *testing.T) {
+	db, seeder := setupFinalizingPendingOpTest(t)
+	require.NoError(t, db.Exec(
+		`UPDATE knowledge_bases SET deleted_at = ? WHERE id = ?`, time.Now(), "kb-1",
+	).Error)
+	op := makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-1", "ingest", "knowledge-1", []byte(`{}`))
+
+	promoted, err := seeder.SeedKnowledgeFinalizingWithPendingOp(
+		context.Background(), "knowledge-1", 3, op,
+	)
+
+	require.NoError(t, err)
+	assert.False(t, promoted)
+	var knowledge types.Knowledge
+	require.NoError(t, db.Select("parse_status", "pending_subtasks_count").First(&knowledge, "id = ?", "knowledge-1").Error)
+	assert.Equal(t, types.ParseStatusProcessing, knowledge.ParseStatus)
+	assert.Zero(t, knowledge.PendingSubtasksCount)
+	var count int64
+	require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&count).Error)
+	assert.Zero(t, count)
+}
+
+// ---------------- TaskPendingOpsRepository ----------------
+
+// TestTaskPendingOps_Enqueue_AssignsIDAndDefaults verifies a freshly
+// inserted op gets a positive ID and the empty payload becomes "{}"
+// rather than NULL/empty.
+func TestTaskPendingOps_Enqueue_AssignsIDAndDefaults(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	ctx := context.Background()
+
+	op := makePendingOp("wiki:ingest", "knowledge_base", "kb-1", "ingest", "k-1", nil)
+	require.NoError(t, repo.Enqueue(ctx, op))
+	assert.NotZero(t, op.ID)
+	assert.Equal(t, json.RawMessage("{}"), op.Payload, "nil payload should default to {}")
+}
+
+// TestTaskPendingOps_Enqueue_RejectsMissingFields covers the validation
+// layer: every required field must be set, otherwise the call returns an
+// error WITHOUT touching the DB.
+func TestTaskPendingOps_Enqueue_RejectsMissingFields(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	ctx := context.Background()
+
+	cases := []struct {
+		name string
+		op   *types.TaskPendingOp
+	}{
+		{"nil op", nil},
+		{"missing task_type", makePendingOp("", "knowledge_base", "kb", "ingest", "", nil)},
+		{"missing scope", makePendingOp("t", "", "kb", "ingest", "", nil)},
+		{"missing scope_id", makePendingOp("t", "s", "", "ingest", "", nil)},
+		{"missing op", makePendingOp("t", "s", "id", "", "", nil)},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			err := repo.Enqueue(ctx, c.op)
+			assert.Error(t, err)
+		})
+	}
+
+	var n int64
+	db.Table("task_pending_ops").Count(&n)
+	assert.Equal(t, int64(0), n)
+}
+
+// TestTaskPendingOps_PeekBatch_ScopedAndOrdered verifies PeekBatch only
+// returns rows for the matching tuple, in id ASC order, and respects
+// the limit.
+func TestTaskPendingOps_PeekBatch_ScopedAndOrdered(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	ctx := context.Background()
+
+	// Three ops in kb-A, two in kb-B, one in different task_type.
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-A", "ingest", "k1", nil)))
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-A", "retract", "k2", nil)))
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-A", "ingest", "k3", nil)))
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-B", "ingest", "k4", nil)))
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-B", "ingest", "k5", nil)))
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("summary:gen", "knowledge_base", "kb-A", "ingest", "k6", nil)))
+
+	// Peek up to 10 from kb-A — should see exactly 3, in insertion order.
+	got, err := repo.PeekBatch(ctx, "wiki:ingest", "knowledge_base", "kb-A", 10)
+	require.NoError(t, err)
+	require.Len(t, got, 3)
+	assert.Equal(t, "k1", got[0].DedupKey)
+	assert.Equal(t, "k2", got[1].DedupKey)
+	assert.Equal(t, "k3", got[2].DedupKey)
+	assert.True(t, got[0].ID < got[1].ID && got[1].ID < got[2].ID, "ids should be ascending")
+
+	// Limit caps result size.
+	got, err = repo.PeekBatch(ctx, "wiki:ingest", "knowledge_base", "kb-A", 2)
+	require.NoError(t, err)
+	assert.Len(t, got, 2)
+
+	// Different task_type isolated.
+	got, err = repo.PeekBatch(ctx, "summary:gen", "knowledge_base", "kb-A", 10)
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+	assert.Equal(t, "k6", got[0].DedupKey)
+}
+
+// TestTaskPendingOps_DeleteByIDs_RemovesOnlyTargets verifies the
+// delete-after-consume path. Empty input must be a no-op so the consumer
+// can call it unconditionally.
+func TestTaskPendingOps_DeleteByIDs_RemovesOnlyTargets(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	ctx := context.Background()
+
+	a := makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "a", nil)
+	b := makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "b", nil)
+	c := makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "c", nil)
+	require.NoError(t, repo.Enqueue(ctx, a))
+	require.NoError(t, repo.Enqueue(ctx, b))
+	require.NoError(t, repo.Enqueue(ctx, c))
+
+	// No-op: empty slice.
+	require.NoError(t, repo.DeleteByIDs(ctx, nil))
+	require.NoError(t, repo.DeleteByIDs(ctx, []int64{}))
+
+	// Delete a + c, keep b.
+	require.NoError(t, repo.DeleteByIDs(ctx, []int64{a.ID, c.ID}))
+
+	got, err := repo.PeekBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10)
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+	assert.Equal(t, "b", got[0].DedupKey)
+}
+
+func TestTaskPendingOps_DeleteByScope_RemovesAllTaskTypesAndIsolatesScopes(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	cleaner, ok := repo.(interfaces.TaskPendingOpsScopeCleaner)
+	require.True(t, ok)
+	ctx := context.Background()
+
+	// The deleted KB can have durable work in more than one wiki queue.
+	require.NoError(t, repo.Enqueue(ctx,
+		makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-delete", "ingest", "k1", nil)))
+	require.NoError(t, repo.Enqueue(ctx,
+		makePendingOp(types.TypeWikiFinalize, types.TaskScopeKnowledgeBase, "kb-delete", "finalize", "k2", nil)))
+
+	// Rows in another KB or another scope must survive even when their IDs or
+	// task types overlap with the deleted KB.
+	require.NoError(t, repo.Enqueue(ctx,
+		makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-keep", "ingest", "k3", nil)))
+	require.NoError(t, repo.Enqueue(ctx,
+		makePendingOp(types.TypeWikiFinalize, types.TaskScopeKnowledge, "kb-delete", "finalize", "k4", nil)))
+
+	require.NoError(t, cleaner.DeleteByScope(ctx, types.TaskScopeKnowledgeBase, "kb-delete"))
+
+	var remaining []*types.TaskPendingOp
+	require.NoError(t, db.Order("id ASC").Find(&remaining).Error)
+	require.Len(t, remaining, 2)
+	assert.Equal(t, "kb-keep", remaining[0].ScopeID)
+	assert.Equal(t, types.TaskScopeKnowledge, remaining[1].Scope)
+	assert.Equal(t, "kb-delete", remaining[1].ScopeID)
+}
+
+func TestTaskPendingOps_DeleteByScope_RejectsMissingScope(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	cleaner, ok := repo.(interfaces.TaskPendingOpsScopeCleaner)
+	require.True(t, ok)
+	ctx := context.Background()
+
+	require.NoError(t, repo.Enqueue(ctx,
+		makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb", "ingest", "k1", nil)))
+
+	assert.Error(t, cleaner.DeleteByScope(ctx, "", "kb"))
+	assert.Error(t, cleaner.DeleteByScope(ctx, types.TaskScopeKnowledgeBase, ""))
+
+	var count int64
+	require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&count).Error)
+	assert.Equal(t, int64(1), count)
+}
+
+func TestTaskPendingOps_EnqueueIfKnowledgeBaseActive(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	require.NoError(t, db.Exec(`CREATE TABLE knowledge_bases (
+		id VARCHAR(64) PRIMARY KEY,
+		tenant_id INTEGER NOT NULL,
+		deleted_at DATETIME
+	)`).Error)
+	require.NoError(t, db.Exec(
+		"INSERT INTO knowledge_bases (id, tenant_id, deleted_at) VALUES (?, ?, NULL), (?, ?, ?)",
+		"kb-active", 1, "kb-deleted", 1, time.Now(),
+	).Error)
+
+	repo := NewTaskPendingOpsRepository(db)
+	guard, ok := repo.(interfaces.TaskPendingOpsKnowledgeBaseGuard)
+	require.True(t, ok)
+	ctx := context.Background()
+
+	accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx,
+		makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-active", "ingest", "active", nil))
+	require.NoError(t, err)
+	assert.True(t, accepted)
+
+	for _, tc := range []struct {
+		name string
+		op   *types.TaskPendingOp
+	}{
+		{
+			name: "soft deleted",
+			op: makePendingOp(
+				types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-deleted", "ingest", "deleted", nil,
+			),
+		},
+		{
+			name: "missing",
+			op: makePendingOp(
+				types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-missing", "ingest", "missing", nil,
+			),
+		},
+		{name: "tenant mismatch", op: &types.TaskPendingOp{
+			TenantID: 2, TaskType: types.TypeWikiIngest, Scope: types.TaskScopeKnowledgeBase,
+			ScopeID: "kb-active", Op: "ingest", DedupKey: "wrong-tenant",
+		}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx, tc.op)
+			require.NoError(t, err)
+			assert.False(t, accepted)
+		})
+	}
+
+	var rows []*types.TaskPendingOp
+	require.NoError(t, db.Find(&rows).Error)
+	require.Len(t, rows, 1)
+	assert.Equal(t, "active", rows[0].DedupKey)
+}
+
+// TestTaskPendingOps_IncrFailCount_ReturnsNewValueAndPersists exercises
+// the UPDATE...RETURNING flow. Successive bumps should observe monotonic
+// counts.
+func TestTaskPendingOps_IncrFailCount_ReturnsNewValueAndPersists(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	ctx := context.Background()
+
+	op := makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k", nil)
+	require.NoError(t, repo.Enqueue(ctx, op))
+
+	n, err := repo.IncrFailCount(ctx, op.ID)
+	require.NoError(t, err)
+	assert.Equal(t, 1, n)
+
+	n, err = repo.IncrFailCount(ctx, op.ID)
+	require.NoError(t, err)
+	assert.Equal(t, 2, n)
+
+	// Persisted value matches what was returned.
+	rows, err := repo.PeekBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10)
+	require.NoError(t, err)
+	require.Len(t, rows, 1)
+	assert.Equal(t, 2, rows[0].FailCount)
+}
+
+// TestTaskPendingOps_PendingCount_ScopedTuple confirms the count covers
+// only the (task_type, scope, scope_id) tuple.
+func TestTaskPendingOps_PendingCount_ScopedTuple(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	ctx := context.Background()
+
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-A", "ingest", "k1", nil)))
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-A", "ingest", "k2", nil)))
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-B", "ingest", "k3", nil)))
+
+	n, err := repo.PendingCount(ctx, "wiki:ingest", "knowledge_base", "kb-A")
+	require.NoError(t, err)
+	assert.Equal(t, int64(2), n)
+
+	n, err = repo.PendingCount(ctx, "wiki:ingest", "knowledge_base", "missing")
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), n)
+}
+
+// TestTaskPendingOps_DeleteByDedupKey_Filters tests the wiki delete-race
+// helper: matching rows go away, others survive, optional op filter
+// narrows the scope, and an empty dedup_key is rejected (so a buggy
+// caller can't wipe the entire queue).
+func TestTaskPendingOps_DeleteByDedupKey_Filters(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	ctx := context.Background()
+
+	// Two ingests + one retract, all keyed on knowledge "k1"; one ingest
+	// for unrelated "k2".
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil)))
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil)))
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil)))
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k2", nil)))
+
+	// Empty key is an error, queue unchanged.
+	err := repo.DeleteByDedupKey(ctx, "wiki:ingest", "knowledge_base", "kb", "", "")
+	assert.Error(t, err)
+	n, _ := repo.PendingCount(ctx, "wiki:ingest", "knowledge_base", "kb")
+	assert.Equal(t, int64(4), n)
+
+	// Drop only "ingest" rows for k1; retract survives.
+	require.NoError(t, repo.DeleteByDedupKey(ctx, "wiki:ingest", "knowledge_base", "kb", "k1", "ingest"))
+	rows, err := repo.PeekBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10)
+	require.NoError(t, err)
+	require.Len(t, rows, 2)
+	// The two survivors must be the retract for k1 and the ingest for k2.
+	keys := map[string]string{}
+	for _, r := range rows {
+		keys[r.Op] = r.DedupKey
+	}
+	assert.Equal(t, "k1", keys["retract"])
+	assert.Equal(t, "k2", keys["ingest"])
+
+	// Drop everything keyed on k1 regardless of op (empty op = wildcard).
+	require.NoError(t, repo.DeleteByDedupKey(ctx, "wiki:ingest", "knowledge_base", "kb", "k1", ""))
+	rows, err = repo.PeekBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10)
+	require.NoError(t, err)
+	require.Len(t, rows, 1)
+	assert.Equal(t, "k2", rows[0].DedupKey)
+}
+
+// TestTaskPendingOps_ClaimBatch_MarksAndReturnsDisjoint verifies that
+// ClaimBatch returns rows in FIFO order, stamps claimed_at, and that a
+// second claim does NOT re-return the already-claimed rows (the disjoint
+// property concurrent consumers rely on).
+func TestTaskPendingOps_ClaimBatch_MarksAndReturnsDisjoint(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	ctx := context.Background()
+
+	for _, k := range []string{"k1", "k2", "k3"} {
+		require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", k, nil)))
+	}
+	// Unrelated tuple must never be claimed.
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "other", "ingest", "z", nil)))
+
+	stale := time.Now().Add(-time.Hour)
+
+	first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 2, stale)
+	require.NoError(t, err)
+	require.Len(t, first, 2)
+	assert.Equal(t, "k1", first[0].DedupKey)
+	assert.Equal(t, "k2", first[1].DedupKey)
+	assert.NotNil(t, first[0].ClaimedAt, "claimed_at should be stamped")
+
+	// Second claim skips the two already-claimed rows and returns the last.
+	second, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, stale)
+	require.NoError(t, err)
+	require.Len(t, second, 1)
+	assert.Equal(t, "k3", second[0].DedupKey)
+
+	// Nothing left to claim.
+	third, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, stale)
+	require.NoError(t, err)
+	assert.Len(t, third, 0)
+}
+
+// TestTaskPendingOps_ClaimBatch_KeepsSameKeyTogether verifies the
+// dedup_key affinity invariant: all rows sharing a knowledge_id are claimed
+// in the SAME batch (never split), and `limit` counts distinct keys, not
+// rows. This is what stops a concurrent batch from processing one op of a
+// document while another batch processes a second op of the same document.
+func TestTaskPendingOps_ClaimBatch_KeepsSameKeyTogether(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	ctx := context.Background()
+
+	// Document k1 has TWO queued ops (ingest then retract); k2 and k3 have
+	// one each. Enqueue order interleaves them so a naive row-ordered claim
+	// would split k1 across batches.
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil)))
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k2", nil)))
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil)))
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k3", nil)))
+
+	stale := time.Now().Add(-time.Hour)
+
+	// limit=2 keys → k1 (both rows) + k2. k1 must NOT be split.
+	first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 2, stale)
+	require.NoError(t, err)
+	require.Len(t, first, 3, "k1's two rows + k2's one row")
+	byKey := map[string]int{}
+	for _, r := range first {
+		byKey[r.DedupKey]++
+	}
+	assert.Equal(t, 2, byKey["k1"], "both k1 ops claimed together")
+	assert.Equal(t, 1, byKey["k2"])
+	assert.Zero(t, byKey["k3"], "k3 belongs to the next batch (limit was 2 keys)")
+
+	// A concurrent-style second claim gets the remaining key only — it can
+	// never see k1's rows again (disjoint).
+	second, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, stale)
+	require.NoError(t, err)
+	require.Len(t, second, 1)
+	assert.Equal(t, "k3", second[0].DedupKey)
+}
+
+// TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim verifies that a
+// row enqueued for an already-claimed (in-flight) document is NOT claimed on
+// its own: a dedup_key with any fresh claim is skipped WHOLESALE so the late
+// sibling waits for the holder to finish (rows deleted → key freed) or for the
+// claim to go stale, keeping same-document ops serialized across concurrent
+// batches.
+func TestTaskPendingOps_ClaimBatch_LateSiblingBlockedByFreshClaim(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	ctx := context.Background()
+
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil)))
+	stale := time.Now().Add(-time.Hour)
+
+	first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
+	require.NoError(t, err)
+	require.Len(t, first, 1)
+	ingestID := first[0].ID
+
+	// A retract for the same, still-in-flight document arrives.
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil)))
+
+	// The retract must NOT be claimed while the ingest claim is fresh —
+	// the whole k1 key is blocked.
+	second, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
+	require.NoError(t, err)
+	require.Len(t, second, 0, "late sibling blocked while holder's claim is fresh")
+
+	// Once the holder finishes (its claimed rows are deleted), the key is
+	// free and the retract becomes claimable.
+	require.NoError(t, repo.DeleteByIDs(ctx, []int64{ingestID}))
+	third, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, stale)
+	require.NoError(t, err)
+	require.Len(t, third, 1)
+	assert.Equal(t, "retract", third[0].Op)
+}
+
+// TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale verifies the
+// other release path: if the holder CRASHES (claim never cleared), the whole
+// key — original row + late sibling — becomes claimable together once the
+// claim goes stale, so the pair is folded back into one batch.
+func TestTaskPendingOps_ClaimBatch_LateSiblingClaimableAfterStale(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	ctx := context.Background()
+
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil)))
+	first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, time.Now().Add(-time.Hour))
+	require.NoError(t, err)
+	require.Len(t, first, 1)
+
+	// Late retract arrives; holder then crashes (claim left stamped).
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil)))
+
+	// A future stale threshold makes the crashed claim eligible again; both
+	// k1 rows are reclaimed together (never split).
+	got, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 5, time.Now().Add(time.Hour))
+	require.NoError(t, err)
+	require.Len(t, got, 2, "stale key reclaims both the ingest and the late retract together")
+	for _, r := range got {
+		assert.Equal(t, "k1", r.DedupKey)
+	}
+}
+
+// TestTaskPendingOps_ClaimBatch_ReclaimsStale verifies a claim older than
+// staleBefore is re-claimable (crash recovery), while a fresh claim is not.
+func TestTaskPendingOps_ClaimBatch_ReclaimsStale(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	ctx := context.Background()
+
+	require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil)))
+
+	// Claim it now.
+	got, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, time.Now().Add(-time.Hour))
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+
+	// A stale threshold in the past — the fresh claim is NOT stale yet.
+	got, err = repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, time.Now().Add(-time.Hour))
+	require.NoError(t, err)
+	assert.Len(t, got, 0, "fresh claim must not be re-claimed")
+
+	// A stale threshold in the future makes every existing claim stale.
+	got, err = repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, time.Now().Add(time.Hour))
+	require.NoError(t, err)
+	require.Len(t, got, 1, "stale claim must be recoverable")
+	assert.Equal(t, "k1", got[0].DedupKey)
+}
+
+// TestTaskPendingOps_ReleaseByIDs_ReturnsToPool verifies a released row
+// becomes immediately claimable again without waiting for staleness.
+func TestTaskPendingOps_ReleaseByIDs_ReturnsToPool(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskPendingOpsRepository(db)
+	ctx := context.Background()
+
+	op := makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil)
+	require.NoError(t, repo.Enqueue(ctx, op))
+
+	stale := time.Now().Add(-time.Hour)
+	got, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, stale)
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+
+	// Claimed → not re-claimable.
+	got, err = repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, stale)
+	require.NoError(t, err)
+	require.Len(t, got, 0)
+
+	// Release → immediately claimable again.
+	require.NoError(t, repo.ReleaseByIDs(ctx, nil)) // no-op tolerated
+	require.NoError(t, repo.ReleaseByIDs(ctx, []int64{op.ID}))
+	got, err = repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, stale)
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+	assert.Equal(t, "k1", got[0].DedupKey)
+}
+
+// ---------------- TaskDeadLetterRepository ----------------
+
+func makeDeadLetter(taskType, scope, scopeID, relatedID, lastErr string) *types.TaskDeadLetter {
+	return &types.TaskDeadLetter{
+		TenantID:  1,
+		TaskType:  taskType,
+		Scope:     scope,
+		ScopeID:   scopeID,
+		RelatedID: relatedID,
+		Payload:   json.RawMessage(`{"x":1}`),
+		LastError: lastErr,
+		FailCount: 5,
+	}
+}
+
+// TestTaskDeadLetter_Insert_DefaultsAndAssignsID covers the empty-payload
+// fallback and ID assignment.
+func TestTaskDeadLetter_Insert_DefaultsAndAssignsID(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskDeadLetterRepository(db)
+	ctx := context.Background()
+
+	dl := &types.TaskDeadLetter{
+		TenantID:  1,
+		TaskType:  "wiki:ingest",
+		ScopeID:   "kb",
+		FailCount: 3,
+		// Scope intentionally empty — should default to "unknown".
+		// Payload intentionally nil — should default to "{}".
+	}
+	require.NoError(t, repo.Insert(ctx, dl))
+	assert.NotZero(t, dl.ID)
+	assert.Equal(t, types.TaskScopeUnknown, dl.Scope)
+	assert.Equal(t, json.RawMessage("{}"), dl.Payload)
+}
+
+// TestTaskDeadLetter_Insert_RejectsMissingFields verifies the guard
+// against rows that would leave the table without the columns ops queries
+// rely on.
+func TestTaskDeadLetter_Insert_RejectsMissingFields(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskDeadLetterRepository(db)
+	ctx := context.Background()
+
+	assert.Error(t, repo.Insert(ctx, nil))
+	assert.Error(t, repo.Insert(ctx, &types.TaskDeadLetter{ScopeID: "kb"}))
+
+	var n int64
+	db.Table("task_dead_letters").Count(&n)
+	assert.Equal(t, int64(0), n)
+}
+
+// TestTaskDeadLetter_ListByScope_NewestFirstAndCursored exercises the
+// cursor pagination path used by the ops console.
+func TestTaskDeadLetter_ListByScope_NewestFirstAndCursored(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskDeadLetterRepository(db)
+	ctx := context.Background()
+
+	// Insert 5 rows for kb-A and 2 for kb-B.
+	for i := 0; i < 5; i++ {
+		require.NoError(t, repo.Insert(ctx, makeDeadLetter("wiki:ingest", "knowledge_base", "kb-A", "k", "boom")))
+	}
+	require.NoError(t, repo.Insert(ctx, makeDeadLetter("wiki:ingest", "knowledge_base", "kb-B", "k", "boom")))
+	require.NoError(t, repo.Insert(ctx, makeDeadLetter("wiki:ingest", "knowledge_base", "kb-B", "k", "boom")))
+
+	// First page of 2 from kb-A, newest first.
+	page1, cursor, err := repo.ListByScope(ctx, "knowledge_base", "kb-A", "", 2)
+	require.NoError(t, err)
+	require.Len(t, page1, 2)
+	assert.True(t, page1[0].ID > page1[1].ID, "newest first")
+	require.NotEmpty(t, cursor)
+
+	// Second page of 2.
+	page2, cursor, err := repo.ListByScope(ctx, "knowledge_base", "kb-A", cursor, 2)
+	require.NoError(t, err)
+	require.Len(t, page2, 2)
+	assert.True(t, page1[1].ID > page2[0].ID, "page2 should continue past page1")
+	require.NotEmpty(t, cursor)
+
+	// Last page — only 1 row left, cursor goes empty since len < limit.
+	page3, cursor, err := repo.ListByScope(ctx, "knowledge_base", "kb-A", cursor, 2)
+	require.NoError(t, err)
+	require.Len(t, page3, 1)
+	assert.Empty(t, cursor)
+
+	// kb-B is isolated.
+	pageB, _, err := repo.ListByScope(ctx, "knowledge_base", "kb-B", "", 10)
+	require.NoError(t, err)
+	require.Len(t, pageB, 2)
+}
+
+// TestTaskDeadLetter_ListByScope_RejectsMissingScope guards the input
+// validation in the public method.
+func TestTaskDeadLetter_ListByScope_RejectsMissingScope(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskDeadLetterRepository(db)
+	ctx := context.Background()
+
+	_, _, err := repo.ListByScope(ctx, "", "kb", "", 10)
+	assert.Error(t, err)
+	_, _, err = repo.ListByScope(ctx, "knowledge_base", "", "", 10)
+	assert.Error(t, err)
+}
+
+// TestTaskDeadLetter_ListByTaskType_FiltersAndPaginates is the cross-KB
+// view: "all summary:generation failures" regardless of which KB they
+// belong to.
+func TestTaskDeadLetter_ListByTaskType_FiltersAndPaginates(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskDeadLetterRepository(db)
+	ctx := context.Background()
+
+	require.NoError(t, repo.Insert(ctx, makeDeadLetter("wiki:ingest", "knowledge_base", "kb-A", "k1", "")))
+	require.NoError(t, repo.Insert(ctx, makeDeadLetter("summary:gen", "knowledge_base", "kb-A", "k2", "")))
+	require.NoError(t, repo.Insert(ctx, makeDeadLetter("summary:gen", "knowledge_base", "kb-B", "k3", "")))
+	require.NoError(t, repo.Insert(ctx, makeDeadLetter("wiki:ingest", "knowledge_base", "kb-B", "k4", "")))
+
+	rows, _, err := repo.ListByTaskType(ctx, "summary:gen", "", 10)
+	require.NoError(t, err)
+	require.Len(t, rows, 2)
+	for _, r := range rows {
+		assert.Equal(t, "summary:gen", r.TaskType)
+	}
+
+	_, _, err = repo.ListByTaskType(ctx, "", "", 10)
+	assert.Error(t, err)
+}
+
+// TestTaskDeadLetter_DeleteByID_IsIdempotent confirms a missing row does
+// not produce an error — operators triggering concurrent deletes should
+// see clean success.
+func TestTaskDeadLetter_DeleteByID_IsIdempotent(t *testing.T) {
+	db := setupTaskQueueTestDB(t)
+	repo := NewTaskDeadLetterRepository(db)
+	ctx := context.Background()
+
+	dl := makeDeadLetter("wiki:ingest", "knowledge_base", "kb", "k", "")
+	require.NoError(t, repo.Insert(ctx, dl))
+
+	require.NoError(t, repo.DeleteByID(ctx, dl.ID))
+	// Second delete on the same id should silently succeed.
+	require.NoError(t, repo.DeleteByID(ctx, dl.ID))
+	// Delete of unknown id should silently succeed.
+	require.NoError(t, repo.DeleteByID(ctx, 99999))
+
+	rows, _, err := repo.ListByScope(ctx, "knowledge_base", "kb", "", 10)
+	require.NoError(t, err)
+	assert.Len(t, rows, 0)
+}