const { test } = require('node:test') const assert = require('node:assert/strict') const { latestAttempts, planRecovery, validatePublishedRecoveryFix, validateFinalRecoveryTrigger, fifthRecoveryScope, validateRecoveryBudget, durableRecoveryStatus, resolvedRecoveryFailure } = require('./wiki_recovery_policy.cjs') const selected = [{ id: 16, model_id: 'model', pending: 0 }] const pilot = { dead_letter_id: 31, model_id: 'model', recovery_status: 'wiki_recovered', pending: false } const failed = { dead_letter_id: 16, attempt_number: 1, model_id: 'model', recovery_status: 'failed', pending: false } test('second attempt requires an explicit investigated reason and does not overwrite history', () => { assert.throws(() => planRecovery(selected, [pilot, failed], 2, ''), /reason/) assert.deepEqual(planRecovery(selected, [pilot, failed], 2, 'Deployed bounded extraction and citation failure handling'), selected) assert.equal(failed.attempt_number, 1) const retry = { ...failed, attempt_number: 2, recovery_status: 'running', pending: true } assert.deepEqual(planRecovery(selected, [pilot, failed, retry], 2, 'Same reviewed retry, lost console response'), []) assert.equal(latestAttempts([pilot, failed, retry]).find(row => row.dead_letter_id === 16), retry) }) test('does not permit concurrent work, a third attempt or automatic bypass of failure', () => { assert.throws(() => planRecovery([{ id: 2, model_id: 'model', pending: 0 }], [pilot, failed], 1, ''), /unresolved/) assert.throws(() => planRecovery(selected, [pilot, failed], 3, 'Reviewed fix'), /Only first/) assert.throws(() => planRecovery(selected, [pilot, { ...failed, pending: true }], 2, 'Reviewed deployed extraction fix'), /Wait/) assert.throws(() => planRecovery(selected, [{ ...pilot, pending_finalize: 1 }, failed], 2, 'Reviewed deployed extraction fix'), /Wait/) assert.throws(() => planRecovery(selected, [pilot], 2, 'Reviewed deployed extraction fix'), /unresolved previous/) }) test('single pilot and exact repeat are safe; larger batches are rejected', () => { assert.deepEqual(planRecovery(selected, [], 1, ''), selected) assert.deepEqual(planRecovery([{ id: 31 }], [pilot], 1, ''), []) assert.throws(() => planRecovery([...selected, { id: 17 }], [], 1, ''), /exactly one/) }) test('final manual recovery is bound to the exact failed record and a newer deployed fix', () => { const source = { ...selected[0], tenant_id: 7, knowledge_id: 'doc', knowledge_base_id: 'kb' } const previous = { ...failed, tenant_id: 7, knowledge_id: 'doc', knowledge_base_id: 'kb', attempt_number: 2, error_code: 'PAGE_WRITE_INCOMPLETE', new_failures: 1, queued_at: '2026-09-18T21:00:00Z' } const review = { tenant_id: 7, knowledge_id: 'doc', knowledge_base_id: 'kb', model_id: 'model', dead_letter_id: 16, max_attempt: 3, previous_attempt: 2, previous_error_code: 'PAGE_WRITE_INCOMPLETE', fix_release: 'public-20260919-061500', published_at: '2026-09-18T22:00:00Z' } const reason = 'Deployed bounded complete-page repair with persistence regression tests' assert.deepEqual(planRecovery([source], [pilot, failed, previous], 3, reason, review), [source]) const completed = { ...previous, attempt_number: 3, recovery_status: 'wiki_recovered' } assert.deepEqual(planRecovery([source], [pilot, failed, previous, completed], 3, reason, review), []) for (const change of [{ tenant_id: 8 }, { knowledge_id: 'other' }, { model_id: 'other' }, { dead_letter_id: 17 }, { previous_error_code: 'OTHER' }, { max_attempt: 4 }, { published_at: previous.queued_at }]) { assert.throws(() => planRecovery([source], [pilot, failed, previous], 3, reason, { ...review, ...change })) } assert.throws(() => planRecovery([source], [pilot, failed, { ...previous, pending: true }], 3, reason, review), /Wait/) assert.throws(() => planRecovery([source], [pilot, failed, previous, completed], 4, reason, review), /exactly this attempt/) }) test('final recovery requires the published immutable API image to be running', () => { const review = { fix_release: 'public-20260919-061500' } const manifest = { stage: 'published', release: review.fix_release, api_image_id: `sha256:${'a'.repeat(64)}`, published_at: '2026-09-18T22:00:00Z' } const runtime = { running: true, release: manifest.release, image_id: manifest.api_image_id } assert.deepEqual(validatePublishedRecoveryFix(review, manifest, runtime), { ...review, published_at: manifest.published_at }) assert.equal(review.published_at, undefined) for (const change of [{ running: false }, { release: 'public-20260918-000000' }, { image_id: `sha256:${'b'.repeat(64)}` }]) { assert.throws(() => validatePublishedRecoveryFix(review, manifest, { ...runtime, ...change })) } assert.throws(() => validatePublishedRecoveryFix(review, { ...manifest, stage: 'origin-ready' }, runtime), /Publish/) assert.throws(() => validatePublishedRecoveryFix(review, { ...manifest, release: 'public-20260918-000000' }, runtime), /current release/) assert.throws(() => validatePublishedRecoveryFix(undefined, manifest, runtime), /persisted/) }) test('triggering a final recovery revalidates the current source and persisted receipt scope', () => { const source = { id: 16, tenant_id: 7, knowledge_id: 'doc', knowledge_base_id: 'kb', model_id: 'model', eligible: true, chunks: 1, pending: 1 } const previous = { ...failed, tenant_id: 7, knowledge_id: 'doc', knowledge_base_id: 'kb', attempt_number: 2, error_code: 'PAGE_WRITE_INCOMPLETE', new_failures: 1, queued_at: '2026-09-18T21:00:00Z' } const review = { tenant_id: 7, knowledge_id: 'doc', knowledge_base_id: 'kb', model_id: 'model', dead_letter_id: 16, max_attempt: 3, previous_attempt: 2, previous_error_code: 'PAGE_WRITE_INCOMPLETE', fix_release: 'public-20260919-061500', published_at: '2026-09-18T22:00:00Z' } const receipt = { ...previous, attempt_number: 3, pending: true, pending_op_id: 99, retry_reason: 'Deployed bounded complete-page repair with persistence regression tests', deployed_fix_review: review } const history = [pilot, failed, previous, receipt] assert.doesNotThrow(() => validateFinalRecoveryTrigger(receipt, source, history, review)) for (const key of ['tenant_id', 'knowledge_id', 'knowledge_base_id', 'model_id']) { assert.throws(() => validateFinalRecoveryTrigger(receipt, { ...source, [key]: 'other' }, history, review), /Source changed/) assert.throws(() => validateFinalRecoveryTrigger({ ...receipt, [key]: 'other' }, source, history, review), /receipt scope/) } assert.throws(() => validateFinalRecoveryTrigger({ ...receipt, dead_letter_id: 17 }, source, history, review), /receipt scope/) assert.throws(() => validateFinalRecoveryTrigger(receipt, { ...source, eligible: false }, history, review), /eligible/) assert.throws(() => validateFinalRecoveryTrigger(receipt, source, [pilot, failed, previous, { ...receipt, pending: false }], review), /current and pending/) assert.throws(() => validateFinalRecoveryTrigger(receipt, source, [pilot, failed, previous, { ...receipt, pending_op_id: 100 }], review), /current and pending/) assert.throws(() => validateFinalRecoveryTrigger({ ...receipt, attempt_number: 4 }, source, history, review), /bounded attempt/) }) test('a fourth recovery requires a new exact review; old approvals and a fifth attempt are refused', () => { const source = { id: 10, tenant_id: 7, knowledge_id: 'doc', knowledge_base_id: 'kb', model_id: 'model', pending: 0 } const previous = { ...failed, ...source, dead_letter_id: 10, attempt_number: 3, error_code: 'MAP_FAILED', new_failures: 1, queued_at: '2026-09-18T22:29:48Z' } const review = { ...source, dead_letter_id: 10, max_attempt: 4, previous_attempt: 3, previous_error_code: 'MAP_FAILED', fix_release: 'public-20260919-170000', published_at: '2026-09-19T09:00:00Z' } const reason = 'Published exact candidate handles and full-source window checkpoints, verified against the failing source' assert.deepEqual(planRecovery([source], [pilot, previous], 4, reason, review), [source]) assert.throws(() => planRecovery([source], [pilot, previous], 4, reason), /deployed-fix review/) for (const change of [{ max_attempt: 3 }, { previous_attempt: 2 }, { previous_error_code: 'PAGE_WRITE_INCOMPLETE' }, { knowledge_id: 'another-source' }, { published_at: previous.queued_at }]) { assert.throws(() => planRecovery([source], [pilot, previous], 4, reason, { ...review, ...change })) } assert.throws(() => planRecovery([source], [pilot, previous], 5, reason, { ...review, max_attempt: 5 }), /preceding attempt/) const receipt = { ...previous, attempt_number: 4, pending: true, pending_op_id: 101, retry_reason: reason } assert.doesNotThrow(() => validateFinalRecoveryTrigger(receipt, { ...source, eligible: true, chunks: 624, pending: 1 }, [pilot, previous, receipt], review)) assert.deepEqual(planRecovery([source], [pilot, previous, receipt], 4, reason, review), []) assert.equal(previous.attempt_number, 3) }) function fifthFixture() { const scope = fifthRecoveryScope const source = { ...scope, id: scope.dead_letter_id, pending: 0, eligible: true, chunks: 624, source_fingerprint: 'a'.repeat(64), model_fingerprint: 'b'.repeat(64), embedding_model_id: 'embedding' } const previous = { ...scope, attempt_number: 4, pending_op_id: 4143, recovery_status: 'failed', new_failures: 1, error_code: 'EXTRACT_FAILED', queued_at: '2026-09-19T01:00:00Z' } const review = { ...source, max_attempt: 5, previous_attempt: 4, previous_pending_op_id: 4143, previous_error_code: previous.error_code, budget_version: 1, api_image_id: `sha256:${'c'.repeat(64)}`, max_provider_requests: 200, deadline_at: new Date(Date.now() + 3600000).toISOString(), fix_release: 'public-20260920-180000', published_at: '2026-09-20T10:00:00Z' } const budget = { ...scope, attempt_number: 5, pending_op_id: 5000, source_fingerprint: source.source_fingerprint, model_fingerprint: source.model_fingerprint, embedding_model_id: source.embedding_model_id, max_provider_requests: review.max_provider_requests, reserved_provider_requests: 12, deadline_at: review.deadline_at, reviewed_release: review.fix_release, reviewed_api_image: review.api_image_id, status: 'active' } const receipt = { ...previous, attempt_number: 5, pending_op_id: 5000, pending: true, recovery_budget: budget, retry_reason: 'Verified persisted whole-operation budget and bounded candidate repair' } return { source, previous, review, receipt, pilot: { ...pilot, model_id: scope.model_id } } } test('only the exact reviewed fifth source is accepted; identities, old budgets and sixth attempts fail closed', () => { const { source, previous, review, receipt, pilot } = fifthFixture() const reason = receipt.retry_reason assert.deepEqual(planRecovery([source], [pilot, previous], 5, reason, review), [source]) assert.deepEqual(planRecovery([source], [pilot, previous, receipt], 5, reason, review), []) for (const key of ['tenant_id', 'knowledge_id', 'knowledge_base_id', 'model_id', 'source_fingerprint', 'model_fingerprint', 'embedding_model_id']) { assert.throws(() => planRecovery([{ ...source, [key]: 'changed' }], [pilot, previous], 5, reason, review)) } for (const change of [{ id: 11 }, { tenant_id: 10060 }]) assert.throws(() => planRecovery([{ ...source, ...change }], [pilot, previous], 5, reason, review)) for (const change of [{ previous_pending_op_id: 4142 }, { max_provider_requests: 0 }, { budget_version: 0 }, { deadline_at: 'invalid' }, { deadline_at: '2020-01-01T00:00:00Z' }, { source_fingerprint: '' }, { api_image_id: '' }]) { assert.throws(() => planRecovery([source], [pilot, previous], 5, reason, { ...review, ...change })) } assert.throws(() => planRecovery([source], [pilot, previous], 6, reason, { ...review, max_attempt: 6 }), /preceding attempt/) assert.throws(() => planRecovery([source], [pilot, { ...previous, pending_op_id: 4142 }], 5, reason, review), /exact fourth/) }) test('fifth trigger requires the immutable active budget; exhaustion, expiry, missing budget and reset counters are rejected', () => { const { source, previous, review, receipt, pilot } = fifthFixture() const triggerSource = { ...source, pending: 1 } assert.doesNotThrow(() => validateFinalRecoveryTrigger(receipt, triggerSource, [pilot, previous, receipt], review)) assert.throws(() => validateRecoveryBudget({ ...receipt, recovery_budget: null }, review), /missing/) for (const change of [{ status: 'completed' }, { status: 'stopped_budget' }, { tenant_id: 9 }, { pending_op_id: 5001 }, { reserved_provider_requests: -1 }, { reserved_provider_requests: 200 }, { max_provider_requests: 201 }, { deadline_at: new Date(Date.parse(review.deadline_at) + 1).toISOString() }, { reviewed_release: 'other' }]) { const changed = { ...receipt, recovery_budget: { ...receipt.recovery_budget, ...change } } assert.throws(() => validateFinalRecoveryTrigger(changed, triggerSource, [pilot, previous, changed], review)) } assert.throws(() => validateRecoveryBudget(receipt, review, Date.parse(review.deadline_at)), /expired/) assert.equal(durableRecoveryStatus({ ...receipt, recovery_budget: { ...receipt.recovery_budget, status: 'stopped_budget' } }), 'stopped_budget') assert.equal(durableRecoveryStatus({ ...receipt, recovery_budget: null }), 'stopped_invalid') assert.equal(durableRecoveryStatus(receipt), null) }) test('new first attempts share the durable review and trigger contract without resetting old receipts', () => { const fifth = fifthFixture() const source = { ...fifth.source, id: 17, knowledge_id: 'next-doc' } const review = { ...fifth.review, dead_letter_id: 17, knowledge_id: source.knowledge_id, max_attempt: 1 } assert.deepEqual(planRecovery([source], [fifth.pilot], 1, '', review), [source]) const receipt = { ...source, dead_letter_id: source.id, attempt_number: 1, pending_op_id: 5001, pending: true, deployed_fix_review: review, recovery_budget: { ...fifth.receipt.recovery_budget, dead_letter_id: source.id, knowledge_id: source.knowledge_id, attempt_number: 1, pending_op_id: 5001 } } assert.deepEqual(planRecovery([source], [fifth.pilot, receipt], 1, '', review), []) assert.doesNotThrow(() => validateFinalRecoveryTrigger(receipt, { ...source, pending: 1 }, [fifth.pilot, receipt], review)) assert.equal(durableRecoveryStatus({ ...receipt, recovery_budget: null }), 'stopped_invalid') for (const change of [{ source_fingerprint: '' }, { model_id: 'different' }, { max_provider_requests: 0 }, { max_attempt: 2 }]) { assert.throws(() => planRecovery([source], [fifth.pilot], 1, '', { ...review, ...change })) } }) test('an explicitly closed terminal attempt does not block a new first attempt', () => { const closed = { dead_letter_id: 10, attempt_number: 7, model_id: 'model', recovery_status: 'stopped_budget', error_code: 'EXTRACT_FAILED', pending: false, pending_finalize: 0, closed: true } const open = { ...closed, closed: false } // An unclosed terminal state is still pending investigation. assert.throws(() => planRecovery([{ id: 4, model_id: 'model', pending: 0 }], [pilot, open], 1, ''), /unresolved/) // A recorded closure turns the same row into a decision, not outstanding work. assert.deepEqual(planRecovery([{ id: 4, model_id: 'model', pending: 0 }], [pilot, closed], 1, ''), [{ id: 4, model_id: 'model', pending: 0 }]) // Closure never excuses work that is still running or backed by an active budget. assert.throws(() => planRecovery([{ id: 4, model_id: 'model', pending: 0 }], [pilot, { ...closed, pending: true }], 1, ''), /Wait/) assert.throws(() => planRecovery([{ id: 4, model_id: 'model', pending: 0 }], [pilot, { ...closed, pending_finalize: 1 }], 1, ''), /Wait/) // Only terminal states may be closed, and a closure cannot be used to justify a retry of itself. assert.throws(() => planRecovery([{ id: 10, model_id: 'model', pending: 0 }], [pilot, { ...closed, recovery_status: 'failed' }], 1, ''), /unresolved/) }) test('a model with no prior attempt may run its first pilot, an unproven one may not', () => { const closedTerminalMimo = { dead_letter_id: 12, attempt_number: 4, model_id: 'mimo', recovery_status: 'stopped_budget', error_code: 'EXTRACT_FAILED', pending: false, pending_finalize: 0, closed: true } const mimoSuccess = { dead_letter_id: 13, attempt_number: 1, model_id: 'mimo', recovery_status: 'wiki_recovered', pending: false } const freshMimo = { id: 3, model_id: 'mimo', pending: 0 } const freshMimoAgain = { id: 5, model_id: 'mimo', pending: 0 } // First attempt for a model that has never been tried is the model's own pilot. assert.deepEqual(planRecovery([freshMimo], [pilot], 1, ''), [freshMimo]) // Once a model has a successful pilot, further first attempts are allowed. assert.deepEqual(planRecovery([freshMimoAgain], [pilot, mimoSuccess], 1, ''), [freshMimoAgain]) // A model whose only attempts ended terminal (and were closed) still has no pilot. assert.throws(() => planRecovery([freshMimoAgain], [pilot, closedTerminalMimo], 1, ''), /successful pilot/) }) test('a reviewed retry accepts a terminal budget stop as the failed predecessor', () => { const source = { id: 3, tenant_id: 10059, knowledge_id: 'k', knowledge_base_id: 'kb', model_id: 'm', pending: 0, source_fingerprint: 'a'.repeat(64), model_fingerprint: 'b'.repeat(64), embedding_model_id: 'e' } const review = { budget_version: 1, max_attempt: 3, previous_attempt: 2, previous_error_code: 'EXTRACT_FAILED', fix_release: 'public-20260921-222500', published_at: '2026-09-21T15:00:00Z', api_image_id: 'sha256:' + 'c'.repeat(64), deadline_at: '2026-09-21T18:00:00Z', max_provider_requests: 700, tenant_id: 10059, dead_letter_id: 3, knowledge_id: 'k', knowledge_base_id: 'kb', model_id: 'm', source_fingerprint: 'a'.repeat(64), model_fingerprint: 'b'.repeat(64), embedding_model_id: 'e' } const earlierSuccess = { tenant_id: 10059, dead_letter_id: 2, attempt_number: 2, model_id: 'm', recovery_status: 'wiki_recovered', pending: false } const prior = attempt => [earlierSuccess, { tenant_id: 10059, dead_letter_id: 3, attempt_number: 2, knowledge_id: 'k', knowledge_base_id: 'kb', model_id: 'm', error_code: 'EXTRACT_FAILED', queued_at: '2026-09-21T14:43:54Z', new_failures: 0, pending: false, recovery_status: attempt, recovery_budget: { status: attempt, deadline_at: '2026-09-21T17:15:00Z', max_provider_requests: 700, reserved_provider_requests: 2, tenant_id: 10059, dead_letter_id: 3, knowledge_id: 'k', knowledge_base_id: 'kb', model_id: 'm', attempt_number: 2, pending_op_id: 4150, source_fingerprint: 'a'.repeat(64), model_fingerprint: 'b'.repeat(64), embedding_model_id: 'e', reviewed_release: 'public-20260921-222500', reviewed_api_image: 'sha256:' + 'c'.repeat(64) } }] const predecessor = attempt => prior(attempt).find(row => row.dead_letter_id === 3) assert.equal(resolvedRecoveryFailure(predecessor('stopped_budget')), true) assert.equal(resolvedRecoveryFailure(predecessor('failed')), true) assert.equal(resolvedRecoveryFailure(predecessor('wiki_recovered')), false) const plan = planRecovery([source], prior('stopped_budget'), 3, 'Reviewed retry after the terminal stop', review) assert.equal(plan.length, 1) assert.throws(() => planRecovery([source], prior('wiki_recovered'), 3, 'Reviewed retry after success', review), /requires a failed preceding attempt/) })