xinmi/scripts/wiki_recovery_policy.test.cjs

const { test } = require('node:test')
const assert = require('node:assert/strict')
const { latestAttempts, planRecovery, validatePublishedRecoveryFix, validateFinalRecoveryTrigger, fifthRecoveryScope, validateRecoveryBudget, durableRecoveryStatus, resolvedRecoveryFailure } = require('./wiki_recovery_policy.cjs')

const selected = [{ id: 16, model_id: 'model', pending: 0 }]
const pilot = { dead_letter_id: 31, model_id: 'model', recovery_status: 'wiki_recovered', pending: false }
const failed = { dead_letter_id: 16, attempt_number: 1, model_id: 'model', recovery_status: 'failed', pending: false }

test('second attempt requires an explicit investigated reason and does not overwrite history', () => {
  assert.throws(() => planRecovery(selected, [pilot, failed], 2, ''), /reason/)
  assert.deepEqual(planRecovery(selected, [pilot, failed], 2, 'Deployed bounded extraction and citation failure handling'), selected)
  assert.equal(failed.attempt_number, 1)
  const retry = { ...failed, attempt_number: 2, recovery_status: 'running', pending: true }
  assert.deepEqual(planRecovery(selected, [pilot, failed, retry], 2, 'Same reviewed retry, lost console response'), [])
  assert.equal(latestAttempts([pilot, failed, retry]).find(row => row.dead_letter_id === 16), retry)
})

test('does not permit concurrent work, a third attempt or automatic bypass of failure', () => {
  assert.throws(() => planRecovery([{ id: 2, model_id: 'model', pending: 0 }], [pilot, failed], 1, ''), /unresolved/)
  assert.throws(() => planRecovery(selected, [pilot, failed], 3, 'Reviewed fix'), /Only first/)
  assert.throws(() => planRecovery(selected, [pilot, { ...failed, pending: true }], 2, 'Reviewed deployed extraction fix'), /Wait/)
  assert.throws(() => planRecovery(selected, [{ ...pilot, pending_finalize: 1 }, failed], 2, 'Reviewed deployed extraction fix'), /Wait/)
  assert.throws(() => planRecovery(selected, [pilot], 2, 'Reviewed deployed extraction fix'), /unresolved previous/)
})

test('single pilot and exact repeat are safe; larger batches are rejected', () => {
  assert.deepEqual(planRecovery(selected, [], 1, ''), selected)
  assert.deepEqual(planRecovery([{ id: 31 }], [pilot], 1, ''), [])
  assert.throws(() => planRecovery([...selected, { id: 17 }], [], 1, ''), /exactly one/)
})

test('final manual recovery is bound to the exact failed record and a newer deployed fix', () => {
  const source = { ...selected[0], tenant_id: 7, knowledge_id: 'doc', knowledge_base_id: 'kb' }
  const previous = { ...failed, tenant_id: 7, knowledge_id: 'doc', knowledge_base_id: 'kb', attempt_number: 2,
    error_code: 'PAGE_WRITE_INCOMPLETE', new_failures: 1, queued_at: '2026-09-18T21:00:00Z' }
  const review = { tenant_id: 7, knowledge_id: 'doc', knowledge_base_id: 'kb', model_id: 'model', dead_letter_id: 16,
    max_attempt: 3, previous_attempt: 2, previous_error_code: 'PAGE_WRITE_INCOMPLETE',
    fix_release: 'public-20260919-061500', published_at: '2026-09-18T22:00:00Z' }
  const reason = 'Deployed bounded complete-page repair with persistence regression tests'
  assert.deepEqual(planRecovery([source], [pilot, failed, previous], 3, reason, review), [source])
  const completed = { ...previous, attempt_number: 3, recovery_status: 'wiki_recovered' }
  assert.deepEqual(planRecovery([source], [pilot, failed, previous, completed], 3, reason, review), [])
  for (const change of [{ tenant_id: 8 }, { knowledge_id: 'other' }, { model_id: 'other' }, { dead_letter_id: 17 },
    { previous_error_code: 'OTHER' }, { max_attempt: 4 }, { published_at: previous.queued_at }]) {
    assert.throws(() => planRecovery([source], [pilot, failed, previous], 3, reason, { ...review, ...change }))
  }
  assert.throws(() => planRecovery([source], [pilot, failed, { ...previous, pending: true }], 3, reason, review), /Wait/)
  assert.throws(() => planRecovery([source], [pilot, failed, previous, completed], 4, reason, review), /exactly this attempt/)
})

test('final recovery requires the published immutable API image to be running', () => {
  const review = { fix_release: 'public-20260919-061500' }
  const manifest = { stage: 'published', release: review.fix_release, api_image_id: `sha256:${'a'.repeat(64)}`, published_at: '2026-09-18T22:00:00Z' }
  const runtime = { running: true, release: manifest.release, image_id: manifest.api_image_id }
  assert.deepEqual(validatePublishedRecoveryFix(review, manifest, runtime), { ...review, published_at: manifest.published_at })
  assert.equal(review.published_at, undefined)
  for (const change of [{ running: false }, { release: 'public-20260918-000000' }, { image_id: `sha256:${'b'.repeat(64)}` }]) {
    assert.throws(() => validatePublishedRecoveryFix(review, manifest, { ...runtime, ...change }))
  }
  assert.throws(() => validatePublishedRecoveryFix(review, { ...manifest, stage: 'origin-ready' }, runtime), /Publish/)
  assert.throws(() => validatePublishedRecoveryFix(review, { ...manifest, release: 'public-20260918-000000' }, runtime), /current release/)
  assert.throws(() => validatePublishedRecoveryFix(undefined, manifest, runtime), /persisted/)
})

test('triggering a final recovery revalidates the current source and persisted receipt scope', () => {
  const source = { id: 16, tenant_id: 7, knowledge_id: 'doc', knowledge_base_id: 'kb', model_id: 'model', eligible: true, chunks: 1, pending: 1 }
  const previous = { ...failed, tenant_id: 7, knowledge_id: 'doc', knowledge_base_id: 'kb', attempt_number: 2,
    error_code: 'PAGE_WRITE_INCOMPLETE', new_failures: 1, queued_at: '2026-09-18T21:00:00Z' }
  const review = { tenant_id: 7, knowledge_id: 'doc', knowledge_base_id: 'kb', model_id: 'model', dead_letter_id: 16,
    max_attempt: 3, previous_attempt: 2, previous_error_code: 'PAGE_WRITE_INCOMPLETE',
    fix_release: 'public-20260919-061500', published_at: '2026-09-18T22:00:00Z' }
  const receipt = { ...previous, attempt_number: 3, pending: true, pending_op_id: 99,
    retry_reason: 'Deployed bounded complete-page repair with persistence regression tests', deployed_fix_review: review }
  const history = [pilot, failed, previous, receipt]
  assert.doesNotThrow(() => validateFinalRecoveryTrigger(receipt, source, history, review))
  for (const key of ['tenant_id', 'knowledge_id', 'knowledge_base_id', 'model_id']) {
    assert.throws(() => validateFinalRecoveryTrigger(receipt, { ...source, [key]: 'other' }, history, review), /Source changed/)
    assert.throws(() => validateFinalRecoveryTrigger({ ...receipt, [key]: 'other' }, source, history, review), /receipt scope/)
  }
  assert.throws(() => validateFinalRecoveryTrigger({ ...receipt, dead_letter_id: 17 }, source, history, review), /receipt scope/)
  assert.throws(() => validateFinalRecoveryTrigger(receipt, { ...source, eligible: false }, history, review), /eligible/)
  assert.throws(() => validateFinalRecoveryTrigger(receipt, source, [pilot, failed, previous, { ...receipt, pending: false }], review), /current and pending/)
  assert.throws(() => validateFinalRecoveryTrigger(receipt, source, [pilot, failed, previous, { ...receipt, pending_op_id: 100 }], review), /current and pending/)
  assert.throws(() => validateFinalRecoveryTrigger({ ...receipt, attempt_number: 4 }, source, history, review), /bounded attempt/)
})

test('a fourth recovery requires a new exact review; old approvals and a fifth attempt are refused', () => {
  const source = { id: 10, tenant_id: 7, knowledge_id: 'doc', knowledge_base_id: 'kb', model_id: 'model', pending: 0 }
  const previous = { ...failed, ...source, dead_letter_id: 10, attempt_number: 3, error_code: 'MAP_FAILED', new_failures: 1,
    queued_at: '2026-09-18T22:29:48Z' }
  const review = { ...source, dead_letter_id: 10, max_attempt: 4, previous_attempt: 3, previous_error_code: 'MAP_FAILED',
    fix_release: 'public-20260919-170000', published_at: '2026-09-19T09:00:00Z' }
  const reason = 'Published exact candidate handles and full-source window checkpoints, verified against the failing source'
  assert.deepEqual(planRecovery([source], [pilot, previous], 4, reason, review), [source])
  assert.throws(() => planRecovery([source], [pilot, previous], 4, reason), /deployed-fix review/)
  for (const change of [{ max_attempt: 3 }, { previous_attempt: 2 }, { previous_error_code: 'PAGE_WRITE_INCOMPLETE' },
    { knowledge_id: 'another-source' }, { published_at: previous.queued_at }]) {
    assert.throws(() => planRecovery([source], [pilot, previous], 4, reason, { ...review, ...change }))
  }
  assert.throws(() => planRecovery([source], [pilot, previous], 5, reason, { ...review, max_attempt: 5 }), /preceding attempt/)
  const receipt = { ...previous, attempt_number: 4, pending: true, pending_op_id: 101, retry_reason: reason }
  assert.doesNotThrow(() => validateFinalRecoveryTrigger(receipt, { ...source, eligible: true, chunks: 624, pending: 1 }, [pilot, previous, receipt], review))
  assert.deepEqual(planRecovery([source], [pilot, previous, receipt], 4, reason, review), [])
  assert.equal(previous.attempt_number, 3)
})

function fifthFixture() {
  const scope = fifthRecoveryScope
  const source = { ...scope, id: scope.dead_letter_id, pending: 0, eligible: true, chunks: 624,
    source_fingerprint: 'a'.repeat(64), model_fingerprint: 'b'.repeat(64), embedding_model_id: 'embedding' }
  const previous = { ...scope, attempt_number: 4, pending_op_id: 4143, recovery_status: 'failed', new_failures: 1,
    error_code: 'EXTRACT_FAILED', queued_at: '2026-09-19T01:00:00Z' }
  const review = { ...source, max_attempt: 5, previous_attempt: 4, previous_pending_op_id: 4143,
    previous_error_code: previous.error_code, budget_version: 1, api_image_id: `sha256:${'c'.repeat(64)}`,
    max_provider_requests: 200, deadline_at: new Date(Date.now() + 3600000).toISOString(),
    fix_release: 'public-20260920-180000', published_at: '2026-09-20T10:00:00Z' }
  const budget = { ...scope, attempt_number: 5, pending_op_id: 5000, source_fingerprint: source.source_fingerprint,
    model_fingerprint: source.model_fingerprint, embedding_model_id: source.embedding_model_id,
    max_provider_requests: review.max_provider_requests, reserved_provider_requests: 12, deadline_at: review.deadline_at,
    reviewed_release: review.fix_release, reviewed_api_image: review.api_image_id, status: 'active' }
  const receipt = { ...previous, attempt_number: 5, pending_op_id: 5000, pending: true, recovery_budget: budget,
    retry_reason: 'Verified persisted whole-operation budget and bounded candidate repair' }
  return { source, previous, review, receipt, pilot: { ...pilot, model_id: scope.model_id } }
}

test('only the exact reviewed fifth source is accepted; identities, old budgets and sixth attempts fail closed', () => {
  const { source, previous, review, receipt, pilot } = fifthFixture()
  const reason = receipt.retry_reason
  assert.deepEqual(planRecovery([source], [pilot, previous], 5, reason, review), [source])
  assert.deepEqual(planRecovery([source], [pilot, previous, receipt], 5, reason, review), [])
  for (const key of ['tenant_id', 'knowledge_id', 'knowledge_base_id', 'model_id', 'source_fingerprint', 'model_fingerprint', 'embedding_model_id']) {
    assert.throws(() => planRecovery([{ ...source, [key]: 'changed' }], [pilot, previous], 5, reason, review))
  }
  for (const change of [{ id: 11 }, { tenant_id: 10060 }]) assert.throws(() => planRecovery([{ ...source, ...change }], [pilot, previous], 5, reason, review))
  for (const change of [{ previous_pending_op_id: 4142 }, { max_provider_requests: 0 }, { budget_version: 0 },
    { deadline_at: 'invalid' }, { deadline_at: '2020-01-01T00:00:00Z' }, { source_fingerprint: '' }, { api_image_id: '' }]) {
    assert.throws(() => planRecovery([source], [pilot, previous], 5, reason, { ...review, ...change }))
  }
  assert.throws(() => planRecovery([source], [pilot, previous], 6, reason, { ...review, max_attempt: 6 }), /preceding attempt/)
  assert.throws(() => planRecovery([source], [pilot, { ...previous, pending_op_id: 4142 }], 5, reason, review), /exact fourth/)
})

test('fifth trigger requires the immutable active budget; exhaustion, expiry, missing budget and reset counters are rejected', () => {
  const { source, previous, review, receipt, pilot } = fifthFixture()
  const triggerSource = { ...source, pending: 1 }
  assert.doesNotThrow(() => validateFinalRecoveryTrigger(receipt, triggerSource, [pilot, previous, receipt], review))
  assert.throws(() => validateRecoveryBudget({ ...receipt, recovery_budget: null }, review), /missing/)
  for (const change of [{ status: 'completed' }, { status: 'stopped_budget' }, { tenant_id: 9 }, { pending_op_id: 5001 },
    { reserved_provider_requests: -1 }, { reserved_provider_requests: 200 }, { max_provider_requests: 201 },
    { deadline_at: new Date(Date.parse(review.deadline_at) + 1).toISOString() }, { reviewed_release: 'other' }]) {
    const changed = { ...receipt, recovery_budget: { ...receipt.recovery_budget, ...change } }
    assert.throws(() => validateFinalRecoveryTrigger(changed, triggerSource, [pilot, previous, changed], review))
  }
  assert.throws(() => validateRecoveryBudget(receipt, review, Date.parse(review.deadline_at)), /expired/)
  assert.equal(durableRecoveryStatus({ ...receipt, recovery_budget: { ...receipt.recovery_budget, status: 'stopped_budget' } }), 'stopped_budget')
  assert.equal(durableRecoveryStatus({ ...receipt, recovery_budget: null }), 'stopped_invalid')
  assert.equal(durableRecoveryStatus(receipt), null)
})

test('new first attempts share the durable review and trigger contract without resetting old receipts', () => {
  const fifth = fifthFixture()
  const source = { ...fifth.source, id: 17, knowledge_id: 'next-doc' }
  const review = { ...fifth.review, dead_letter_id: 17, knowledge_id: source.knowledge_id, max_attempt: 1 }
  assert.deepEqual(planRecovery([source], [fifth.pilot], 1, '', review), [source])
  const receipt = { ...source, dead_letter_id: source.id, attempt_number: 1, pending_op_id: 5001, pending: true,
    deployed_fix_review: review, recovery_budget: { ...fifth.receipt.recovery_budget, dead_letter_id: source.id,
      knowledge_id: source.knowledge_id, attempt_number: 1, pending_op_id: 5001 } }
  assert.deepEqual(planRecovery([source], [fifth.pilot, receipt], 1, '', review), [])
  assert.doesNotThrow(() => validateFinalRecoveryTrigger(receipt, { ...source, pending: 1 }, [fifth.pilot, receipt], review))
  assert.equal(durableRecoveryStatus({ ...receipt, recovery_budget: null }), 'stopped_invalid')
  for (const change of [{ source_fingerprint: '' }, { model_id: 'different' }, { max_provider_requests: 0 }, { max_attempt: 2 }]) {
    assert.throws(() => planRecovery([source], [fifth.pilot], 1, '', { ...review, ...change }))
  }
})

test('an explicitly closed terminal attempt does not block a new first attempt', () => {
  const closed = { dead_letter_id: 10, attempt_number: 7, model_id: 'model', recovery_status: 'stopped_budget',
    error_code: 'EXTRACT_FAILED', pending: false, pending_finalize: 0, closed: true }
  const open = { ...closed, closed: false }
  // An unclosed terminal state is still pending investigation.
  assert.throws(() => planRecovery([{ id: 4, model_id: 'model', pending: 0 }], [pilot, open], 1, ''), /unresolved/)
  // A recorded closure turns the same row into a decision, not outstanding work.
  assert.deepEqual(planRecovery([{ id: 4, model_id: 'model', pending: 0 }], [pilot, closed], 1, ''),
    [{ id: 4, model_id: 'model', pending: 0 }])
  // Closure never excuses work that is still running or backed by an active budget.
  assert.throws(() => planRecovery([{ id: 4, model_id: 'model', pending: 0 }], [pilot, { ...closed, pending: true }], 1, ''), /Wait/)
  assert.throws(() => planRecovery([{ id: 4, model_id: 'model', pending: 0 }], [pilot, { ...closed, pending_finalize: 1 }], 1, ''), /Wait/)
  // Only terminal states may be closed, and a closure cannot be used to justify a retry of itself.
  assert.throws(() => planRecovery([{ id: 10, model_id: 'model', pending: 0 }], [pilot, { ...closed, recovery_status: 'failed' }], 1, ''), /unresolved/)
})

test('a model with no prior attempt may run its first pilot, an unproven one may not', () => {
  const closedTerminalMimo = { dead_letter_id: 12, attempt_number: 4, model_id: 'mimo', recovery_status: 'stopped_budget',
    error_code: 'EXTRACT_FAILED', pending: false, pending_finalize: 0, closed: true }
  const mimoSuccess = { dead_letter_id: 13, attempt_number: 1, model_id: 'mimo', recovery_status: 'wiki_recovered', pending: false }
  const freshMimo = { id: 3, model_id: 'mimo', pending: 0 }
  const freshMimoAgain = { id: 5, model_id: 'mimo', pending: 0 }
  // First attempt for a model that has never been tried is the model's own pilot.
  assert.deepEqual(planRecovery([freshMimo], [pilot], 1, ''), [freshMimo])
  // Once a model has a successful pilot, further first attempts are allowed.
  assert.deepEqual(planRecovery([freshMimoAgain], [pilot, mimoSuccess], 1, ''), [freshMimoAgain])
  // A model whose only attempts ended terminal (and were closed) still has no pilot.
  assert.throws(() => planRecovery([freshMimoAgain], [pilot, closedTerminalMimo], 1, ''), /successful pilot/)
})

test('a reviewed retry accepts a terminal budget stop as the failed predecessor', () => {
  const source = { id: 3, tenant_id: 10059, knowledge_id: 'k', knowledge_base_id: 'kb', model_id: 'm', pending: 0,
    source_fingerprint: 'a'.repeat(64), model_fingerprint: 'b'.repeat(64), embedding_model_id: 'e' }
  const review = { budget_version: 1, max_attempt: 3, previous_attempt: 2, previous_error_code: 'EXTRACT_FAILED',
    fix_release: 'public-20260921-222500', published_at: '2026-09-21T15:00:00Z', api_image_id: 'sha256:' + 'c'.repeat(64),
    deadline_at: '2026-09-21T18:00:00Z', max_provider_requests: 700, tenant_id: 10059, dead_letter_id: 3,
    knowledge_id: 'k', knowledge_base_id: 'kb', model_id: 'm', source_fingerprint: 'a'.repeat(64), model_fingerprint: 'b'.repeat(64),
    embedding_model_id: 'e' }
  const earlierSuccess = { tenant_id: 10059, dead_letter_id: 2, attempt_number: 2, model_id: 'm', recovery_status: 'wiki_recovered', pending: false }
  const prior = attempt => [earlierSuccess, { tenant_id: 10059, dead_letter_id: 3, attempt_number: 2, knowledge_id: 'k', knowledge_base_id: 'kb',
    model_id: 'm', error_code: 'EXTRACT_FAILED', queued_at: '2026-09-21T14:43:54Z', new_failures: 0, pending: false,
    recovery_status: attempt, recovery_budget: { status: attempt, deadline_at: '2026-09-21T17:15:00Z', max_provider_requests: 700,
      reserved_provider_requests: 2, tenant_id: 10059, dead_letter_id: 3, knowledge_id: 'k', knowledge_base_id: 'kb', model_id: 'm',
      attempt_number: 2, pending_op_id: 4150, source_fingerprint: 'a'.repeat(64), model_fingerprint: 'b'.repeat(64),
      embedding_model_id: 'e', reviewed_release: 'public-20260921-222500',
      reviewed_api_image: 'sha256:' + 'c'.repeat(64) } }]
  const predecessor = attempt => prior(attempt).find(row => row.dead_letter_id === 3)
  assert.equal(resolvedRecoveryFailure(predecessor('stopped_budget')), true)
  assert.equal(resolvedRecoveryFailure(predecessor('failed')), true)
  assert.equal(resolvedRecoveryFailure(predecessor('wiki_recovered')), false)
  const plan = planRecovery([source], prior('stopped_budget'), 3, 'Reviewed retry after the terminal stop', review)
  assert.equal(plan.length, 1)
  assert.throws(() => planRecovery([source], prior('wiki_recovered'), 3, 'Reviewed retry after success', review),
    /requires a failed preceding attempt/)
})