xinmi/scripts/wiki_recovery_policy.cjs

const assert = require('node:assert/strict')
const { validateSixthRecovery } = require('./wiki_sixth_recovery_review.cjs')
const { validateSeventhRecovery } = require('./wiki_seventh_recovery_review.cjs')

const unresolved = row => ['failed', 'partial', 'unverified', 'skipped', 'stopped_budget', 'stopped_invalid'].includes(row.recovery_status)
// A terminal attempt that carries an explicit closure record is a recorded
// decision, not outstanding work: it can never continue, so it must not block
// unrelated first attempts. Anything still running, or without a closure, keeps
// blocking; closure is never accepted for a non-terminal outcome.
const terminalRecoveryStatuses = ['stopped_budget', 'stopped_invalid']
const closedTerminal = row => Boolean(row) && row.closed === true &&
  terminalRecoveryStatuses.includes(row.recovery_status) && !row.pending && !(row.pending_finalize > 0)
const blocksNewWork = row => unresolved(row) && !closedTerminal(row)
const attemptOf = row => Number(row.attempt_number || 1)
// Manual recovery is bounded. The fourth attempt is reserved for a separately
// published citation/full-reading fix; the scheduler never advances attempts.
const maxReviewedAttempt = 4
const fifthRecoveryScope = Object.freeze({ tenant_id: 10059, dead_letter_id: 10,
  knowledge_id: 'd52e5191-1129-4934-be64-3dcc0651e52b',
  knowledge_base_id: '5d28a7e5-e00e-44a7-a0e1-c72342cdcf68',
  model_id: '404e005b-83ae-5e2e-a3b3-999f9d5cd47e' })
const supportedReviewedAttempt = attempt => Number.isSafeInteger(attempt) && attempt >= 3 && (attempt <= maxReviewedAttempt || attempt === 5 || attempt === 6 || attempt === 7)

function validateFifthRecovery(source, previous, review) {
  for (const [key, value] of Object.entries(fifthRecoveryScope)) {
    assert.equal(key === 'dead_letter_id' ? source.id : source[key], value, `Fifth recovery source mismatch: ${key}`)
    assert.equal(review[key], value, `Fifth recovery review mismatch: ${key}`)
  }
  assert.equal(previous?.pending_op_id, 4143, 'Fifth recovery requires the exact fourth pending operation')
  assert.equal(review.previous_pending_op_id, 4143, 'Review must identify the exact fourth pending operation')
  assert.equal(previous.error_code, 'EXTRACT_FAILED', 'Fifth recovery requires the reviewed extraction failure')
  validateDurableRecoveryReview(source, review, 5)
}

// A reviewed retry needs the immediately preceding attempt to have resolved as a
// failure. A terminal budget state (stopped_budget / stopped_invalid) is such a
// failure: it can never continue, so requiring a live dead letter as well would
// make those documents permanently unrecoverable.
function resolvedRecoveryFailure(row) {
  if (!row) return false
  if (row.recovery_status === 'wiki_recovered') return false
  if (row.recovery_status === 'failed') return true
  return terminalRecoveryStatuses.includes(row.recovery_status) || row.recovery_budget?.status === 'stopped_budget' ||
    row.recovery_budget?.status === 'stopped_invalid'
}

function validateDurableRecoveryReview(source, review, attempt) {
  assert.equal(review?.budget_version, 1, 'Every new recovery requires durable budget version 1')
  assert.equal(review.max_attempt, attempt, 'Review must permit exactly this attempt')
  assert.match(review.fix_release || '', /^public-\d{8}-\d{6}$/, 'Review must identify a published fix release')
  for (const key of ['tenant_id', 'knowledge_id', 'knowledge_base_id', 'model_id']) {
    assert.ok(source[key] !== undefined && source[key] === review[key], `Recovery review scope mismatch: ${key}`)
  }
  assert.equal(source.id, review.dead_letter_id, 'Review identifies a different original record')
  for (const key of ['source_fingerprint', 'model_fingerprint']) {
    assert.match(review[key] || '', /^[a-f0-9]{64}$/, `Missing reviewed ${key}`)
    assert.equal(source[key], review[key], `Source changed since review: ${key}`)
  }
  assert.equal(source.embedding_model_id, review.embedding_model_id, 'Embedding model changed since review')
  assert.match(review.api_image_id || '', /^sha256:[a-f0-9]{64}$/, 'Recovery requires the exact API image')
  assert.ok(Number.isSafeInteger(review.max_provider_requests) && review.max_provider_requests > 0, 'Recovery needs an explicit provider request budget')
  assert.ok(Number.isFinite(Date.parse(review.deadline_at)), 'Recovery needs a fixed deadline')
}

function validateRecoveryBudget(row, review, now = Date.now()) {
  const budget = row.recovery_budget
  assert.ok(budget, 'Recovery budget is missing; no trigger is allowed')
  for (const key of ['tenant_id', 'dead_letter_id', 'knowledge_id', 'knowledge_base_id', 'model_id', 'attempt_number', 'pending_op_id']) {
    assert.equal(budget[key], row[key], `Recovery budget identity mismatch: ${key}`)
  }
  for (const key of ['source_fingerprint', 'model_fingerprint', 'embedding_model_id', 'max_provider_requests']) {
    assert.equal(budget[key], review[key], `Recovery budget changed: ${key}`)
  }
  assert.equal(Date.parse(budget.deadline_at), Date.parse(review.deadline_at), 'Recovery deadline cannot be extended')
  assert.equal(budget.reviewed_release, review.fix_release, 'Recovery budget release mismatch')
  assert.equal(budget.reviewed_api_image, review.api_image_id, 'Recovery budget image mismatch')
  assert.equal(budget.status, 'active', 'Recovery budget is terminal; no trigger is allowed')
  assert.ok(Date.parse(budget.deadline_at) > now, 'Recovery deadline has expired')
  assert.ok(Number.isSafeInteger(budget.reserved_provider_requests) && budget.reserved_provider_requests >= 0 &&
    budget.reserved_provider_requests < budget.max_provider_requests, 'Recovery provider request budget exhausted')
}

function durableRecoveryStatus(row) {
  const budget = row.recovery_budget
  if (!budget) return attemptOf(row) >= 5 || row.deployed_fix_review?.budget_version === 1 ? 'stopped_invalid' : null
  if (budget.status !== 'active' && budget.status !== 'completed') return budget.status === 'stopped_budget' ? 'stopped_budget' : 'stopped_invalid'
  if (budget.status === 'active' && Date.parse(budget.deadline_at) <= Date.now()) return 'stopped_budget'
  return null
}

function describeReadCoverage(row) {
  // Legacy pages and a completed prefix summary do not prove whole-source
  // reading. Source processing is also distinct from semantic verification.
  if (row.read_coverage_version === 'wiki-full-read-v1' && row.source_read_complete === true) return 'source_processed'
  const stages = Object.values(row.read_progress || {})
  if (row.span_status === 'failed' || row.recovery_budget?.stop_reason === 'phase_failed') return 'failed'
  if (['stopped_budget', 'stopped_invalid'].includes(row.recovery_budget?.status)) return row.recovery_budget.status
  if (stages.some(stage => stage.status === 'paused_budget')) return 'paused_budget'
  if (stages.length) return 'in_progress'
  return 'not_verified'
}

function latestAttempts(rows) {
  const latest = new Map()
  for (const row of rows) {
    const previous = latest.get(row.dead_letter_id)
    if (!previous || attemptOf(row) > attemptOf(previous)) latest.set(row.dead_letter_id, row)
  }
  return [...latest.values()]
}

function planRecovery(selected, history, attempt, reason, review) {
  assert.ok(attempt === 1 || attempt === 2 || (supportedReviewedAttempt(attempt) && review), 'Only first and explicitly audited second attempts are supported without a bounded deployed-fix review')
  assert.equal(selected.length, 1, 'Recovery applies exactly one original record per batch')
  if (attempt > 1) assert.ok(reason.trim().length >= 12, 'Another attempt requires --reason= describing the investigated failure and deployed fix')
  if (review?.budget_version === 1) validateDurableRecoveryReview(selected[0], review, attempt)
  if (attempt >= 3) {
    assert.equal(review.max_attempt, attempt, 'Review must permit exactly this attempt, not an unbounded retry')
    assert.equal(review.previous_attempt, attempt - 1, 'Review must identify the immediately preceding attempt')
    assert.match(review.fix_release || '', /^public-\d{8}-\d{6}$/, 'Review must identify a published fix release')
    const source = selected[0]
    const previous = history.find(row => row.dead_letter_id === source.id && attemptOf(row) === attempt - 1)
    if (attempt === 7) validateSeventhRecovery(source, previous, review)
    else if (attempt === 6) validateSixthRecovery(source, previous, review)
    else assert.ok(resolvedRecoveryFailure(previous), 'Reviewed recovery requires a failed preceding attempt')
    if (attempt === 5) validateFifthRecovery(source, previous, review)
    for (const key of ['tenant_id', 'knowledge_id', 'knowledge_base_id', 'model_id']) {
      assert.ok(review[key] !== undefined && review[key] === previous[key], `Review scope mismatch: ${key}`)
      assert.equal(source[key], review[key], `Source changed since review: ${key}`)
    }
    assert.equal(review.dead_letter_id, source.id, 'Review identifies a different original record')
    assert.ok(review.previous_error_code && review.previous_error_code === previous.error_code, 'Reviewed failure no longer matches')
    assert.ok(Date.parse(review.published_at) > Date.parse(previous.queued_at), 'Fix release must postdate the reviewed failed attempt')
  }
  const latest = latestAttempts(history)
  const fresh = selected.filter(row => !history.some(prior => prior.dead_letter_id === row.id && attemptOf(prior) === attempt))
  if (!fresh.length) return []
  if (review?.budget_version === 1) assert.ok(Date.parse(review.deadline_at) > Date.now(), 'Recovery deadline has expired')
  assert.ok(!latest.some(row => row.pending || row.pending_finalize > 0), 'Wait for current recovery and Wiki finalization before enqueueing another')
  assert.ok(fresh.every(row => row.pending === 0), 'A selected document already has a pending operation')
  if (attempt > 1) {
    assert.ok(fresh.every(row => latest.some(prior => prior.dead_letter_id === row.id && attemptOf(prior) === attempt - 1 && unresolved(prior))), 'Another attempt requires an unresolved previous attempt')
  } else {
    assert.ok(!latest.some(blocksNewWork), 'Investigate unresolved recovery outcomes before more work')
  }
  // A model that has never been attempted runs its first (pilot) attempt. Once a
  // model has attempts, one of them must have completed: an unproven model is
  // never scaled out across the remaining originals.
  if (latest.length > 0) assert.ok(fresh.every(row => !latest.some(prior => prior.model_id === row.model_id) ||
    latest.some(prior => prior.model_id === row.model_id && prior.recovery_status === 'wiki_recovered')), 'A successful pilot for this model is required')
  return fresh
}

function validatePublishedRecoveryFix(review, manifest, runtime) {
  assert.ok(review, 'Final recovery requires its persisted deployed-fix review')
  assert.equal(manifest.stage, 'published', 'Publish and verify the fix before another recovery')
  assert.equal(review.fix_release, manifest.release, 'Reviewed fix is not the current release')
  assert.equal(runtime.release, manifest.release, 'Active API does not match the reviewed release')
  assert.match(manifest.api_image_id || '', /^sha256:[a-f0-9]{64}$/, 'Published release must identify an immutable API image')
  assert.equal(runtime.image_id, manifest.api_image_id, 'Active API image does not match the published fix')
  assert.equal(runtime.running, true, 'Reviewed API must be running')
  if (review.budget_version === 1 || review.max_attempt === 5) assert.equal(review.api_image_id, manifest.api_image_id, 'Recovery image must match the published fix')
  return { ...review, published_at: manifest.published_at }
}

function validateFinalRecoveryTrigger(row, source, history, review) {
  const attempt = attemptOf(row)
  assert.ok((attempt === 1 || attempt === 2 || supportedReviewedAttempt(attempt)) && review?.max_attempt === attempt, 'Reviewed recovery trigger must match its bounded attempt')
  assert.ok(source?.eligible && source.chunks > 0 && source.pending > 0, 'Reviewed source is no longer eligible or pending')
  for (const key of ['tenant_id', 'dead_letter_id', 'knowledge_id', 'knowledge_base_id', 'model_id']) {
    assert.ok(review[key] !== undefined && row[key] === review[key], `Recovery receipt scope mismatch: ${key}`)
  }
  const current = latestAttempts(history).find(saved => saved.dead_letter_id === row.dead_letter_id)
  assert.ok(current?.pending && attemptOf(current) === attempt && current.pending_op_id === row.pending_op_id, 'Final recovery receipt is no longer current and pending')
  assert.deepEqual(planRecovery([source], history, attempt, row.retry_reason || '', review), [], 'Trigger requires an existing final recovery receipt')
  if (review?.budget_version === 1 || attempt === 5) validateRecoveryBudget(current, review)
}

module.exports = { attemptOf, latestAttempts, planRecovery, validatePublishedRecoveryFix, validateFinalRecoveryTrigger, maxReviewedAttempt,
  describeReadCoverage, fifthRecoveryScope, supportedReviewedAttempt, validateRecoveryBudget, durableRecoveryStatus, validateDurableRecoveryReview,
  resolvedRecoveryFailure,
  terminalRecoveryStatuses, closedTerminal, blocksNewWork }