const assert = require('node:assert/strict') const { validateSixthRecovery } = require('./wiki_sixth_recovery_review.cjs') const { validateSeventhRecovery } = require('./wiki_seventh_recovery_review.cjs') const unresolved = row => ['failed', 'partial', 'unverified', 'skipped', 'stopped_budget', 'stopped_invalid'].includes(row.recovery_status) // A terminal attempt that carries an explicit closure record is a recorded // decision, not outstanding work: it can never continue, so it must not block // unrelated first attempts. Anything still running, or without a closure, keeps // blocking; closure is never accepted for a non-terminal outcome. const terminalRecoveryStatuses = ['stopped_budget', 'stopped_invalid'] const closedTerminal = row => Boolean(row) && row.closed === true && terminalRecoveryStatuses.includes(row.recovery_status) && !row.pending && !(row.pending_finalize > 0) const blocksNewWork = row => unresolved(row) && !closedTerminal(row) const attemptOf = row => Number(row.attempt_number || 1) // Manual recovery is bounded. The fourth attempt is reserved for a separately // published citation/full-reading fix; the scheduler never advances attempts. const maxReviewedAttempt = 4 const fifthRecoveryScope = Object.freeze({ tenant_id: 10059, dead_letter_id: 10, knowledge_id: 'd52e5191-1129-4934-be64-3dcc0651e52b', knowledge_base_id: '5d28a7e5-e00e-44a7-a0e1-c72342cdcf68', model_id: '404e005b-83ae-5e2e-a3b3-999f9d5cd47e' }) const supportedReviewedAttempt = attempt => Number.isSafeInteger(attempt) && attempt >= 3 && (attempt <= maxReviewedAttempt || attempt === 5 || attempt === 6 || attempt === 7) function validateFifthRecovery(source, previous, review) { for (const [key, value] of Object.entries(fifthRecoveryScope)) { assert.equal(key === 'dead_letter_id' ? source.id : source[key], value, `Fifth recovery source mismatch: ${key}`) assert.equal(review[key], value, `Fifth recovery review mismatch: ${key}`) } assert.equal(previous?.pending_op_id, 4143, 'Fifth recovery requires the exact fourth pending operation') assert.equal(review.previous_pending_op_id, 4143, 'Review must identify the exact fourth pending operation') assert.equal(previous.error_code, 'EXTRACT_FAILED', 'Fifth recovery requires the reviewed extraction failure') validateDurableRecoveryReview(source, review, 5) } // A reviewed retry needs the immediately preceding attempt to have resolved as a // failure. A terminal budget state (stopped_budget / stopped_invalid) is such a // failure: it can never continue, so requiring a live dead letter as well would // make those documents permanently unrecoverable. function resolvedRecoveryFailure(row) { if (!row) return false if (row.recovery_status === 'wiki_recovered') return false if (row.recovery_status === 'failed') return true return terminalRecoveryStatuses.includes(row.recovery_status) || row.recovery_budget?.status === 'stopped_budget' || row.recovery_budget?.status === 'stopped_invalid' } function validateDurableRecoveryReview(source, review, attempt) { assert.equal(review?.budget_version, 1, 'Every new recovery requires durable budget version 1') assert.equal(review.max_attempt, attempt, 'Review must permit exactly this attempt') assert.match(review.fix_release || '', /^public-\d{8}-\d{6}$/, 'Review must identify a published fix release') for (const key of ['tenant_id', 'knowledge_id', 'knowledge_base_id', 'model_id']) { assert.ok(source[key] !== undefined && source[key] === review[key], `Recovery review scope mismatch: ${key}`) } assert.equal(source.id, review.dead_letter_id, 'Review identifies a different original record') for (const key of ['source_fingerprint', 'model_fingerprint']) { assert.match(review[key] || '', /^[a-f0-9]{64}$/, `Missing reviewed ${key}`) assert.equal(source[key], review[key], `Source changed since review: ${key}`) } assert.equal(source.embedding_model_id, review.embedding_model_id, 'Embedding model changed since review') assert.match(review.api_image_id || '', /^sha256:[a-f0-9]{64}$/, 'Recovery requires the exact API image') assert.ok(Number.isSafeInteger(review.max_provider_requests) && review.max_provider_requests > 0, 'Recovery needs an explicit provider request budget') assert.ok(Number.isFinite(Date.parse(review.deadline_at)), 'Recovery needs a fixed deadline') } function validateRecoveryBudget(row, review, now = Date.now()) { const budget = row.recovery_budget assert.ok(budget, 'Recovery budget is missing; no trigger is allowed') for (const key of ['tenant_id', 'dead_letter_id', 'knowledge_id', 'knowledge_base_id', 'model_id', 'attempt_number', 'pending_op_id']) { assert.equal(budget[key], row[key], `Recovery budget identity mismatch: ${key}`) } for (const key of ['source_fingerprint', 'model_fingerprint', 'embedding_model_id', 'max_provider_requests']) { assert.equal(budget[key], review[key], `Recovery budget changed: ${key}`) } assert.equal(Date.parse(budget.deadline_at), Date.parse(review.deadline_at), 'Recovery deadline cannot be extended') assert.equal(budget.reviewed_release, review.fix_release, 'Recovery budget release mismatch') assert.equal(budget.reviewed_api_image, review.api_image_id, 'Recovery budget image mismatch') assert.equal(budget.status, 'active', 'Recovery budget is terminal; no trigger is allowed') assert.ok(Date.parse(budget.deadline_at) > now, 'Recovery deadline has expired') assert.ok(Number.isSafeInteger(budget.reserved_provider_requests) && budget.reserved_provider_requests >= 0 && budget.reserved_provider_requests < budget.max_provider_requests, 'Recovery provider request budget exhausted') } function durableRecoveryStatus(row) { const budget = row.recovery_budget if (!budget) return attemptOf(row) >= 5 || row.deployed_fix_review?.budget_version === 1 ? 'stopped_invalid' : null if (budget.status !== 'active' && budget.status !== 'completed') return budget.status === 'stopped_budget' ? 'stopped_budget' : 'stopped_invalid' if (budget.status === 'active' && Date.parse(budget.deadline_at) <= Date.now()) return 'stopped_budget' return null } function describeReadCoverage(row) { // Legacy pages and a completed prefix summary do not prove whole-source // reading. Source processing is also distinct from semantic verification. if (row.read_coverage_version === 'wiki-full-read-v1' && row.source_read_complete === true) return 'source_processed' const stages = Object.values(row.read_progress || {}) if (row.span_status === 'failed' || row.recovery_budget?.stop_reason === 'phase_failed') return 'failed' if (['stopped_budget', 'stopped_invalid'].includes(row.recovery_budget?.status)) return row.recovery_budget.status if (stages.some(stage => stage.status === 'paused_budget')) return 'paused_budget' if (stages.length) return 'in_progress' return 'not_verified' } function latestAttempts(rows) { const latest = new Map() for (const row of rows) { const previous = latest.get(row.dead_letter_id) if (!previous || attemptOf(row) > attemptOf(previous)) latest.set(row.dead_letter_id, row) } return [...latest.values()] } function planRecovery(selected, history, attempt, reason, review) { assert.ok(attempt === 1 || attempt === 2 || (supportedReviewedAttempt(attempt) && review), 'Only first and explicitly audited second attempts are supported without a bounded deployed-fix review') assert.equal(selected.length, 1, 'Recovery applies exactly one original record per batch') if (attempt > 1) assert.ok(reason.trim().length >= 12, 'Another attempt requires --reason= describing the investigated failure and deployed fix') if (review?.budget_version === 1) validateDurableRecoveryReview(selected[0], review, attempt) if (attempt >= 3) { assert.equal(review.max_attempt, attempt, 'Review must permit exactly this attempt, not an unbounded retry') assert.equal(review.previous_attempt, attempt - 1, 'Review must identify the immediately preceding attempt') assert.match(review.fix_release || '', /^public-\d{8}-\d{6}$/, 'Review must identify a published fix release') const source = selected[0] const previous = history.find(row => row.dead_letter_id === source.id && attemptOf(row) === attempt - 1) if (attempt === 7) validateSeventhRecovery(source, previous, review) else if (attempt === 6) validateSixthRecovery(source, previous, review) else assert.ok(resolvedRecoveryFailure(previous), 'Reviewed recovery requires a failed preceding attempt') if (attempt === 5) validateFifthRecovery(source, previous, review) for (const key of ['tenant_id', 'knowledge_id', 'knowledge_base_id', 'model_id']) { assert.ok(review[key] !== undefined && review[key] === previous[key], `Review scope mismatch: ${key}`) assert.equal(source[key], review[key], `Source changed since review: ${key}`) } assert.equal(review.dead_letter_id, source.id, 'Review identifies a different original record') assert.ok(review.previous_error_code && review.previous_error_code === previous.error_code, 'Reviewed failure no longer matches') assert.ok(Date.parse(review.published_at) > Date.parse(previous.queued_at), 'Fix release must postdate the reviewed failed attempt') } const latest = latestAttempts(history) const fresh = selected.filter(row => !history.some(prior => prior.dead_letter_id === row.id && attemptOf(prior) === attempt)) if (!fresh.length) return [] if (review?.budget_version === 1) assert.ok(Date.parse(review.deadline_at) > Date.now(), 'Recovery deadline has expired') assert.ok(!latest.some(row => row.pending || row.pending_finalize > 0), 'Wait for current recovery and Wiki finalization before enqueueing another') assert.ok(fresh.every(row => row.pending === 0), 'A selected document already has a pending operation') if (attempt > 1) { assert.ok(fresh.every(row => latest.some(prior => prior.dead_letter_id === row.id && attemptOf(prior) === attempt - 1 && unresolved(prior))), 'Another attempt requires an unresolved previous attempt') } else { assert.ok(!latest.some(blocksNewWork), 'Investigate unresolved recovery outcomes before more work') } // A model that has never been attempted runs its first (pilot) attempt. Once a // model has attempts, one of them must have completed: an unproven model is // never scaled out across the remaining originals. if (latest.length > 0) assert.ok(fresh.every(row => !latest.some(prior => prior.model_id === row.model_id) || latest.some(prior => prior.model_id === row.model_id && prior.recovery_status === 'wiki_recovered')), 'A successful pilot for this model is required') return fresh } function validatePublishedRecoveryFix(review, manifest, runtime) { assert.ok(review, 'Final recovery requires its persisted deployed-fix review') assert.equal(manifest.stage, 'published', 'Publish and verify the fix before another recovery') assert.equal(review.fix_release, manifest.release, 'Reviewed fix is not the current release') assert.equal(runtime.release, manifest.release, 'Active API does not match the reviewed release') assert.match(manifest.api_image_id || '', /^sha256:[a-f0-9]{64}$/, 'Published release must identify an immutable API image') assert.equal(runtime.image_id, manifest.api_image_id, 'Active API image does not match the published fix') assert.equal(runtime.running, true, 'Reviewed API must be running') if (review.budget_version === 1 || review.max_attempt === 5) assert.equal(review.api_image_id, manifest.api_image_id, 'Recovery image must match the published fix') return { ...review, published_at: manifest.published_at } } function validateFinalRecoveryTrigger(row, source, history, review) { const attempt = attemptOf(row) assert.ok((attempt === 1 || attempt === 2 || supportedReviewedAttempt(attempt)) && review?.max_attempt === attempt, 'Reviewed recovery trigger must match its bounded attempt') assert.ok(source?.eligible && source.chunks > 0 && source.pending > 0, 'Reviewed source is no longer eligible or pending') for (const key of ['tenant_id', 'dead_letter_id', 'knowledge_id', 'knowledge_base_id', 'model_id']) { assert.ok(review[key] !== undefined && row[key] === review[key], `Recovery receipt scope mismatch: ${key}`) } const current = latestAttempts(history).find(saved => saved.dead_letter_id === row.dead_letter_id) assert.ok(current?.pending && attemptOf(current) === attempt && current.pending_op_id === row.pending_op_id, 'Final recovery receipt is no longer current and pending') assert.deepEqual(planRecovery([source], history, attempt, row.retry_reason || '', review), [], 'Trigger requires an existing final recovery receipt') if (review?.budget_version === 1 || attempt === 5) validateRecoveryBudget(current, review) } module.exports = { attemptOf, latestAttempts, planRecovery, validatePublishedRecoveryFix, validateFinalRecoveryTrigger, maxReviewedAttempt, describeReadCoverage, fifthRecoveryScope, supportedReviewedAttempt, validateRecoveryBudget, durableRecoveryStatus, validateDurableRecoveryReview, resolvedRecoveryFailure, terminalRecoveryStatuses, closedTerminal, blocksNewWork }