xinmi/scripts/recover_wiki_deadletters.cjs

const fs = require('node:fs')
const path = require('node:path')
const assert = require('node:assert/strict')
const { execFileSync } = require('node:child_process')
const { attemptOf, latestAttempts, planRecovery, validatePublishedRecoveryFix, validateFinalRecoveryTrigger, supportedReviewedAttempt, describeReadCoverage, durableRecoveryStatus, validateDurableRecoveryReview } = require('./wiki_recovery_policy.cjs')
const { recoveryFingerprintsSQL } = require('./wiki_recovery_budget_sql.cjs')
const { loadClosures, annotateClosed } = require('./wiki_recovery_closures.cjs')
const { validateSixthDiagnosticEvidence, validateSixthReplayEvidence, verifyReplayInputFiles, replayPins, candidateSourceTreeHash, sixthGuardPausedOperation } = require('./wiki_sixth_recovery_review.cjs')
const { seventhPins, validateSeventhCorrectionEvidence, verifySeventhBuildFiles, seventhRecoveryEnqueueSQL } = require('./wiki_seventh_recovery_review.cjs')
const fingerprints = recoveryFingerprintsSQL()

const root = path.resolve(__dirname, '../..')
const directory = path.join(root, 'acceptance/wiki-deadletter-recovery-20260919')
const tenant = 10059
const database = 'yawan_native_acceptance_20260915'
const table = 'wiki_dead_letter_recoveries'
// The service dead-letters at 6. Starting at 5 permits one new recovery
// attempt, without repeating the historical six-attempt model-call burst.
const recoveryInitialFailCount = 5
const args = process.argv.slice(2)
const apply = args.includes('--apply')
const triggerOnly = args.includes('--trigger')
const idsArg = args.find(arg => arg.startsWith('--ids='))?.slice(6)
const ids = idsArg ? idsArg.split(',').map(Number) : []
const attempt = Number(args.find(arg => arg.startsWith('--attempt='))?.slice(10) || 1)
const retryReason = args.find(arg => arg.startsWith('--reason='))?.slice(9) || ''
const reviewPath = args.find(arg => arg.startsWith('--review-file='))?.slice(14)
assert.ok(ids.every(id => Number.isSafeInteger(id) && id >= 1 && id <= 33), 'Only original dead letters 1..33 are allowed')
assert.equal(new Set(ids).size, ids.length, 'Duplicate IDs')
assert.ok(!apply || ids.length === 1, 'Apply requires --ids= with exactly one original record')
assert.ok(!args.some(arg => !['--apply', '--trigger'].includes(arg) && !['--ids=', '--attempt=', '--reason=', '--review-file='].some(prefix => arg.startsWith(prefix))), 'Unknown argument')
assert.ok(attempt === 1 || attempt === 2 || (supportedReviewedAttempt(attempt) && reviewPath), 'Further bounded recovery requires an explicit deployed-fix review file')
assert.ok(attempt === 1 || apply, 'Another attempt requires --apply')
assert.ok(!apply || reviewPath, 'Every new recovery requires --review-file= with an explicit persistent request limit and deadline')
assert.ok(!reviewPath || apply, 'Review files are only valid for a bounded manual recovery')
const literal = value => `'${String(value).replaceAll("'", "''")}'`
const docker = (args, input) => execFileSync('docker', ['--context', 'colima-yawan', ...args], {
  input, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], maxBuffer: 32 * 1024 * 1024,
})
function sql(statement, readOnly = true) {
  const args = ['exec', '-i']
  if (readOnly) args.push('-e', 'PGOPTIONS=-c default_transaction_read_only=on')
  return docker([...args, 'WeKnora-postgres', 'psql', '-X', '-q', '-U', 'postgres', '-d', database, '-At', '-v', 'ON_ERROR_STOP=1'], statement).trim()
}
const queryJSON = (statement, readOnly = true) => JSON.parse(sql(statement, readOnly))
const hasLedger = () => sql(`SELECT to_regclass('public.${table}') IS NOT NULL`) === 't'
const writePrivate = (name, value) => fs.writeFileSync(path.join(directory, name), JSON.stringify(value, null, 2) + '\n', { mode: 0o600, flag: 'wx' })
const jsonArray = expression => `CASE WHEN jsonb_typeof(${expression})='array' THEN ${expression} ELSE '[]'::jsonb END`
const hasSource = (page, knowledge) => `EXISTS (SELECT 1 FROM jsonb_array_elements_text(${jsonArray(`${page}.source_refs`)}) ref WHERE split_part(ref,'|',1)=${knowledge})`

function inventory(fingerprintIDs = ids.length ? ids : [10]) {
  return queryJSON(`SELECT coalesce(json_agg(row_to_json(x)), '[]') FROM (
    SELECT d.id,d.tenant_id,d.related_id AS knowledge_id,d.scope_id AS knowledge_base_id,k.name AS knowledge_base,
      k.wiki_config->>'synthesis_model_id' AS model_id,
      k.embedding_model_id,
      CASE WHEN d.id IN (${fingerprintIDs.join(',')}) THEN ${fingerprints.source} END AS source_fingerprint,
      CASE WHEN d.id IN (${fingerprintIDs.join(',')}) THEN ${fingerprints.model} END AS model_fingerprint,
      n.parse_status,n.pending_subtasks_count,
      (d.payload->>'op'='ingest' AND d.payload->>'knowledge_id'=d.related_id
       AND d.scope='knowledge_base' AND d.fail_count=6 AND n.deleted_at IS NULL
       AND k.deleted_at IS NULL AND k.indexing_strategy->>'wiki_enabled'='true'
       AND n.parse_status='completed' AND n.pending_subtasks_count=0
       AND EXISTS (SELECT 1 FROM models m WHERE m.id=k.wiki_config->>'synthesis_model_id' AND m.tenant_id=d.tenant_id AND m.type='KnowledgeQA' AND m.deleted_at IS NULL)) AS eligible,
      (SELECT count(*) FROM chunks c WHERE c.tenant_id=d.tenant_id AND c.knowledge_id=n.id AND c.deleted_at IS NULL) AS chunks,
      (SELECT coalesce(sum(length(c.content)),0) FROM chunks c WHERE c.tenant_id=d.tenant_id AND c.knowledge_id=n.id AND c.deleted_at IS NULL) AS chunk_chars,
      (SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=d.tenant_id AND p.knowledge_base_id=d.scope_id AND p.deleted_at IS NULL AND ${hasSource('p', 'd.related_id')}) AS pages,
      (SELECT count(*) FROM task_pending_ops o WHERE o.tenant_id=d.tenant_id AND o.scope_id=d.scope_id AND o.task_type='wiki:ingest' AND o.dedup_key=d.related_id) AS pending
    FROM task_dead_letters d
    JOIN knowledges n ON n.id=d.related_id AND n.tenant_id=d.tenant_id AND n.knowledge_base_id=d.scope_id
    JOIN knowledge_bases k ON k.id=d.scope_id AND k.tenant_id=d.tenant_id
    WHERE d.tenant_id=${tenant} AND d.task_type='wiki:ingest' AND d.id BETWEEN 1 AND 33
    ORDER BY chunk_chars,d.id
  ) x`)
}

function status() {
  if (!hasLedger()) return []
  const budgetsAvailable = sql("SELECT to_regclass('public.wiki_recovery_budgets') IS NOT NULL") === 't'
  const rows = queryJSON(`SELECT coalesce(json_agg(row_to_json(x)), '[]') FROM (
    SELECT r.*,coalesce((to_jsonb(r)->>'attempt_number')::int,1) AS attempt_number,o.claimed_at,o.fail_count,
      ${budgetsAvailable ? '(SELECT row_to_json(b) FROM wiki_recovery_budgets b WHERE b.tenant_id=r.tenant_id AND b.dead_letter_id=r.dead_letter_id AND b.attempt_number=r.attempt_number AND b.pending_op_id=r.pending_op_id)' : 'NULL::json'} AS recovery_budget,
      s.status AS span_status,s.error_code,
      s.output->>'pages_written' AS pages_written,s.output->>'pages_dropped' AS pages_dropped,
      s.output->>'pages_total' AS pages_total,s.output->>'failed_slug_writes' AS failed_slug_writes,
      s.output->>'classify_batches' AS classify_batches,
      s.output->>'read_coverage_version' AS read_coverage_version,
      (s.output->>'source_read_complete')::boolean AS source_read_complete,
      (s.output->>'source_runes')::bigint AS source_runes,
      (s.output->>'source_windows')::int AS source_windows,
      coalesce((SELECT jsonb_object_agg(c.item_key,c.payload::jsonb) FROM wiki_ingest_checkpoints c
        WHERE c.tenant_id=r.tenant_id AND c.knowledge_base_id=r.knowledge_base_id AND c.knowledge_id=r.knowledge_id
          AND c.stage='read_progress' AND c.updated_at>=r.queued_at AND c.updated_at<coalesce(r.next_queued_at,'infinity')
          AND c.fingerprint=(SELECT latest.fingerprint FROM wiki_ingest_checkpoints latest
            WHERE latest.tenant_id=r.tenant_id AND latest.knowledge_base_id=r.knowledge_base_id AND latest.knowledge_id=r.knowledge_id
              AND latest.stage='read_progress' AND latest.updated_at>=r.queued_at AND latest.updated_at<coalesce(r.next_queued_at,'infinity')
            ORDER BY latest.updated_at DESC,latest.fingerprint LIMIT 1)), '{}'::jsonb) AS read_progress,
      coalesce((SELECT jsonb_object_agg(c.item_key,c.payload::jsonb) FROM wiki_ingest_checkpoints c
        WHERE c.tenant_id=r.tenant_id AND c.knowledge_base_id=r.knowledge_base_id AND c.knowledge_id=r.knowledge_id
          AND c.stage='page_progress' AND c.updated_at>=r.queued_at AND c.updated_at<coalesce(r.next_queued_at,'infinity')
          AND c.fingerprint=(SELECT latest.fingerprint FROM wiki_ingest_checkpoints latest
            WHERE latest.tenant_id=r.tenant_id AND latest.knowledge_base_id=r.knowledge_base_id AND latest.knowledge_id=r.knowledge_id
              AND latest.stage='page_progress' AND latest.updated_at>=r.queued_at AND latest.updated_at<coalesce(r.next_queued_at,'infinity')
            ORDER BY latest.updated_at DESC,latest.fingerprint LIMIT 1)), '{}'::jsonb) AS page_progress,
      (SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=r.tenant_id AND p.knowledge_base_id=r.knowledge_base_id AND p.deleted_at IS NULL AND ${hasSource('p', 'r.knowledge_id')}) AS current_pages,
      (SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=r.tenant_id AND p.knowledge_base_id=r.knowledge_base_id AND p.deleted_at IS NULL AND p.page_type='summary' AND p.status='published' AND length(trim(p.content))>0 AND ${hasSource('p', 'r.knowledge_id')}) AS published_summaries,
      jsonb_array_length(${jsonArray("s.output->'pages_written_preview'")}) AS expected_pages,
      (SELECT count(*) FROM jsonb_array_elements(${jsonArray("s.output->'pages_written_preview'")}) expected
        WHERE NOT EXISTS (SELECT 1 FROM wiki_pages p WHERE p.tenant_id=r.tenant_id AND p.knowledge_base_id=r.knowledge_base_id
          AND p.slug=expected->>'slug' AND p.deleted_at IS NULL AND p.status='published' AND length(trim(p.content))>0 AND ${hasSource('p', 'r.knowledge_id')})) AS missing_expected_pages,
      (SELECT count(*) FROM knowledge_processing_spans child WHERE child.knowledge_id=r.knowledge_id AND child.started_at>=r.queued_at AND child.started_at<coalesce(r.next_queued_at,'infinity')
        AND child.name LIKE 'postprocess.wiki.page[%'
        AND (child.status='failed' OR child.output->>'addition_failed'='true')) AS failed_page_spans,
      (SELECT count(*) FROM task_dead_letters d WHERE d.tenant_id=r.tenant_id AND d.task_type='wiki:ingest' AND d.scope_id=r.knowledge_base_id AND d.related_id=r.knowledge_id AND d.id>r.dead_letter_highwater AND d.failed_at<coalesce(r.next_queued_at,'infinity')) AS new_failures,
      (SELECT count(*) FROM task_pending_ops f WHERE f.tenant_id=r.tenant_id AND f.scope_id=r.knowledge_base_id AND f.task_type='wiki:finalize'
        AND (coalesce(r.deployed_fix_review->>'budget_version','')<>'1' OR (f.payload->>'recovery_dead_letter_id'=r.dead_letter_id::text AND f.payload->>'recovery_attempt'=r.attempt_number::text AND f.payload->>'recovery_pending_op_id'=r.pending_op_id::text))) AS pending_finalize,
      o.id IS NOT NULL AS pending
    FROM (SELECT ledger.*,lead(queued_at) OVER (PARTITION BY tenant_id,dead_letter_id ORDER BY queued_at) AS next_queued_at FROM ${table} ledger) r
    LEFT JOIN task_pending_ops o ON o.id=r.pending_op_id AND o.tenant_id=r.tenant_id
    LEFT JOIN LATERAL (SELECT status,error_code,output FROM knowledge_processing_spans s
      WHERE s.knowledge_id=r.knowledge_id AND s.name='postprocess.wiki' AND s.started_at>=r.queued_at AND s.started_at<coalesce(r.next_queued_at,'infinity')
      ORDER BY s.started_at DESC,s.id DESC LIMIT 1) s ON true
    WHERE r.tenant_id=${tenant} ORDER BY r.dead_letter_id,r.queued_at
  ) x`).map(row => ({ ...row, full_read_status: describeReadCoverage(row), recovery_status: durableRecoveryStatus(row) || (row.pending ? (row.claimed_at ? 'running' : 'queued')
    : row.new_failures > 0 || row.span_status === 'failed' ? 'failed'
    : row.span_status === 'skipped' ? 'skipped'
    : Number(row.pages_dropped) > 0 || Number(row.failed_slug_writes) > 0 || row.missing_expected_pages > 0 || row.failed_page_spans > 0 ? 'partial'
    : row.span_status === 'done' && row.pages_dropped !== null && Number(row.pages_dropped) === 0
      && row.current_pages > 0 && row.published_summaries > 0 && row.expected_pages > 0
      && (row.deployed_fix_review?.budget_version !== 1 || (row.recovery_budget?.status === 'completed' && describeReadCoverage(row) === 'source_processed'))
      && row.expected_pages === Number(row.pages_written) && row.expected_pages === Number(row.pages_total) ? 'wiki_recovered'
    : 'unverified') }))
  return annotateClosed(rows, loadClosures(directory, tenant), tenant)
}

function verifyPublishedFix(review) {
  const manifest = JSON.parse(fs.readFileSync(path.join(root, 'public-release.json'), 'utf8'))
  const runtime = JSON.parse(docker(['inspect', 'yawan-native-acceptance', '--format', '{"running":{{json .State.Running}},"image_id":{{json .Image}},"release":{{json (index .Config.Labels "yawan.release")}}}']))
  const validated = validatePublishedRecoveryFix(review, manifest, runtime)
  if (review.max_attempt === 6) {
    const sourceHash = candidateSourceTreeHash(path.resolve(__dirname, '..'))
    if (review.candidate_diagnostic.kind === replayPins.kind) {
      verifyReplayInputFiles(root)
      const evidence = fs.readFileSync(path.join(root, 'acceptance/wiki-layout-identity-replay-private-20260921/evidence.json'))
      validateSixthReplayEvidence(validated, evidence, sourceHash)
    } else {
      const evidence = fs.readFileSync(path.join(root, 'acceptance/wiki-repair-window-diagnostic-private-20260920/evidence.json'))
      validateSixthDiagnosticEvidence(validated, evidence, sourceHash)
    }
  }
  if (review.max_attempt === 7) {
    const evidenceDirectory = path.join(root, seventhPins.evidence_directory)
    const build = verifySeventhBuildFiles(root)
    const binaryHash = docker(['exec', 'yawan-native-acceptance', 'sha256sum', '/app/WeKnora']).trim()
    assert.match(binaryHash, /^[a-f0-9]{64}\s+\/app\/WeKnora$/, 'Running API binary hash unavailable')
    validateSeventhCorrectionEvidence(validated, fs.readFileSync(path.join(evidenceDirectory, 'evidence.json')),
      build.sourceTreeHash, binaryHash.slice(0, 64), fs.readFileSync(path.join(evidenceDirectory, 'regression-tests.jsonl')),
      fs.readFileSync(path.join(evidenceDirectory, 'semantic-review.json')), build.buildManifestBytes, build.frozenBinarySHA)
  }
  return validated
}

function trigger(rows) {
  if (!rows.length) return { triggered_scopes: 0 }
  assert.ok(rows.every(row => [1, 2].includes(attemptOf(row)) || supportedReviewedAttempt(attemptOf(row))), 'Recovery trigger cannot exceed the attempt limit')
  assert.ok(rows.every(row => row.deployed_fix_review?.budget_version === 1), 'Legacy recovery receipts cannot execute without a durable reviewed budget')
  const finalRows = rows
  if (finalRows.length) {
    const sources = inventory(rows.map(row => row.dead_letter_id))
    const history = status()
    for (const row of finalRows) {
      const review = verifyPublishedFix(row.deployed_fix_review)
      validateFinalRecoveryTrigger(row, sources.find(source => source.id === row.dead_letter_id), history, review)
    }
  }
  const scopes = [...new Set(rows.map(row => row.knowledge_base_id))].map(id => ({tenant_id: tenant, knowledge_base_id: id, language: 'zh-CN'}))
  docker(['cp', path.join(root, 'acceptance/wiki-backfill-trigger'), 'yawan-native-acceptance:/tmp/wiki-backfill-trigger'])
  return JSON.parse(docker(['exec', '-i', 'yawan-native-acceptance', '/tmp/wiki-backfill-trigger'], JSON.stringify(scopes)))
}

function main() {
  const before = inventory()
  const prior = status()
  let triggered
  if (apply) {
    let review
    if (reviewPath) {
      review = verifyPublishedFix(JSON.parse(fs.readFileSync(reviewPath, 'utf8')))
    }
    const selected = ids.map(id => before.find(row => row.id === id))
    assert.ok(selected.every(row => row?.eligible && row.chunks > 0), 'Every selected original record must remain eligible')
    validateDurableRecoveryReview(selected[0], review, attempt)
    const fresh = planRecovery(selected, prior, attempt, retryReason, review)
    if (fresh.length) {
      fs.mkdirSync(directory, { recursive: true, mode: 0o700 })
      const run = Date.now()
      const backup = queryJSON(`SELECT coalesce(json_agg(row_to_json(d)), '[]') FROM task_dead_letters d WHERE d.tenant_id=${tenant} AND d.task_type='wiki:ingest' AND d.id IN (${fresh.map(row => row.id).join(',')})`)
      writePrivate(`original-records-${run}.json`, backup)
      writePrivate(`before-${run}.json`, { tenant_id: tenant, attempt_number: attempt, retry_reason: retryReason, deployed_fix_review: review, selected: fresh, all_originals: before, prior })
      // The receipt and pending operation commit together. Re-running after a lost
      // response cannot enqueue the original dead letter a second time.
      sql(`BEGIN;
        SELECT pg_advisory_xact_lock(${tenant}, 20260919);
        CREATE TABLE IF NOT EXISTS ${table} (
          tenant_id bigint NOT NULL,dead_letter_id bigint NOT NULL,knowledge_id varchar NOT NULL,
          knowledge_base_id varchar NOT NULL,model_id varchar NOT NULL,pending_op_id bigint NOT NULL,
          queued_at timestamptz NOT NULL,baseline_pages bigint NOT NULL,baseline_chunks bigint NOT NULL,
          chunk_chars bigint NOT NULL,dead_letter_highwater bigint NOT NULL,
          PRIMARY KEY(tenant_id,dead_letter_id));
        ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS attempt_number integer NOT NULL DEFAULT 1;
        ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS retry_reason text NOT NULL DEFAULT '';
        ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS deployed_fix_review jsonb;
        ALTER TABLE ${table} DROP CONSTRAINT IF EXISTS ${table}_pkey;
        ALTER TABLE ${table} ADD PRIMARY KEY(tenant_id,dead_letter_id,attempt_number);
        LOCK TABLE task_pending_ops IN SHARE ROW EXCLUSIVE MODE;
        LOCK TABLE knowledges, knowledge_bases, chunks, models IN SHARE MODE;
        ${attempt === 7 ? 'LOCK TABLE wiki_recovery_budgets, task_dead_letters, knowledge_processing_spans, wiki_ingest_checkpoints IN SHARE MODE;' : ''}
        WITH source AS (
          SELECT d.*,nextval(pg_get_serial_sequence('task_pending_ops','id')) AS recovery_op_id,
            k.wiki_config->>'synthesis_model_id' AS model_id
          FROM task_dead_letters d JOIN knowledges n ON n.id=d.related_id AND n.tenant_id=d.tenant_id AND n.knowledge_base_id=d.scope_id
          JOIN knowledge_bases k ON k.id=d.scope_id AND k.tenant_id=d.tenant_id
          WHERE d.tenant_id=${tenant} AND d.task_type='wiki:ingest' AND d.scope='knowledge_base'
            AND d.id IN (${fresh.map(row => row.id).join(',')}) AND d.payload->>'op'='ingest' AND d.payload->>'knowledge_id'=d.related_id
            AND n.deleted_at IS NULL AND k.deleted_at IS NULL AND n.parse_status='completed' AND n.pending_subtasks_count=0
            AND k.indexing_strategy->>'wiki_enabled'='true'
            ${review ? `AND d.related_id=${literal(review.knowledge_id)} AND d.scope_id=${literal(review.knowledge_base_id)} AND k.wiki_config->>'synthesis_model_id'=${literal(review.model_id)}` : ''}
            AND ${fingerprints.source}=${literal(review.source_fingerprint)} AND ${fingerprints.model}=${literal(review.model_fingerprint)}
              AND k.embedding_model_id=${literal(review.embedding_model_id)} AND ${literal(review.deadline_at)}::timestamptz>now()
              ${attempt === 5 ? `AND EXISTS (SELECT 1 FROM ${table} p WHERE p.tenant_id=d.tenant_id AND p.dead_letter_id=d.id AND p.attempt_number=4 AND p.pending_op_id=4143)` : ''}
              ${attempt === 6 ? `AND d.id=10 AND d.tenant_id=10059
                AND ${review.max_provider_requests} BETWEEN 1 AND 1000
                AND ${literal(review.budget_started_at)}::timestamptz<=now()
                AND ${literal(review.deadline_at)}::timestamptz>${literal(review.budget_started_at)}::timestamptz
                AND ${literal(review.deadline_at)}::timestamptz<=${literal(review.budget_started_at)}::timestamptz+interval '120 minutes'
                AND EXISTS (SELECT 1 FROM ${table} p JOIN wiki_recovery_budgets b
                  ON b.tenant_id=p.tenant_id AND b.dead_letter_id=p.dead_letter_id AND b.attempt_number=p.attempt_number AND b.pending_op_id=p.pending_op_id
                  WHERE p.tenant_id=d.tenant_id AND p.dead_letter_id=d.id AND p.attempt_number=5 AND p.pending_op_id=4144
                    AND p.knowledge_id=d.related_id AND p.knowledge_base_id=d.scope_id AND p.model_id=k.wiki_config->>'synthesis_model_id'
                    AND b.status='stopped_budget' AND b.knowledge_id=d.related_id AND b.knowledge_base_id=d.scope_id
                    AND b.model_id=k.wiki_config->>'synthesis_model_id' AND b.embedding_model_id=k.embedding_model_id
                    AND b.source_fingerprint=${literal(review.source_fingerprint)} AND b.model_fingerprint=${literal(review.model_fingerprint)}
                    AND (EXISTS (SELECT 1 FROM task_dead_letters failure WHERE failure.tenant_id=p.tenant_id
                      AND failure.task_type='wiki:ingest' AND failure.scope_id=p.knowledge_base_id
                      AND failure.related_id=p.knowledge_id AND failure.id>p.dead_letter_highwater)
                      OR (b.stop_reason='guard_rejected' AND b.last_phase='wiki_candidate_slug' AND b.reserved_provider_requests=4
                        AND jsonb_typeof(b.paused_payload::jsonb)='array' AND jsonb_array_length(b.paused_payload::jsonb)=1
                        AND b.paused_payload::jsonb @> ${literal(JSON.stringify([sixthGuardPausedOperation]))}::jsonb
                        AND (SELECT span.status='failed' AND span.error_code='EXTRACT_FAILED' FROM knowledge_processing_spans span
                          WHERE span.knowledge_id=p.knowledge_id AND span.name='postprocess.wiki' AND span.started_at>=p.queued_at
                          ORDER BY span.started_at DESC,span.id DESC LIMIT 1))))
                AND NOT EXISTS (SELECT 1 FROM ${table} later WHERE later.tenant_id=d.tenant_id AND later.dead_letter_id=d.id AND later.attempt_number>5)` : ''}
              ${attempt === 7 ? seventhRecoveryEnqueueSQL(review, table, literal) : ''}
              AND NOT EXISTS (SELECT 1 FROM task_pending_ops f WHERE f.tenant_id=d.tenant_id AND f.scope_id=d.scope_id AND f.task_type='wiki:finalize')
            AND NOT EXISTS (SELECT 1 FROM ${table} r WHERE r.tenant_id=d.tenant_id AND r.dead_letter_id=d.id AND r.attempt_number=${attempt})
            AND (${attempt}=1 OR EXISTS (SELECT 1 FROM ${table} r WHERE r.tenant_id=d.tenant_id AND r.dead_letter_id=d.id AND r.attempt_number=${attempt - 1}))
            AND NOT EXISTS (SELECT 1 FROM ${table} r JOIN task_pending_ops o ON o.id=r.pending_op_id AND o.tenant_id=r.tenant_id WHERE r.tenant_id=${tenant})
            AND NOT EXISTS (SELECT 1 FROM task_pending_ops o WHERE o.tenant_id=d.tenant_id AND o.scope_id=d.scope_id AND o.task_type='wiki:ingest' AND o.dedup_key=d.related_id)
        ), budget AS (
          INSERT INTO wiki_recovery_budgets(tenant_id,dead_letter_id,attempt_number,pending_op_id,knowledge_base_id,knowledge_id,model_id,embedding_model_id,
            source_fingerprint,model_fingerprint,reviewed_release,reviewed_api_image,max_provider_requests,reserved_provider_requests,deadline_at,status,paused_payload,verified_progress)
          SELECT tenant_id,id,${attempt},recovery_op_id,scope_id,related_id,model_id,${literal(review.embedding_model_id)},
            ${literal(review.source_fingerprint)},${literal(review.model_fingerprint)},${literal(review.fix_release)},${literal(review.api_image_id)},
            ${review.max_provider_requests},0,${literal(review.deadline_at)}::timestamptz,'active','[]','{}' FROM source RETURNING pending_op_id
        ), inserted AS (
          INSERT INTO task_pending_ops(id,tenant_id,task_type,scope,scope_id,op,dedup_key,payload,fail_count,enqueued_at)
          SELECT recovery_op_id,tenant_id,task_type,scope,scope_id,'ingest',related_id,
            payload||jsonb_build_object('recovery_dead_letter_id',id,'recovery_attempt',${attempt},'recovery_pending_op_id',recovery_op_id,'recovery_budget_version',1),${recoveryInitialFailCount},now() FROM source
            WHERE EXISTS (SELECT 1 FROM budget b WHERE b.pending_op_id=source.recovery_op_id) RETURNING id
        ) INSERT INTO ${table} (tenant_id,dead_letter_id,knowledge_id,knowledge_base_id,model_id,pending_op_id,queued_at,baseline_pages,baseline_chunks,chunk_chars,dead_letter_highwater,attempt_number,retry_reason,deployed_fix_review)
          SELECT d.tenant_id,d.id,d.related_id,d.scope_id,d.model_id,d.recovery_op_id,now(),
            (SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=d.tenant_id AND p.knowledge_base_id=d.scope_id AND p.deleted_at IS NULL AND ${hasSource('p', 'd.related_id')}),
            (SELECT count(*) FROM chunks c WHERE c.tenant_id=d.tenant_id AND c.knowledge_id=d.related_id AND c.deleted_at IS NULL),
            (SELECT coalesce(sum(length(c.content)),0) FROM chunks c WHERE c.tenant_id=d.tenant_id AND c.knowledge_id=d.related_id AND c.deleted_at IS NULL),
            (SELECT coalesce(max(id),0) FROM task_dead_letters),${attempt},${literal(retryReason)},${review ? literal(JSON.stringify(review)) + '::jsonb' : 'NULL'}
          FROM source d JOIN inserted i ON i.id=d.recovery_op_id;
        COMMIT;`, false)
      assert.ok(status().some(row => row.dead_letter_id === fresh[0].id && attemptOf(row) === attempt), 'Recovery was not enqueued because state changed; inspect status before retrying')
    }
    triggered = trigger(status().filter(row => ids.includes(row.dead_letter_id) && attemptOf(row) === attempt && row.pending))
  } else if (triggerOnly) {
    triggered = trigger(latestAttempts(prior).filter(row => row.pending && (!ids.length || ids.includes(row.dead_letter_id))))
  }
  const history = status()
  const recovered = latestAttempts(history)
  const report = { checked_at: new Date().toISOString(), tenant_id: tenant, apply, triggered,
    original_dead_letters: before.length, attempted: recovered.length,
    remaining_not_attempted: before.filter(row => !recovered.some(recovery => recovery.dead_letter_id === row.id)).length,
    scope: 'Wiki recovery only. Original records and checkpoints are preserved. One document per batch. Further attempts require their exact reviewed published fix. Only original dead letter 10 has exact fifth/sixth/seventh-attempt exceptions. The seventh requires separately reviewed fresh source identity correction tests, matching frozen source and binary, a newer published fix, at most 1000 actual provider sends and a fixed 120-minute maximum. No eighth attempt or automatic attempt increase. Terminal budget states stop scheduling. Historical page recovery does not prove full-source reading.',
    candidates: before, recoveries: recovered, attempt_history: history }
  fs.mkdirSync(directory, { recursive: true, mode: 0o700 })
  const reportFile = path.join(directory, `status-${Date.now()}.json`)
  writePrivate(path.basename(reportFile), report)
  const { candidates, attempt_history, ...summary } = report
  console.log(JSON.stringify({ ...summary, report_directory: directory, report_file: reportFile }, null, 2))
}

try { main() } catch (error) {
  // Docker/psql exceptions may contain full SQL or sensitive runtime output.
  console.error(error instanceof assert.AssertionError ? error.message : 'Wiki recovery stopped; inspect private records and container health. No automatic re-enqueue was attempted.')
  process.exitCode = 1
}