xinmi/scripts/run_wiki_deadletter_recovery.cjs

const fs = require('node:fs')
const path = require('node:path')
const assert = require('node:assert/strict')
const { execFileSync, spawn } = require('node:child_process')
const { recoverySchedulingGate, remainingRecoveryCandidates } = require('./wiki_recovery_scheduler_policy.cjs')

const args = process.argv.slice(2)
const options = { 'max-minutes': 180, 'max-documents': 32, 'batch-size': 1 }
let reviewDirectory
for (const arg of args) {
  if (arg === '--apply' || arg === '--detach') continue
  if (arg.startsWith('--review-directory=')) {
    reviewDirectory = path.resolve(arg.slice('--review-directory='.length))
    continue
  }
  const match = /^--(max-minutes|max-documents|batch-size)=(\d+)$/.exec(arg)
  assert.ok(match, 'Unknown recovery runner option')
  options[match[1]] = Number(match[2])
}
assert.ok(args.includes('--apply'), 'Explicit --apply required')
assert.ok(options['max-minutes'] >= 1 && options['max-minutes'] <= 180, 'Time limit must be 1..180 minutes')
assert.ok(options['max-documents'] >= 1 && options['max-documents'] <= 32, 'Document limit must be 1..32')
assert.equal(options['batch-size'], 1, 'Recovery is limited to one document per batch')

const directory = path.resolve(__dirname, '../../acceptance/wiki-deadletter-recovery-20260919')
const recoveryScript = path.join(__dirname, 'recover_wiki_deadletters.cjs')
const stateFile = path.join(directory, 'runner-state.json')
const lockFile = path.join(directory, 'runner.lock')
fs.mkdirSync(directory, { recursive: true, mode: 0o700 })

if (args.includes('--detach')) {
  const log = fs.openSync(path.join(directory, 'runner-console.log'), 'a', 0o600)
  const child = spawn(process.execPath, [__filename, ...args.filter(arg => arg !== '--detach')], {
    detached: true, stdio: ['ignore', log, log], cwd: path.dirname(recoveryScript),
  })
  child.unref()
  fs.closeSync(log)
  console.log(JSON.stringify({ launched_pid: child.pid, state_file: stateFile }))
} else {
  main().catch(() => {
    console.error('Recovery runner stopped. Inspect runner-state.json and private status reports.')
    process.exitCode = 1
  })
}

function summarize(report) {
  const counts = {}
  const readCoverage = {}
  for (const row of report.recoveries) counts[row.recovery_status] = (counts[row.recovery_status] || 0) + 1
  for (const row of report.recoveries) readCoverage[row.full_read_status || 'not_verified'] = (readCoverage[row.full_read_status || 'not_verified'] || 0) + 1
  const { historical_full_read_unverified_ids, source_reading_complete } = recoverySchedulingGate(report)
  return { attempted: report.attempted, remaining_not_attempted: report.remaining_not_attempted, counts,
    full_read_counts: readCoverage,
    historical_full_read_unverified_ids, source_reading_complete,
    current_reading: report.recoveries.filter(row => row.pending).map(row => ({ dead_letter_id: row.dead_letter_id, read_progress: row.read_progress || {}, page_progress: row.page_progress || {} })),
    page_associations: report.recoveries.reduce((sum, row) => sum + row.current_pages, 0),
    new_failure_records: report.recoveries.reduce((sum, row) => sum + row.new_failures, 0),
    report_file: report.report_file }
}

async function main() {
  // An existing lock requires operator inspection, even if its process died.
  // Never start a second scheduler on the basis of a possibly reused PID.
  const lock = fs.openSync(lockFile, 'wx', 0o600)
  fs.writeFileSync(lock, JSON.stringify({ pid: process.pid, started_at: new Date().toISOString() }))
  fs.closeSync(lock)
  const started = Date.now()
  let queued = 0
  let lastReport
  let stopping = false
  process.on('SIGTERM', () => { stopping = true })
  process.on('SIGINT', () => { stopping = true })
  const save = (state, extra = {}) => {
    const value = { state, pid: process.pid, started_at: new Date(started).toISOString(),
      updated_at: new Date().toISOString(), options, newly_enqueued: queued,
      ...(lastReport ? summarize(lastReport) : {}), ...extra }
    const temporary = `${stateFile}.${process.pid}.tmp`
    fs.writeFileSync(temporary, JSON.stringify(value, null, 2) + '\n', { mode: 0o600 })
    fs.renameSync(temporary, stateFile)
    fs.appendFileSync(path.join(directory, 'runner-events.jsonl'), JSON.stringify(value) + '\n', { mode: 0o600 })
    console.log(JSON.stringify(value))
  }
  const inspect = (commandArgs = []) => {
    const consoleReport = JSON.parse(execFileSync(process.execPath, [recoveryScript, ...commandArgs], {
      encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 120000, maxBuffer: 4 * 1024 * 1024,
    }))
    assert.equal(path.dirname(consoleReport.report_file), directory, 'Unexpected report directory')
    lastReport = { ...JSON.parse(fs.readFileSync(consoleReport.report_file, 'utf8')), report_file: consoleReport.report_file }
    return lastReport
  }
  try {
    save('starting')
    while (true) {
      const report = inspect()
      const { state, ...coverage } = recoverySchedulingGate(report)
      if (state === 'stopped_for_review') {
        save(state, coverage)
        break
      }
      if (stopping || Date.now() - started >= options['max-minutes'] * 60000) {
        save(stopping ? 'stopped_by_signal' : 'time_limit_reached')
        break
      }
      if (state === 'waiting_for_current_batch') {
        save(state, coverage)
        await new Promise(resolve => setTimeout(resolve, 45000))
        continue
      }
      if (report.remaining_not_attempted === 0) {
        save(state, coverage)
        break
      }
      if (queued >= options['max-documents']) {
        save('document_limit_reached')
        break
      }
      const candidates = remainingRecoveryCandidates(report)
      const ids = candidates.slice(0, Math.min(options['batch-size'], options['max-documents'] - queued)).map(row => row.id)
      assert.ok(ids.length > 0, 'No eligible remaining originals')
      const reviewFile = reviewDirectory && path.join(reviewDirectory, `dead-letter-${ids[0]}-attempt-1.json`)
      if (!reviewFile || !fs.existsSync(reviewFile)) {
        save('stopped_for_review', { batch_ids: ids, reason: 'A per-document reviewed request limit and fixed deadline are required before enqueueing.' })
        break
      }
      save('enqueueing_batch', { batch_ids: ids })
      inspect(['--apply', `--ids=${ids.join(',')}`, `--review-file=${reviewFile}`])
      queued += ids.length
      save('batch_enqueued', { batch_ids: ids })
    }
  } catch (error) {
    save('stopped_on_error', { reason: error instanceof assert.AssertionError ? error.message : 'Recovery command failed; no automatic retry was scheduled.' })
    throw error
  } finally {
    fs.unlinkSync(lockFile)
  }
}