const fs = require('node:fs') const path = require('node:path') const assert = require('node:assert/strict') const { execFileSync, spawn } = require('node:child_process') const { recoverySchedulingGate, remainingRecoveryCandidates } = require('./wiki_recovery_scheduler_policy.cjs') const args = process.argv.slice(2) const options = { 'max-minutes': 180, 'max-documents': 32, 'batch-size': 1 } let reviewDirectory for (const arg of args) { if (arg === '--apply' || arg === '--detach') continue if (arg.startsWith('--review-directory=')) { reviewDirectory = path.resolve(arg.slice('--review-directory='.length)) continue } const match = /^--(max-minutes|max-documents|batch-size)=(\d+)$/.exec(arg) assert.ok(match, 'Unknown recovery runner option') options[match[1]] = Number(match[2]) } assert.ok(args.includes('--apply'), 'Explicit --apply required') assert.ok(options['max-minutes'] >= 1 && options['max-minutes'] <= 180, 'Time limit must be 1..180 minutes') assert.ok(options['max-documents'] >= 1 && options['max-documents'] <= 32, 'Document limit must be 1..32') assert.equal(options['batch-size'], 1, 'Recovery is limited to one document per batch') const directory = path.resolve(__dirname, '../../acceptance/wiki-deadletter-recovery-20260919') const recoveryScript = path.join(__dirname, 'recover_wiki_deadletters.cjs') const stateFile = path.join(directory, 'runner-state.json') const lockFile = path.join(directory, 'runner.lock') fs.mkdirSync(directory, { recursive: true, mode: 0o700 }) if (args.includes('--detach')) { const log = fs.openSync(path.join(directory, 'runner-console.log'), 'a', 0o600) const child = spawn(process.execPath, [__filename, ...args.filter(arg => arg !== '--detach')], { detached: true, stdio: ['ignore', log, log], cwd: path.dirname(recoveryScript), }) child.unref() fs.closeSync(log) console.log(JSON.stringify({ launched_pid: child.pid, state_file: stateFile })) } else { main().catch(() => { console.error('Recovery runner stopped. Inspect runner-state.json and private status reports.') process.exitCode = 1 }) } function summarize(report) { const counts = {} const readCoverage = {} for (const row of report.recoveries) counts[row.recovery_status] = (counts[row.recovery_status] || 0) + 1 for (const row of report.recoveries) readCoverage[row.full_read_status || 'not_verified'] = (readCoverage[row.full_read_status || 'not_verified'] || 0) + 1 const { historical_full_read_unverified_ids, source_reading_complete } = recoverySchedulingGate(report) return { attempted: report.attempted, remaining_not_attempted: report.remaining_not_attempted, counts, full_read_counts: readCoverage, historical_full_read_unverified_ids, source_reading_complete, current_reading: report.recoveries.filter(row => row.pending).map(row => ({ dead_letter_id: row.dead_letter_id, read_progress: row.read_progress || {}, page_progress: row.page_progress || {} })), page_associations: report.recoveries.reduce((sum, row) => sum + row.current_pages, 0), new_failure_records: report.recoveries.reduce((sum, row) => sum + row.new_failures, 0), report_file: report.report_file } } async function main() { // An existing lock requires operator inspection, even if its process died. // Never start a second scheduler on the basis of a possibly reused PID. const lock = fs.openSync(lockFile, 'wx', 0o600) fs.writeFileSync(lock, JSON.stringify({ pid: process.pid, started_at: new Date().toISOString() })) fs.closeSync(lock) const started = Date.now() let queued = 0 let lastReport let stopping = false process.on('SIGTERM', () => { stopping = true }) process.on('SIGINT', () => { stopping = true }) const save = (state, extra = {}) => { const value = { state, pid: process.pid, started_at: new Date(started).toISOString(), updated_at: new Date().toISOString(), options, newly_enqueued: queued, ...(lastReport ? summarize(lastReport) : {}), ...extra } const temporary = `${stateFile}.${process.pid}.tmp` fs.writeFileSync(temporary, JSON.stringify(value, null, 2) + '\n', { mode: 0o600 }) fs.renameSync(temporary, stateFile) fs.appendFileSync(path.join(directory, 'runner-events.jsonl'), JSON.stringify(value) + '\n', { mode: 0o600 }) console.log(JSON.stringify(value)) } const inspect = (commandArgs = []) => { const consoleReport = JSON.parse(execFileSync(process.execPath, [recoveryScript, ...commandArgs], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 120000, maxBuffer: 4 * 1024 * 1024, })) assert.equal(path.dirname(consoleReport.report_file), directory, 'Unexpected report directory') lastReport = { ...JSON.parse(fs.readFileSync(consoleReport.report_file, 'utf8')), report_file: consoleReport.report_file } return lastReport } try { save('starting') while (true) { const report = inspect() const { state, ...coverage } = recoverySchedulingGate(report) if (state === 'stopped_for_review') { save(state, coverage) break } if (stopping || Date.now() - started >= options['max-minutes'] * 60000) { save(stopping ? 'stopped_by_signal' : 'time_limit_reached') break } if (state === 'waiting_for_current_batch') { save(state, coverage) await new Promise(resolve => setTimeout(resolve, 45000)) continue } if (report.remaining_not_attempted === 0) { save(state, coverage) break } if (queued >= options['max-documents']) { save('document_limit_reached') break } const candidates = remainingRecoveryCandidates(report) const ids = candidates.slice(0, Math.min(options['batch-size'], options['max-documents'] - queued)).map(row => row.id) assert.ok(ids.length > 0, 'No eligible remaining originals') const reviewFile = reviewDirectory && path.join(reviewDirectory, `dead-letter-${ids[0]}-attempt-1.json`) if (!reviewFile || !fs.existsSync(reviewFile)) { save('stopped_for_review', { batch_ids: ids, reason: 'A per-document reviewed request limit and fixed deadline are required before enqueueing.' }) break } save('enqueueing_batch', { batch_ids: ids }) inspect(['--apply', `--ids=${ids.join(',')}`, `--review-file=${reviewFile}`]) queued += ids.length save('batch_enqueued', { batch_ids: ids }) } } catch (error) { save('stopped_on_error', { reason: error instanceof assert.AssertionError ? error.message : 'Recovery command failed; no automatic retry was scheduled.' }) throw error } finally { fs.unlinkSync(lockFile) } }