331 lines
26 KiB
JavaScript
331 lines
26 KiB
JavaScript
|
const { test } = require('node:test')
|
||
|
|
const assert = require('node:assert/strict')
|
||
|
|
const crypto = require('node:crypto')
|
||
|
|
const fs = require('node:fs')
|
||
|
|
const path = require('node:path')
|
||
|
|
const vm = require('node:vm')
|
||
|
|
const cp = require('node:child_process')
|
||
|
|
const policy = require('./wiki_recovery_policy.cjs')
|
||
|
|
const sixth = require('./wiki_sixth_recovery_review.cjs')
|
||
|
|
const sha = value => crypto.createHash('sha256').update(value).digest('hex')
|
||
|
|
const reason = 'Reviewed successful exact-window diagnostic and published bounded candidate correction'
|
||
|
|
|
||
|
|
function fixture() {
|
||
|
|
const now = Date.now()
|
||
|
|
const scope = sixth.sixthRecoveryScope, pins = sixth.diagnosticPins
|
||
|
|
const source = { ...scope, id: 10, eligible: true, pending: 0, chunks: 624 }
|
||
|
|
const evidence = { run_id: '00000000-0000-4000-8000-000000000003', status: 'finished', candidate_source_only: true,
|
||
|
|
source_tree_hash: 'a'.repeat(64), protected_state_unchanged: true, previous_diagnostics_unchanged: true,
|
||
|
|
started_at: '2026-09-20T16:20:00Z', actual_send_reservations: 2,
|
||
|
|
pins: { tenant: 10059, deadLetter: 10, priorAttempt: 5, priorOperation: 4144, document: scope.knowledge_id,
|
||
|
|
kb: scope.knowledge_base_id, model: scope.model_id, embedding: scope.embedding_model_id,
|
||
|
|
sourceFingerprint: scope.source_fingerprint, modelFingerprint: scope.model_fingerprint,
|
||
|
|
release: pins.release, image: pins.image, maxSends: 21, durationMs: 900000 },
|
||
|
|
outcome: { status: 'exact_window_candidates_validated', source_hash: pins.source,
|
||
|
|
window: { start_rune: pins.start, end_rune: pins.end, sha256: pins.window }, candidate_count: 193,
|
||
|
|
actual_send_reservations: 2, leaves: [{ start_rune: pins.start, end_rune: pins.end, sha256: pins.window }] } }
|
||
|
|
const bytes = JSON.stringify(evidence)
|
||
|
|
const review = { ...scope, budget_version: 1, max_attempt: 6, previous_attempt: 5, previous_pending_op_id: 4144,
|
||
|
|
previous_error_code: 'EXTRACT_FAILED', fix_release: 'public-20260921-010000', api_image_id: `sha256:${'b'.repeat(64)}`,
|
||
|
|
published_at: '2026-09-20T17:00:00Z', public_gates_passed: true, max_provider_requests: 1000,
|
||
|
|
budget_started_at: new Date(now - 1000).toISOString(), deadline_at: new Date(now + 7199000).toISOString(),
|
||
|
|
candidate_diagnostic: { kind: 'candidate_repair_window_v3', run_id: evidence.run_id,
|
||
|
|
evidence_sha256: sha(bytes), source_tree_hash: evidence.source_tree_hash, semantic_reviewed: true } }
|
||
|
|
const previous = { ...scope, attempt_number: 5, pending_op_id: 4144, recovery_status: 'stopped_budget', new_failures: 1,
|
||
|
|
error_code: 'EXTRACT_FAILED', queued_at: '2026-09-20T08:58:56Z', pending: false, pending_finalize: 0,
|
||
|
|
recovery_budget: { ...scope, attempt_number: 5, pending_op_id: 4144, status: 'stopped_budget' } }
|
||
|
|
const pilot = { ...scope, dead_letter_id: 31, pending: false, recovery_status: 'wiki_recovered', span_status: 'done',
|
||
|
|
pages_dropped: 0, failed_slug_writes: 0, missing_expected_pages: 0, failed_page_spans: 0, current_pages: 1,
|
||
|
|
published_summaries: 1, expected_pages: 1, pages_written: 1, pages_total: 1 }
|
||
|
|
const receipt = { ...scope, attempt_number: 6, pending_op_id: 6000, pending: true, pending_finalize: 0,
|
||
|
|
deployed_fix_review: review, retry_reason: reason, new_failures: 0,
|
||
|
|
recovery_budget: { ...scope, attempt_number: 6, pending_op_id: 6000, status: 'active',
|
||
|
|
max_provider_requests: 1000, reserved_provider_requests: 0, deadline_at: review.deadline_at,
|
||
|
|
reviewed_release: review.fix_release, reviewed_api_image: review.api_image_id } }
|
||
|
|
return { source, evidence, bytes, review, previous, pilot, receipt }
|
||
|
|
}
|
||
|
|
|
||
|
|
function replayFixture() {
|
||
|
|
const data = fixture()
|
||
|
|
data.evidence = { ...sixth.replayPins, run_id: '00000000-0000-4000-8000-000000000006',
|
||
|
|
source_tree_hash: 'a'.repeat(64), status: 'passed', model_calls: 0, production_writes: 0,
|
||
|
|
entity_count: 22, concept_count: 67, candidate_count: 89, rejected_count: 0,
|
||
|
|
source_preserved: true, original_receipt_preserved: true,
|
||
|
|
started_at: '2026-09-20T16:20:00Z', completed_at: '2026-09-20T16:21:00Z' }
|
||
|
|
data.bytes = JSON.stringify(data.evidence)
|
||
|
|
data.review.candidate_diagnostic = { kind: sixth.replayPins.kind, run_id: data.evidence.run_id,
|
||
|
|
evidence_sha256: sha(data.bytes), source_tree_hash: data.evidence.source_tree_hash, semantic_reviewed: true }
|
||
|
|
return data
|
||
|
|
}
|
||
|
|
|
||
|
|
function guardPausedFixture() {
|
||
|
|
const data = replayFixture()
|
||
|
|
Object.assign(data.previous, { new_failures: 0, span_status: 'failed' })
|
||
|
|
Object.assign(data.previous.recovery_budget, { stop_reason: 'guard_rejected', last_phase: 'wiki_candidate_slug',
|
||
|
|
reserved_provider_requests: 4, paused_payload: JSON.stringify([{ ...sixth.sixthGuardPausedOperation,
|
||
|
|
enqueued_at: data.previous.queued_at, claimed_at: '2026-09-20T08:58:59Z' }]) })
|
||
|
|
return data
|
||
|
|
}
|
||
|
|
|
||
|
|
test('reviewed guard pause permits zero new dead letters without changing the terminal receipt', () => {
|
||
|
|
const { source, previous, pilot, receipt, review } = guardPausedFixture()
|
||
|
|
const saved = JSON.stringify(previous)
|
||
|
|
assert.deepEqual(policy.planRecovery([source], [pilot, previous], 6, reason, review), [source])
|
||
|
|
assert.doesNotThrow(() => policy.validateFinalRecoveryTrigger(receipt, { ...source, pending: 1 }, [pilot, previous, receipt], review))
|
||
|
|
assert.equal(JSON.stringify(previous), saved)
|
||
|
|
})
|
||
|
|
|
||
|
|
test('zero-deadletter exception rejects missing or changed failed span and terminal guard evidence', () => {
|
||
|
|
const { source, previous, pilot, review } = guardPausedFixture()
|
||
|
|
for (const change of [{ span_status: undefined }, { span_status: 'done' }, { span_status: 'running' },
|
||
|
|
{ error_code: undefined }, { error_code: 'other' }, { pending: true }, { pending_finalize: 1 },
|
||
|
|
{ new_failures: -1 }, { new_failures: undefined }, { new_failures: '0' }]) {
|
||
|
|
assert.throws(() => policy.planRecovery([source], [pilot, { ...previous, ...change }], 6, reason, review))
|
||
|
|
}
|
||
|
|
for (const change of [{ status: 'active' }, { stop_reason: undefined }, { stop_reason: 'deadline' },
|
||
|
|
{ last_phase: undefined }, { last_phase: 'wiki_summary' }, { reserved_provider_requests: 3 },
|
||
|
|
{ paused_payload: undefined }, { paused_payload: 'invalid' }, { paused_payload: '{}' },
|
||
|
|
{ paused_payload: '[]' }, { paused_payload: '[null]' },
|
||
|
|
{ paused_payload: JSON.stringify([sixth.sixthGuardPausedOperation, sixth.sixthGuardPausedOperation]) }]) {
|
||
|
|
const changed = { ...previous, recovery_budget: { ...previous.recovery_budget, ...change } }
|
||
|
|
assert.throws(() => policy.planRecovery([source], [pilot, changed], 6, reason, review))
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
test('zero-deadletter exception checks every paused operation and nested recovery scope field', () => {
|
||
|
|
const { source, previous, pilot, review } = guardPausedFixture()
|
||
|
|
for (const key of Object.keys(sixth.sixthGuardPausedOperation)) {
|
||
|
|
if (key === 'payload') continue
|
||
|
|
for (const value of [undefined, typeof sixth.sixthGuardPausedOperation[key] === 'number' ? -1 : 'changed']) {
|
||
|
|
const paused = { ...sixth.sixthGuardPausedOperation, [key]: value }
|
||
|
|
const changed = { ...previous, recovery_budget: { ...previous.recovery_budget, paused_payload: JSON.stringify([paused]) } }
|
||
|
|
assert.throws(() => policy.planRecovery([source], [pilot, changed], 6, reason, review), key)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for (const key of Object.keys(sixth.sixthGuardPausedOperation.payload)) {
|
||
|
|
for (const value of [undefined, 'changed']) {
|
||
|
|
const paused = { ...sixth.sixthGuardPausedOperation, payload: { ...sixth.sixthGuardPausedOperation.payload, [key]: value } }
|
||
|
|
const changed = { ...previous, recovery_budget: { ...previous.recovery_budget, paused_payload: JSON.stringify([paused]) } }
|
||
|
|
assert.throws(() => policy.planRecovery([source], [pilot, changed], 6, reason, review), key)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
test('saved-response replay preserves distinct identities and original failure without a new provider call', () => {
|
||
|
|
const { evidence, review, bytes } = replayFixture()
|
||
|
|
assert.doesNotThrow(() => sixth.validateSixthReplayEvidence(review, bytes, evidence.source_tree_hash))
|
||
|
|
assert.throws(() => sixth.validateSixthReplayEvidence(review, bytes + '\n', evidence.source_tree_hash))
|
||
|
|
assert.throws(() => sixth.validateSixthReplayEvidence(review, bytes, 'c'.repeat(64)))
|
||
|
|
const changes = [{ status: 'failed' }, { model_calls: 1 }, { production_writes: 1 },
|
||
|
|
{ source_preserved: false }, { original_receipt_preserved: false }, { rejected_count: 1 },
|
||
|
|
{ entity_count: 21 }, { concept_count: 66 }, { candidate_count: 88 },
|
||
|
|
{ input_run_id: 'other' }, { source_window_sha256: 'c'.repeat(64) },
|
||
|
|
{ response_sha256: 'c'.repeat(64) }, { input_evidence_sha256: 'c'.repeat(64) },
|
||
|
|
{ run_id: 'other' }, { completed_at: 'invalid' }, { completed_at: '2026-09-21T16:20:00Z' }]
|
||
|
|
for (const change of changes) {
|
||
|
|
const changed = JSON.stringify({ ...evidence, ...change })
|
||
|
|
const changedReview = { ...review, candidate_diagnostic: { ...review.candidate_diagnostic, evidence_sha256: sha(changed) } }
|
||
|
|
assert.throws(() => sixth.validateSixthReplayEvidence(changedReview, changed, evidence.source_tree_hash))
|
||
|
|
}
|
||
|
|
assert.throws(() => sixth.validateSixthDiagnosticEvidence(review, bytes, evidence.source_tree_hash))
|
||
|
|
})
|
||
|
|
|
||
|
|
test('exact sixth scope uses a new receipt while attempt five and the default limit stay unchanged', () => {
|
||
|
|
const { source, previous, pilot, receipt, review } = fixture()
|
||
|
|
const original = JSON.stringify(previous)
|
||
|
|
assert.equal(policy.maxReviewedAttempt, 4)
|
||
|
|
assert.deepEqual(policy.planRecovery([source], [pilot, previous], 6, reason, review), [source])
|
||
|
|
assert.deepEqual(policy.planRecovery([source], [pilot, previous, receipt], 6, reason, review), [])
|
||
|
|
assert.doesNotThrow(() => policy.validateFinalRecoveryTrigger(receipt, { ...source, pending: 1 }, [pilot, previous, receipt], review))
|
||
|
|
assert.equal(JSON.stringify(previous), original)
|
||
|
|
assert.throws(() => policy.planRecovery([source], [pilot, previous], 7, reason, { ...review, max_attempt: 7 }))
|
||
|
|
assert.equal(policy.durableRecoveryStatus({ ...receipt, recovery_budget: undefined }), 'stopped_invalid')
|
||
|
|
for (const key of Object.keys(sixth.sixthRecoveryScope)) {
|
||
|
|
const sourceKey = key === 'dead_letter_id' ? 'id' : key
|
||
|
|
assert.throws(() => policy.planRecovery([{ ...source, [sourceKey]: 'changed' }], [pilot, previous], 6, reason, review))
|
||
|
|
assert.throws(() => policy.planRecovery([source], [pilot, previous], 6, reason, { ...review, [key]: 'changed' }))
|
||
|
|
}
|
||
|
|
for (const change of [{ attempt_number: 4 }, { pending_op_id: 4143 }, { recovery_status: 'failed' },
|
||
|
|
{ pending: true }, { pending_finalize: 1 }, { error_code: 'other' }, { new_failures: 0 }, { recovery_budget: undefined },
|
||
|
|
{ recovery_budget: { ...previous.recovery_budget, status: 'active' } },
|
||
|
|
{ recovery_budget: { ...previous.recovery_budget, pending_op_id: 4143 } }]) {
|
||
|
|
assert.throws(() => policy.planRecovery([source], [pilot, { ...previous, ...change }], 6, reason, review))
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
test('sixth budgets, semantic review and published candidate are bounded independently', () => {
|
||
|
|
const { review, source, previous, pilot, receipt } = fixture()
|
||
|
|
for (const change of [{ max_provider_requests: 1001 }, { max_provider_requests: 0 }, { budget_version: 0 },
|
||
|
|
{ previous_pending_op_id: 4143 }, { public_gates_passed: false }, { candidate_diagnostic: undefined },
|
||
|
|
{ candidate_diagnostic: { ...review.candidate_diagnostic, semantic_reviewed: false } },
|
||
|
|
{ api_image_id: sixth.diagnosticPins.image }, { fix_release: sixth.diagnosticPins.release },
|
||
|
|
{ budget_started_at: 'invalid' }, { budget_started_at: new Date(Date.now() + 50000).toISOString() },
|
||
|
|
{ deadline_at: new Date(Date.parse(review.budget_started_at) + 7200001).toISOString() },
|
||
|
|
{ deadline_at: review.budget_started_at }]) {
|
||
|
|
assert.throws(() => policy.planRecovery([source], [pilot, previous], 6, reason, { ...review, ...change }))
|
||
|
|
}
|
||
|
|
for (const change of [{ status: 'stopped_budget' }, { status: 'completed' }, { reserved_provider_requests: 1000 },
|
||
|
|
{ max_provider_requests: 1001 }, { deadline_at: new Date(Date.parse(review.deadline_at) + 1).toISOString() }]) {
|
||
|
|
const changed = { ...receipt, recovery_budget: { ...receipt.recovery_budget, ...change } }
|
||
|
|
assert.throws(() => policy.validateFinalRecoveryTrigger(changed, { ...source, pending: 1 }, [pilot, previous, changed], review))
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
test('sixth recovery checks successful V3 bytes and exact candidate source rather than a summary claim', () => {
|
||
|
|
const { evidence, bytes, review } = fixture()
|
||
|
|
assert.doesNotThrow(() => sixth.validateSixthDiagnosticEvidence(review, bytes, evidence.source_tree_hash))
|
||
|
|
assert.throws(() => sixth.validateSixthDiagnosticEvidence(review, bytes + '\n', evidence.source_tree_hash), /changed/)
|
||
|
|
assert.throws(() => sixth.validateSixthDiagnosticEvidence(review, bytes, 'c'.repeat(64)), /Current candidate/)
|
||
|
|
const changes = [value => { value.status = 'stopped' }, value => { value.protected_state_unchanged = false },
|
||
|
|
value => { value.previous_diagnostics_unchanged = false }, value => { value.candidate_source_only = false },
|
||
|
|
value => { value.run_id = 'different' }, value => { value.pins.maxSends = 22 },
|
||
|
|
value => { value.outcome.status = 'failed' }, value => { value.outcome.candidate_count = 0 },
|
||
|
|
value => { value.outcome.window.start_rune++ }, value => { value.outcome.source_hash = 'c'.repeat(64) },
|
||
|
|
value => { value.outcome.actual_send_reservations++ }, value => { value.actual_send_reservations = 22 },
|
||
|
|
value => { value.outcome.leaves[0].start_rune++ }, value => { value.outcome.leaves[0].end_rune-- }]
|
||
|
|
for (const change of changes) {
|
||
|
|
const changed = structuredClone(evidence); change(changed)
|
||
|
|
const changedBytes = JSON.stringify(changed)
|
||
|
|
const changedReview = { ...review, candidate_diagnostic: { ...review.candidate_diagnostic, evidence_sha256: sha(changedBytes) } }
|
||
|
|
assert.throws(() => sixth.validateSixthDiagnosticEvidence(changedReview, changedBytes, evidence.source_tree_hash))
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
function runCLI(data, trigger = false, corruptEvidence = false) {
|
||
|
|
let transaction, triggered = 0
|
||
|
|
const errors = []
|
||
|
|
const context = { __dirname,
|
||
|
|
process: { argv: ['node', 'recover_wiki_deadletters.cjs', ...(trigger ? ['--trigger', '--ids=10'] :
|
||
|
|
['--apply', '--ids=10', '--attempt=6', `--reason=${reason}`, '--review-file=/fixture/review.json'])] },
|
||
|
|
console: { log() {}, error(value) { errors.push(value) } },
|
||
|
|
require(name) {
|
||
|
|
if (name === 'node:fs') return { mkdirSync() {}, writeFileSync() {}, readFileSync(file) {
|
||
|
|
if (file === '/fixture/review.json') return JSON.stringify(data.review)
|
||
|
|
if (path.basename(file) === 'public-release.json') return JSON.stringify({ stage: 'published',
|
||
|
|
release: data.review.fix_release, api_image_id: data.review.api_image_id, published_at: data.review.published_at })
|
||
|
|
assert.equal(path.basename(file), 'evidence.json')
|
||
|
|
return data.bytes + (corruptEvidence ? '\n' : '')
|
||
|
|
} }
|
||
|
|
if (name === './wiki_sixth_recovery_review.cjs') return { ...sixth,
|
||
|
|
candidateSourceTreeHash: () => data.evidence.source_tree_hash,
|
||
|
|
verifyReplayInputFiles: () => { assert.ok(!data.inputDrift, 'Original replay input changed') } }
|
||
|
|
if (name === './wiki_recovery_budget_sql.cjs') return { recoveryFingerprintsSQL: () => ({
|
||
|
|
source: `'${data.source.source_fingerprint}'::text`, model: `'${data.source.model_fingerprint}'::text` }) }
|
||
|
|
if (name === 'node:child_process') return { execFileSync(command, args, options = {}) {
|
||
|
|
assert.equal(command, 'docker')
|
||
|
|
if (args.includes('inspect')) return JSON.stringify({ running: true, release: data.review.fix_release, image_id: data.review.api_image_id })
|
||
|
|
if (args.includes('cp')) return ''
|
||
|
|
if (args.includes('/tmp/wiki-backfill-trigger')) { triggered++; return '{}' }
|
||
|
|
assert.ok(args.includes('psql'))
|
||
|
|
if (options.input.startsWith('BEGIN;')) { transaction = options.input; throw new Error('Captured transaction') }
|
||
|
|
if (options.input.includes('to_regclass')) return 't'
|
||
|
|
if (options.input.includes('FROM (SELECT ledger.*')) return JSON.stringify([data.pilot, data.previous, ...(trigger ? [data.receipt] : [])])
|
||
|
|
if (options.input.includes('SELECT coalesce(json_agg(row_to_json(d))')) return '[]'
|
||
|
|
return JSON.stringify([{ ...data.source, pending: trigger ? 1 : 0 }])
|
||
|
|
} }
|
||
|
|
return require(name)
|
||
|
|
} }
|
||
|
|
vm.runInNewContext(fs.readFileSync(path.join(__dirname, 'recover_wiki_deadletters.cjs'), 'utf8'), context)
|
||
|
|
return { transaction, triggered, errors }
|
||
|
|
}
|
||
|
|
|
||
|
|
test('actual CLI checks immutable V3 evidence before enqueue and every explicit trigger', () => {
|
||
|
|
const data = fixture()
|
||
|
|
const good = runCLI(data)
|
||
|
|
assert.ok(good.transaction)
|
||
|
|
for (const required of ['p.attempt_number=5 AND p.pending_op_id=4144', "b.status='stopped_budget'", 'later.attempt_number>5',
|
||
|
|
"interval '120 minutes'", '1000 BETWEEN 1 AND 1000', 'b.source_fingerprint=', 'b.model_fingerprint=']) assert.ok(good.transaction.includes(required))
|
||
|
|
assert.equal(runCLI(data, true).triggered, 1)
|
||
|
|
assert.equal(runCLI(data, false, true).transaction, undefined)
|
||
|
|
assert.equal(runCLI(data, true, true).triggered, 0)
|
||
|
|
})
|
||
|
|
|
||
|
|
test('actual CLI rechecks replay and immutable original inputs before enqueue or trigger', () => {
|
||
|
|
const data = replayFixture()
|
||
|
|
assert.ok(runCLI(data).transaction)
|
||
|
|
assert.equal(runCLI(data, true).triggered, 1)
|
||
|
|
assert.equal(runCLI(data, false, true).transaction, undefined)
|
||
|
|
assert.equal(runCLI(data, true, true).triggered, 0)
|
||
|
|
data.inputDrift = true
|
||
|
|
assert.equal(runCLI(data).transaction, undefined)
|
||
|
|
assert.equal(runCLI(data, true).triggered, 0)
|
||
|
|
})
|
||
|
|
|
||
|
|
test('actual CLI validates the reviewed paused operation before enqueue and trigger', () => {
|
||
|
|
const data = guardPausedFixture()
|
||
|
|
const good = runCLI(data)
|
||
|
|
assert.ok(good.transaction)
|
||
|
|
assert.equal(runCLI(data, true).triggered, 1)
|
||
|
|
for (const required of ["b.stop_reason='guard_rejected'", "b.last_phase='wiki_candidate_slug'", 'b.reserved_provider_requests=4',
|
||
|
|
'jsonb_array_length(b.paused_payload::jsonb)=1', 'b.paused_payload::jsonb @>', "span.status='failed' AND span.error_code='EXTRACT_FAILED'",
|
||
|
|
'ORDER BY span.started_at DESC,span.id DESC LIMIT 1']) assert.ok(good.transaction.includes(required), required)
|
||
|
|
data.previous.recovery_budget.paused_payload = '[]'
|
||
|
|
assert.equal(runCLI(data).transaction, undefined)
|
||
|
|
assert.equal(runCLI(data, true).triggered, 0)
|
||
|
|
})
|
||
|
|
|
||
|
|
test('actual sixth enqueue SQL preserves old receipts, refuses drift and rolls back atomically', { skip: process.env.VERIFY_SIXTH_SQL !== '1' }, async () => {
|
||
|
|
const data = guardPausedFixture(), scope = sixth.sixthRecoveryScope, quote = value => `'${String(value).replaceAll("'", "''")}'`
|
||
|
|
const transaction = runCLI(data).transaction
|
||
|
|
assert.ok(transaction)
|
||
|
|
const name = 'yawan-wiki-sixth-sql-test-' + crypto.randomUUID()
|
||
|
|
const docker = (args, input) => cp.execFileSync('docker', ['--context', 'colima-yawan', ...args], { input, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 30000 })
|
||
|
|
const sql = input => docker(['exec', '-i', name, 'psql', '-X', '-qAt', '-U', 'postgres', '-v', 'ON_ERROR_STOP=1'], input).trim()
|
||
|
|
const setup = `SET search_path=pg_temp;
|
||
|
|
CREATE TEMP TABLE task_pending_ops(id bigserial PRIMARY KEY,tenant_id bigint,task_type text,scope text,scope_id text,op text,dedup_key text,payload jsonb,fail_count int,enqueued_at timestamptz);
|
||
|
|
CREATE TEMP TABLE task_dead_letters(id bigint PRIMARY KEY,tenant_id bigint,task_type text,scope text,scope_id text,related_id text,payload jsonb);
|
||
|
|
CREATE TEMP TABLE knowledges(id text,tenant_id bigint,knowledge_base_id text,parse_status text,pending_subtasks_count int,deleted_at timestamptz);
|
||
|
|
CREATE TEMP TABLE knowledge_bases(id text,tenant_id bigint,wiki_config jsonb,indexing_strategy jsonb,embedding_model_id text,deleted_at timestamptz);
|
||
|
|
CREATE TEMP TABLE chunks(tenant_id bigint,knowledge_id text,content text,deleted_at timestamptz);
|
||
|
|
CREATE TEMP TABLE models(id text);
|
||
|
|
CREATE TEMP TABLE wiki_pages(tenant_id bigint,knowledge_base_id text,source_refs jsonb,deleted_at timestamptz);
|
||
|
|
CREATE TEMP TABLE knowledge_processing_spans(id bigint,knowledge_id text,name text,status text,error_code text,started_at timestamptz);
|
||
|
|
CREATE TEMP TABLE wiki_dead_letter_recoveries(tenant_id bigint NOT NULL,dead_letter_id bigint NOT NULL,knowledge_id varchar NOT NULL,knowledge_base_id varchar NOT NULL,model_id varchar NOT NULL,pending_op_id bigint NOT NULL,queued_at timestamptz NOT NULL,baseline_pages bigint NOT NULL,baseline_chunks bigint NOT NULL,chunk_chars bigint NOT NULL,dead_letter_highwater bigint NOT NULL,attempt_number integer NOT NULL,retry_reason text NOT NULL DEFAULT '',deployed_fix_review jsonb,PRIMARY KEY(tenant_id,dead_letter_id,attempt_number));
|
||
|
|
${fs.readFileSync(path.join(__dirname, '../migrations/versioned/000214_wiki_recovery_budgets.up.sql'), 'utf8').replace('CREATE TABLE IF NOT EXISTS', 'CREATE TEMP TABLE')}
|
||
|
|
INSERT INTO task_dead_letters VALUES(10,10059,'wiki:ingest','knowledge_base',${quote(scope.knowledge_base_id)},${quote(scope.knowledge_id)},jsonb_build_object('op','ingest','knowledge_id',${quote(scope.knowledge_id)}));
|
||
|
|
INSERT INTO knowledges VALUES(${quote(scope.knowledge_id)},10059,${quote(scope.knowledge_base_id)},'completed',0,NULL);
|
||
|
|
INSERT INTO knowledge_bases VALUES(${quote(scope.knowledge_base_id)},10059,jsonb_build_object('synthesis_model_id',${quote(scope.model_id)}),'{"wiki_enabled":true}',${quote(scope.embedding_model_id)},NULL);
|
||
|
|
INSERT INTO chunks VALUES(10059,${quote(scope.knowledge_id)},'fixture',NULL);
|
||
|
|
INSERT INTO wiki_dead_letter_recoveries VALUES(10059,10,${quote(scope.knowledge_id)},${quote(scope.knowledge_base_id)},${quote(scope.model_id)},4144,'2026-09-20T08:58:56Z',23,624,227094,45,5,'retained original',NULL);
|
||
|
|
INSERT INTO wiki_recovery_budgets(tenant_id,dead_letter_id,attempt_number,pending_op_id,knowledge_base_id,knowledge_id,model_id,embedding_model_id,source_fingerprint,model_fingerprint,reviewed_release,reviewed_api_image,max_provider_requests,reserved_provider_requests,deadline_at,status,paused_payload,verified_progress)
|
||
|
|
VALUES(10059,10,5,4144,${quote(scope.knowledge_base_id)},${quote(scope.knowledge_id)},${quote(scope.model_id)},${quote(scope.embedding_model_id)},${quote(scope.source_fingerprint)},${quote(scope.model_fingerprint)},'retained-release','retained-image',1000,4,'2026-09-20T15:00:00Z','stopped_budget','[]','{}');
|
||
|
|
UPDATE wiki_recovery_budgets SET stop_reason='guard_rejected',last_phase='wiki_candidate_slug',paused_payload=${quote(data.previous.recovery_budget.paused_payload)};
|
||
|
|
INSERT INTO knowledge_processing_spans VALUES(1,${quote(scope.knowledge_id)},'postprocess.wiki','failed','EXTRACT_FAILED','2026-09-20T08:59:00Z');
|
||
|
|
CREATE TEMP TABLE prior_receipt AS SELECT to_jsonb(r) AS body FROM wiki_dead_letter_recoveries r WHERE attempt_number=5;
|
||
|
|
CREATE TEMP TABLE prior_budget AS SELECT to_jsonb(b) AS body FROM wiki_recovery_budgets b WHERE attempt_number=5;
|
||
|
|
`
|
||
|
|
const counts = `SELECT json_build_object('ops',(SELECT count(*) FROM task_pending_ops),'new_budgets',(SELECT count(*) FROM wiki_recovery_budgets WHERE attempt_number=6),'new_receipts',(SELECT count(*) FROM wiki_dead_letter_recoveries WHERE attempt_number=6),'old_receipt_unchanged',(SELECT to_jsonb(r)=(SELECT body FROM prior_receipt) FROM wiki_dead_letter_recoveries r WHERE attempt_number=5),'old_budget_unchanged',(SELECT to_jsonb(b)=(SELECT body FROM prior_budget) FROM wiki_recovery_budgets b WHERE attempt_number=5));`
|
||
|
|
const inspect = input => JSON.parse(sql(input + counts).split('\n').at(-1))
|
||
|
|
try {
|
||
|
|
docker(['run', '-d', '--name', name, '--network', 'none', '--cpus', '1', '--memory', '768m', '--tmpfs', '/var/lib/postgresql/data:rw', '-e', 'POSTGRES_HOST_AUTH_METHOD=trust', 'paradedb/paradedb:v0.22.2-pg17'])
|
||
|
|
let ready = false
|
||
|
|
for (let i = 0; i < 60; i++) {
|
||
|
|
try { docker(['exec', name, 'pg_isready', '-U', 'postgres']); ready = true; break } catch {}
|
||
|
|
await new Promise(resolve => setTimeout(resolve, 500))
|
||
|
|
}
|
||
|
|
assert.ok(ready, 'Disposable PostgreSQL did not become ready')
|
||
|
|
assert.deepEqual(inspect(setup + transaction + transaction), { ops: 1, new_budgets: 1, new_receipts: 1, old_receipt_unchanged: true, old_budget_unchanged: true })
|
||
|
|
for (const drift of ["UPDATE wiki_recovery_budgets SET status='active';", 'UPDATE wiki_dead_letter_recoveries SET pending_op_id=4143;',
|
||
|
|
"UPDATE wiki_recovery_budgets SET source_fingerprint='changed';", "UPDATE wiki_recovery_budgets SET stop_reason='deadline';",
|
||
|
|
"UPDATE wiki_recovery_budgets SET last_phase='wiki_summary';", 'UPDATE wiki_recovery_budgets SET reserved_provider_requests=3;',
|
||
|
|
"UPDATE wiki_recovery_budgets SET paused_payload='[]';",
|
||
|
|
"UPDATE wiki_recovery_budgets SET paused_payload=jsonb_set(paused_payload::jsonb,'{0,id}','4143')::text;",
|
||
|
|
"UPDATE wiki_recovery_budgets SET paused_payload=jsonb_set(paused_payload::jsonb,'{0,scope_id}','\"changed\"')::text;",
|
||
|
|
"UPDATE wiki_recovery_budgets SET paused_payload=jsonb_set(paused_payload::jsonb,'{0,payload,recovery_attempt}','4')::text;",
|
||
|
|
'DELETE FROM knowledge_processing_spans;', "UPDATE knowledge_processing_spans SET status='done';",
|
||
|
|
"UPDATE knowledge_processing_spans SET error_code='other';",
|
||
|
|
`INSERT INTO knowledge_processing_spans VALUES(2,${quote(scope.knowledge_id)},'postprocess.wiki','done','','2026-09-20T09:00:00Z');`]) {
|
||
|
|
const result = inspect(setup + drift + transaction)
|
||
|
|
assert.equal(result.ops, 0); assert.equal(result.new_budgets, 0); assert.equal(result.new_receipts, 0)
|
||
|
|
}
|
||
|
|
const legacyFailure = `UPDATE wiki_recovery_budgets SET paused_payload='[]',stop_reason='',last_phase='';
|
||
|
|
INSERT INTO task_dead_letters VALUES(46,10059,'wiki:ingest','knowledge_base',${quote(scope.knowledge_base_id)},${quote(scope.knowledge_id)},'{}');`
|
||
|
|
const legacyResult = inspect(setup + legacyFailure + transaction)
|
||
|
|
assert.equal(legacyResult.ops, 1); assert.equal(legacyResult.new_budgets, 1); assert.equal(legacyResult.new_receipts, 1)
|
||
|
|
assert.deepEqual(inspect(setup + `ALTER TABLE wiki_dead_letter_recoveries ADD CONSTRAINT reject_new CHECK(attempt_number<>6);\n\\set ON_ERROR_STOP off\n` + transaction + '\n\\set ON_ERROR_STOP on\n'),
|
||
|
|
{ ops: 0, new_budgets: 0, new_receipts: 0, old_receipt_unchanged: true, old_budget_unchanged: true })
|
||
|
|
} finally {
|
||
|
|
docker(['rm', '-f', name])
|
||
|
|
}
|
||
|
|
})
|