const assert = require('node:assert/strict') const { execFileSync, spawn } = require('node:child_process') const fs = require('node:fs') const path = require('node:path') const crypto = require('node:crypto') const deployment = path.resolve(__dirname, '../..') const chunkID = process.env.GRAPH_PROBE_CHUNK || '1144acfb-18de-4687-8bf5-df0e9c8f1dd5' const requestedModelID = process.env.GRAPH_PROBE_MODEL_ID const expectedSourceHash = process.env.GRAPH_PROBE_SOURCE_SHA256 const reportTag = process.env.GRAPH_PROBE_REPORT_TAG || '' const maxSendsText = process.env.GRAPH_PROBE_MAX_SENDS ?? '3' const maxSends = Number(maxSendsText) const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i function fail(stage, error) { console.error(JSON.stringify({ passed: false, stage, error, production_write: false })) process.exit(1) } if (!['1', '2', '3'].includes(maxSendsText)) fail('input', 'Probe HTTP send cap must be an integer from 1 to 3') if (!uuid.test(chunkID) || !requestedModelID || !uuid.test(requestedModelID) || !expectedSourceHash || !reportTag) { fail('input', 'A bounded probe requires exact chunk/model IDs, source SHA-256 and a single-use report tag') } if ((expectedSourceHash !== undefined && !/^[0-9a-f]{64}$/.test(expectedSourceHash)) || (reportTag && !/^[a-z0-9][a-z0-9-]{0,63}$/.test(reportTag))) { fail('input', 'Expected source SHA-256 or report tag is invalid') } const suffix = (requestedModelID ? `-${requestedModelID.toLowerCase()}` : '') + (reportTag ? `-${reportTag}` : '') const outputPath = path.join(deployment, 'acceptance', `graph-extraction-probe-${chunkID.toLowerCase()}${suffix}.json`) const receiptPath = outputPath.replace(/\.json$/, '.spent.json') if (reportTag && fs.existsSync(outputPath)) fail('report_write', 'Tagged probe report already exists; historical evidence cannot be overwritten') if (fs.existsSync(receiptPath)) fail('execution_receipt', 'Probe allocation is already spent; its receipt must be retained') const dockerPrefix = ['--context', 'colima-yawan'] const modelSelector = requestedModelID ? `'${requestedModelID}'` : 'kb.summary_model_id' const query = `BEGIN READ ONLY; SELECT json_build_object('model',to_jsonb(m),'chunk',to_jsonb(c),'extract_config',kb.extract_config) FROM chunks c JOIN knowledge_bases kb ON kb.id=c.knowledge_base_id AND kb.tenant_id=c.tenant_id JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id JOIN models m ON m.id=${modelSelector} AND m.tenant_id=c.tenant_id AND m.type='KnowledgeQA' WHERE c.id='${chunkID}' AND c.tenant_id=10059 AND c.deleted_at IS NULL AND c.is_enabled=true AND c.chunk_type='text' AND k.deleted_at IS NULL AND k.parse_status='completed' AND kb.deleted_at IS NULL AND m.deleted_at IS NULL; COMMIT;` function readJSON(args, input, stage) { try { const result = execFileSync('docker', [...dockerPrefix, ...args], { input, encoding: 'utf8', maxBuffer: 10 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], }) if (!result.trim()) fail(stage, 'No matching source or same-tenant chat model') return JSON.parse(result) } catch { fail(stage, 'Probe setup failed; raw command output is withheld') } } const input = readJSON(['exec', '-i', 'WeKnora-postgres', 'psql', '-X', '-qAt', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915'], query, 'source_lookup') if (expectedSourceHash && crypto.createHash('sha256').update(input.chunk.content).digest('hex') !== expectedSourceHash) { fail('source_validation', 'Reviewed source SHA-256 changed; no model call was made') } const environment = readJSON(['inspect', 'yawan-native-acceptance', '--format', '{{json .Config.Env}}'], undefined, 'runtime_lookup') input.aes_key = environment.find(value => value.startsWith('SYSTEM_AES_KEY='))?.slice('SYSTEM_AES_KEY='.length) || '' try { assert.equal(input.chunk.id.toLowerCase(), chunkID.toLowerCase()) assert.equal(input.chunk.tenant_id, 10059) assert.equal(input.model.tenant_id, input.chunk.tenant_id) assert.equal(input.model.type, 'KnowledgeQA') if (requestedModelID) assert.equal(input.model.id.toLowerCase(), requestedModelID.toLowerCase()) } catch { fail('source_validation', 'Probe source or same-tenant chat model validation failed') } let receiptFD try { receiptFD = fs.openSync(receiptPath, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, 0o600) fs.writeFileSync(receiptFD, JSON.stringify({ execution_id: path.basename(outputPath), tenant_id: input.chunk.tenant_id, chunk_id: chunkID, model_id: requestedModelID, source_sha256: expectedSourceHash, report_tag: reportTag, allocated_http_sends: maxSends, model_deadline_seconds: 180, spent_at: new Date().toISOString(), }) + '\n') fs.fsyncSync(receiptFD) const directoryFD = fs.openSync(path.dirname(receiptPath), fs.constants.O_RDONLY) try { fs.fsyncSync(directoryFD) } finally { fs.closeSync(directoryFD) } } catch { fail('execution_receipt', 'Cannot reserve the single-use probe allocation; no model process was started') } finally { if (receiptFD !== undefined) fs.closeSync(receiptFD) } input.execution_id = path.basename(outputPath) input.max_http_sends = maxSends const child = spawn('docker', [...dockerPrefix, 'run', '--rm', '-i', '-v', `${deployment}/src:/src:ro`, '-v', `${deployment}/go-cache:/go`, '-v', `${deployment}/go-build:/build-cache`, '-w', '/src', '-e', 'CGO_ENABLED=1', '-e', 'GOCACHE=/build-cache', '-e', `GRAPH_PROBE_MAX_SENDS=${maxSends}`, 'golang:1.26-bookworm', 'go', 'run', './scripts/graph_extraction_probe'], { stdio: ['pipe', 'pipe', 'pipe'] }) child.stdin.end(JSON.stringify(input)) input.aes_key = '' input.model.parameters.api_key = '' let output = '' child.stdout.on('data', data => { output += data }) child.stderr.resume() child.on('error', () => fail('model_probe', 'Could not start read-only model probe')) child.on('close', code => { const line = output.split('\n').find(value => value.startsWith('GRAPH_EXTRACTION_PROBE_RESULT=')) if (code !== 0 || !line) { fail('model_probe', 'Graph extraction probe failed before a result; raw model output is withheld') } let report try { report = JSON.parse(line.slice('GRAPH_EXTRACTION_PROBE_RESULT='.length)) } catch { fail('model_probe', 'Graph extraction probe returned an invalid report') } if (report.diagnostics_version !== 2 || report.execution_id !== input.execution_id || report.http_send_limit !== maxSends || !Number.isInteger(report.http_send_reservations) || report.http_send_reservations < 0 || report.http_send_reservations > maxSends || !Number.isInteger(report.model_calls) || report.model_calls < 1 || report.model_calls > 2 || (report.passed && report.http_send_reservations === 0)) { fail('model_probe_budget', 'Probe result has missing or invalid actual-send accounting; allocation remains spent') } report.model_id = input.model.id report.source_content = input.chunk.content if (chunkID === '1144acfb-18de-4687-8bf5-df0e9c8f1dd5') { const relations = report.graph?.relation || [] const expected = [ ['过敏者', ['慎用于']], ['粘膜破损处', ['禁用于', '不适用于', '不能用于']], ['性生活中对性病的预防', ['禁用于', '不适用于', '不能用于']], ] report.semantic_checks = expected.map(([target, labels]) => ({ target, passed: relations.some(relation => relation.node1 === '本品' && relation.node2 === target && labels.includes(relation.type)), })) report.passed = report.passed && report.semantic_checks.every(check => check.passed) } let fd try { fd = fs.openSync(outputPath, fs.constants.O_WRONLY | fs.constants.O_CREAT | (reportTag ? fs.constants.O_EXCL : fs.constants.O_TRUNC) | fs.constants.O_NOFOLLOW, 0o600) fs.fchmodSync(fd, 0o600) fs.writeFileSync(fd, JSON.stringify(report, null, 2) + '\n') } catch { fail('report_write', 'Could not save the private probe report') } finally { if (fd !== undefined) fs.closeSync(fd) } console.log(JSON.stringify({ passed: report.passed, chunk_id: report.chunk_id, model_id: report.model_id, model_name: report.model_name, source_sha256: report.source_sha256, elapsed_ms: report.elapsed_ms, model_calls: report.model_calls, http_send_reservations: report.http_send_reservations, http_send_limit: report.http_send_limit, execution_id: report.execution_id, receipt_path: receiptPath, relation_count: report.graph?.relation?.length || 0, error_category: report.error ? (report.error.includes('graph_needs_review') ? 'graph_needs_review' : 'extraction_failed') : null, production_write: false, report_path: outputPath, })) if (!report.passed) process.exitCode = 1 })