WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

2026-09-22 16:19:06 +08:00
commit f7fc992f57
3929 changed files with 1006607 additions and 0 deletions
@@ -0,0 +1,155 @@
+const assert = require('node:assert/strict')
+const { execFileSync, spawn } = require('node:child_process')
+const fs = require('node:fs')
+const path = require('node:path')
+const crypto = require('node:crypto')
+
+const deployment = path.resolve(__dirname, '../..')
+const chunkID = process.env.GRAPH_PROBE_CHUNK || '1144acfb-18de-4687-8bf5-df0e9c8f1dd5'
+const requestedModelID = process.env.GRAPH_PROBE_MODEL_ID
+const expectedSourceHash = process.env.GRAPH_PROBE_SOURCE_SHA256
+const reportTag = process.env.GRAPH_PROBE_REPORT_TAG || ''
+const maxSendsText = process.env.GRAPH_PROBE_MAX_SENDS ?? '3'
+const maxSends = Number(maxSendsText)
+const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
+function fail(stage, error) {
+  console.error(JSON.stringify({ passed: false, stage, error, production_write: false }))
+  process.exit(1)
+}
+if (!['1', '2', '3'].includes(maxSendsText)) fail('input', 'Probe HTTP send cap must be an integer from 1 to 3')
+if (!uuid.test(chunkID) || !requestedModelID || !uuid.test(requestedModelID) || !expectedSourceHash || !reportTag) {
+  fail('input', 'A bounded probe requires exact chunk/model IDs, source SHA-256 and a single-use report tag')
+}
+if ((expectedSourceHash !== undefined && !/^[0-9a-f]{64}$/.test(expectedSourceHash))
+  || (reportTag && !/^[a-z0-9][a-z0-9-]{0,63}$/.test(reportTag))) {
+  fail('input', 'Expected source SHA-256 or report tag is invalid')
+}
+const suffix = (requestedModelID ? `-${requestedModelID.toLowerCase()}` : '') + (reportTag ? `-${reportTag}` : '')
+const outputPath = path.join(deployment, 'acceptance', `graph-extraction-probe-${chunkID.toLowerCase()}${suffix}.json`)
+const receiptPath = outputPath.replace(/\.json$/, '.spent.json')
+if (reportTag && fs.existsSync(outputPath)) fail('report_write', 'Tagged probe report already exists; historical evidence cannot be overwritten')
+if (fs.existsSync(receiptPath)) fail('execution_receipt', 'Probe allocation is already spent; its receipt must be retained')
+const dockerPrefix = ['--context', 'colima-yawan']
+const modelSelector = requestedModelID ? `'${requestedModelID}'` : 'kb.summary_model_id'
+const query = `BEGIN READ ONLY;
+SELECT json_build_object('model',to_jsonb(m),'chunk',to_jsonb(c),'extract_config',kb.extract_config)
+FROM chunks c JOIN knowledge_bases kb ON kb.id=c.knowledge_base_id AND kb.tenant_id=c.tenant_id
+JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id
+JOIN models m ON m.id=${modelSelector} AND m.tenant_id=c.tenant_id AND m.type='KnowledgeQA'
+WHERE c.id='${chunkID}' AND c.tenant_id=10059 AND c.deleted_at IS NULL AND c.is_enabled=true AND c.chunk_type='text'
+AND k.deleted_at IS NULL AND k.parse_status='completed' AND kb.deleted_at IS NULL AND m.deleted_at IS NULL;
+COMMIT;`
+function readJSON(args, input, stage) {
+  try {
+    const result = execFileSync('docker', [...dockerPrefix, ...args], {
+      input, encoding: 'utf8', maxBuffer: 10 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'],
+    })
+    if (!result.trim()) fail(stage, 'No matching source or same-tenant chat model')
+    return JSON.parse(result)
+  } catch {
+    fail(stage, 'Probe setup failed; raw command output is withheld')
+  }
+}
+const input = readJSON(['exec', '-i', 'WeKnora-postgres',
+  'psql', '-X', '-qAt', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915'],
+query, 'source_lookup')
+if (expectedSourceHash && crypto.createHash('sha256').update(input.chunk.content).digest('hex') !== expectedSourceHash) {
+  fail('source_validation', 'Reviewed source SHA-256 changed; no model call was made')
+}
+const environment = readJSON(['inspect', 'yawan-native-acceptance', '--format', '{{json .Config.Env}}'], undefined, 'runtime_lookup')
+input.aes_key = environment.find(value => value.startsWith('SYSTEM_AES_KEY='))?.slice('SYSTEM_AES_KEY='.length) || ''
+try {
+  assert.equal(input.chunk.id.toLowerCase(), chunkID.toLowerCase())
+  assert.equal(input.chunk.tenant_id, 10059)
+  assert.equal(input.model.tenant_id, input.chunk.tenant_id)
+  assert.equal(input.model.type, 'KnowledgeQA')
+  if (requestedModelID) assert.equal(input.model.id.toLowerCase(), requestedModelID.toLowerCase())
+} catch {
+  fail('source_validation', 'Probe source or same-tenant chat model validation failed')
+}
+let receiptFD
+try {
+  receiptFD = fs.openSync(receiptPath, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, 0o600)
+  fs.writeFileSync(receiptFD, JSON.stringify({
+    execution_id: path.basename(outputPath), tenant_id: input.chunk.tenant_id, chunk_id: chunkID,
+    model_id: requestedModelID, source_sha256: expectedSourceHash, report_tag: reportTag,
+    allocated_http_sends: maxSends, model_deadline_seconds: 180, spent_at: new Date().toISOString(),
+  }) + '\n')
+  fs.fsyncSync(receiptFD)
+  const directoryFD = fs.openSync(path.dirname(receiptPath), fs.constants.O_RDONLY)
+  try { fs.fsyncSync(directoryFD) } finally { fs.closeSync(directoryFD) }
+} catch {
+  fail('execution_receipt', 'Cannot reserve the single-use probe allocation; no model process was started')
+} finally {
+  if (receiptFD !== undefined) fs.closeSync(receiptFD)
+}
+input.execution_id = path.basename(outputPath)
+input.max_http_sends = maxSends
+const child = spawn('docker', [...dockerPrefix, 'run', '--rm', '-i',
+  '-v', `${deployment}/src:/src:ro`, '-v', `${deployment}/go-cache:/go`, '-v', `${deployment}/go-build:/build-cache`,
+  '-w', '/src', '-e', 'CGO_ENABLED=1', '-e', 'GOCACHE=/build-cache',
+  '-e', `GRAPH_PROBE_MAX_SENDS=${maxSends}`,
+  'golang:1.26-bookworm', 'go', 'run', './scripts/graph_extraction_probe'], { stdio: ['pipe', 'pipe', 'pipe'] })
+child.stdin.end(JSON.stringify(input))
+input.aes_key = ''
+input.model.parameters.api_key = ''
+let output = ''
+child.stdout.on('data', data => { output += data })
+child.stderr.resume()
+child.on('error', () => fail('model_probe', 'Could not start read-only model probe'))
+child.on('close', code => {
+  const line = output.split('\n').find(value => value.startsWith('GRAPH_EXTRACTION_PROBE_RESULT='))
+  if (code !== 0 || !line) {
+    fail('model_probe', 'Graph extraction probe failed before a result; raw model output is withheld')
+  }
+  let report
+  try {
+    report = JSON.parse(line.slice('GRAPH_EXTRACTION_PROBE_RESULT='.length))
+  } catch {
+    fail('model_probe', 'Graph extraction probe returned an invalid report')
+  }
+  if (report.diagnostics_version !== 2 || report.execution_id !== input.execution_id
+    || report.http_send_limit !== maxSends || !Number.isInteger(report.http_send_reservations)
+    || report.http_send_reservations < 0 || report.http_send_reservations > maxSends
+    || !Number.isInteger(report.model_calls) || report.model_calls < 1 || report.model_calls > 2
+    || (report.passed && report.http_send_reservations === 0)) {
+    fail('model_probe_budget', 'Probe result has missing or invalid actual-send accounting; allocation remains spent')
+  }
+  report.model_id = input.model.id
+  report.source_content = input.chunk.content
+  if (chunkID === '1144acfb-18de-4687-8bf5-df0e9c8f1dd5') {
+    const relations = report.graph?.relation || []
+    const expected = [
+      ['过敏者', ['慎用于']],
+      ['粘膜破损处', ['禁用于', '不适用于', '不能用于']],
+      ['性生活中对性病的预防', ['禁用于', '不适用于', '不能用于']],
+    ]
+    report.semantic_checks = expected.map(([target, labels]) => ({
+      target,
+      passed: relations.some(relation => relation.node1 === '本品'
+        && relation.node2 === target && labels.includes(relation.type)),
+    }))
+    report.passed = report.passed && report.semantic_checks.every(check => check.passed)
+  }
+  let fd
+  try {
+    fd = fs.openSync(outputPath, fs.constants.O_WRONLY | fs.constants.O_CREAT | (reportTag ? fs.constants.O_EXCL : fs.constants.O_TRUNC) | fs.constants.O_NOFOLLOW, 0o600)
+    fs.fchmodSync(fd, 0o600)
+    fs.writeFileSync(fd, JSON.stringify(report, null, 2) + '\n')
+  } catch {
+    fail('report_write', 'Could not save the private probe report')
+  } finally {
+    if (fd !== undefined) fs.closeSync(fd)
+  }
+  console.log(JSON.stringify({
+    passed: report.passed, chunk_id: report.chunk_id, model_id: report.model_id,
+    model_name: report.model_name, source_sha256: report.source_sha256,
+    elapsed_ms: report.elapsed_ms, model_calls: report.model_calls,
+    http_send_reservations: report.http_send_reservations, http_send_limit: report.http_send_limit,
+    execution_id: report.execution_id, receipt_path: receiptPath,
+    relation_count: report.graph?.relation?.length || 0,
+    error_category: report.error ? (report.error.includes('graph_needs_review') ? 'graph_needs_review' : 'extraction_failed') : null,
+    production_write: false, report_path: outputPath,
+  }))
+  if (!report.passed) process.exitCode = 1
+})