xinmi/scripts/map_native_graph_evidence.mjs

#!/usr/bin/env node
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';

const database = process.env.GRAPH_MIGRATION_DATABASE || 'yawan_native_acceptance_20260915';
if (!/^yawan_native_acceptance_[0-9]+$/.test(database)) throw new Error('Only isolated acceptance databases are allowed');
if (!process.argv.includes('--apply')) throw new Error('Usage: node scripts/map_native_graph_evidence.mjs --apply');
const tenant = Number(process.env.GRAPH_MIGRATION_TENANT || 10059);
if (!Number.isSafeInteger(tenant) || tenant < 1) throw new Error('Invalid tenant');
const base = ['--context','colima-yawan','exec','-i','WeKnora-postgres','psql','-X','-v','ON_ERROR_STOP=1','-U','postgres','-d',database,'-At'];
const quote = value => `'${String(value).replaceAll("'", "''")}'`;
function sql(query) {
  const result = spawnSync('docker',base,{input:query,encoding:'utf8',maxBuffer:256*1024*1024});
  if(result.status!==0) throw new Error(result.stderr);
  return result.stdout.trim().split('\n').filter(Boolean);
}
const normalize = value => value.normalize('NFKC').replace(/\s+/gu,'');
const documents = sql(`WITH candidates AS (SELECT k.id,k.knowledge_base_id,s.source_file,count(*) OVER (PARTITION BY s.knowledge_base_id,s.source_file) AS matches FROM (SELECT DISTINCT knowledge_base_id,source_file FROM native_graph_sources WHERE tenant_id=${tenant}) s JOIN knowledges k ON k.tenant_id=${tenant} AND k.knowledge_base_id=s.knowledge_base_id AND k.file_name=regexp_replace(s.source_file,'\\.jsonl$','.md') AND k.deleted_at IS NULL) SELECT json_build_object('id',id,'kb',knowledge_base_id,'file',source_file) FROM candidates WHERE matches=1 ORDER BY id;`).map(JSON.parse);
let mapped = 0, ambiguous = 0;
for (const document of documents) {
  const rows=sql(`SELECT json_build_object('id',source_chunk_id,'text',excerpt) FROM native_graph_sources WHERE tenant_id=${tenant} AND knowledge_base_id=${quote(document.kb)} AND source_file=${quote(document.file)} AND native_chunk_id IS NULL;`).map(JSON.parse);
  const prefixes=new Map();
  for (const row of rows) {
    row.normalized=normalize(row.text);row.matches=[];
    if([...row.normalized].length<80)continue;
    const key=row.normalized.slice(0,16);
    if(!prefixes.has(key))prefixes.set(key,[]);
    prefixes.get(key).push(row);
  }
  if(!prefixes.size)continue;
  const chunks=sql(`SELECT json_build_object('id',id,'content',content) FROM chunks WHERE tenant_id=${tenant} AND knowledge_base_id=${quote(document.kb)} AND knowledge_id=${quote(document.id)} AND chunk_type='text' AND deleted_at IS NULL AND is_enabled=true ORDER BY id;`).map(JSON.parse);
  for(const chunk of chunks) {
    const content=normalize(chunk.content);
    const seen=new Set();
    for(let pos=0;pos<=content.length-16;pos++) {
      const candidates=prefixes.get(content.slice(pos,pos+16));
      if(!candidates)continue;
      for(const row of candidates) {
        if(seen.has(row.id)||row.matches.length>1)continue;
        if(content.startsWith(row.normalized,pos)) {
          row.matches.push({id:chunk.id,hash:createHash('sha256').update(chunk.content).digest('hex')});seen.add(row.id);
        }
      }
    }
  }
  const matches=rows.filter(row=>row.matches.length===1);
  ambiguous+=rows.filter(row=>row.matches.length>1).length;
  if(matches.length) {
    const values=matches.map(row=>`(${quote(row.id)},${quote(row.matches[0].id)},${quote(row.matches[0].hash)})`).join(',\n');
    sql(`BEGIN; UPDATE native_graph_sources s SET native_knowledge_id=${quote(document.id)},native_chunk_id=m.chunk_id,native_content_sha256=m.hash,mapping_state='normalized_full_excerpt' FROM (VALUES ${values}) m(source_id,chunk_id,hash) WHERE s.tenant_id=${tenant} AND s.knowledge_base_id=${quote(document.kb)} AND s.source_file=${quote(document.file)} AND s.source_chunk_id=m.source_id AND s.native_chunk_id IS NULL; COMMIT;`);
  }
  mapped+=matches.length;
  console.log(JSON.stringify({file:document.file,candidates:rows.length,mapped:matches.length}));
}
console.log(JSON.stringify({mapped,ambiguous,normalization:'Unicode NFKC and whitespace removal; entire excerpt >= 80 codepoints; unique native chunk in matching tenant, KB and source document'}));