WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

2026-09-22 16:19:06 +08:00
commit f7fc992f57
3929 changed files with 1006607 additions and 0 deletions
@@ -0,0 +1,235 @@
+const assert = require('node:assert/strict')
+const fs = require('node:fs')
+const path = require('node:path')
+const { execFileSync } = require('node:child_process')
+const { assertPublicVerification, publicVerificationReports, fixtureReconciliationCounts } = require('./publish_public_runtime.cjs')
+
+const root = path.resolve(__dirname, '../..')
+const tenantID = 10059
+const reportNames = publicVerificationReports
+const memoryTables = ['memory_item_embeddings', 'memory_items', 'memory_tombstones', 'memory_topic_stats', 'memory_doc_affinity', 'memory_subjects']
+const fixtureIDFields = ['users', 'roles', 'agents', 'skills', 'humans', 'kbs', 'workbench', 'docs', 'tags', 'sessions', 'messages', 'resources']
+const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
+const literal = value => `'${String(value).replaceAll("'", "''")}'`
+const idsSQL = values => values.length ? values.map(literal).join(',') : 'NULL'
+const completed = report => report.checked_at || report.time
+const runWindow = (alias, report) => `${alias}.created_at >= ${literal(report.started_at)}::timestamptz AND ${alias}.created_at <= ${literal(completed(report))}::timestamptz`
+const unique = values => [...new Set(values)]
+
+function validateIDs(value, name, minimum = 0) {
+  assert.ok(Array.isArray(value) && value.length >= minimum, `Missing fixture IDs: ${name}`)
+  value.forEach(id => assert.match(id, uuid, `Invalid fixture ID: ${name}`))
+  assert.equal(new Set(value).size, value.length, `Duplicate fixture IDs: ${name}`)
+  return value
+}
+
+function validateReports(manifest, reports, now = Date.now()) {
+  assert.match(manifest.release, /^public-\d{8}-\d{6}$/)
+  assert.equal(manifest.url, 'https://agent.pducv.cn')
+  for (const name of reportNames) {
+    assert.ok(reports[name], `Missing verification report: ${name}`)
+    assertPublicVerification(manifest, reports[name], name, now)
+    for (const field of fixtureIDFields) {
+      if (reports[name].fixtures?.[field] !== undefined) validateIDs(reports[name].fixtures[field], `${name}.${field}`)
+    }
+  }
+  const specifications = [
+    ['public-resource-isolation', 'resource-isolation', {users: 2, roles: 1, agents: 2, skills: 1, kbs: 1}],
+    ['public-resource-files', 'file-isolation', {users: 2, roles: 1, kbs: 2, docs: 1, sessions: 1, messages: 2, resources: 3}],
+    ['public-indexed-retrieval-isolation', 'indexed-isolation', {users: 2, roles: 1, kbs: 1, docs: 1, tags: 2}],
+  ]
+  for (const [name, prefix, fields] of specifications) {
+    const report = reports[name]
+    assert.match(report.prefix, new RegExp(`^${prefix}-[0-9]+$`))
+    for (const [field, minimum] of Object.entries(fields)) validateIDs(report.fixtures?.[field], `${name}.${field}`, minimum)
+  }
+  const resource = reports['public-resource-isolation']
+  assert.deepEqual(resource.fixtures.memory_subjects, resource.fixtures.users.map(id => `web_user:${id}`))
+  const file = reports['public-resource-files']
+  for (const [field, count] of Object.entries({users: 2, sessions: 1, messages: 2, resources: 3})) assert.equal(file.fixtures[field].length, count, `Unexpected file fixture count: ${field}`)
+  assert.equal(file.fixtures.message_scopes?.length, 1)
+  const scope = file.fixtures.message_scopes[0]
+  assert.equal(scope.tenant_id, tenantID)
+  assert.equal(scope.session_id, file.fixtures.sessions[0])
+  assert.equal(scope.user_id, file.fixtures.users[0])
+  assert.equal(scope.title, file.prefix)
+  assert.equal(scope.username, `${file.prefix}-alice`)
+  assert.equal(scope.email, `${scope.username}@example.test`)
+  assert.ok(Array.isArray(scope.messages), 'Missing file fixture messages')
+  assert.deepEqual(scope.messages.map(message => message.id), file.fixtures.messages)
+  assert.deepEqual(scope.messages.map(message => message.content), ['Disposable attachment fixture.', 'Unrelated message.'])
+  const tenant = reports['public-tenant-isolation']
+  assert.match(tenant.prefix, /^tenant-isolation-[0-9]+$/)
+  assert.ok(Number.isSafeInteger(tenant.second_tenant) && tenant.second_tenant > tenantID)
+  const access = reports['public-access-control']
+  if (access.prefix !== undefined) assert.match(access.prefix, /^access-check-[0-9]+$/)
+}
+
+function readOnlySQL(query) {
+  assert.match(query.trimStart(), /^(SELECT|WITH)\b/)
+  return `BEGIN READ ONLY;\nSET LOCAL statement_timeout='15s';\n${query}\nROLLBACK;`
+}
+
+function executeReadOnly(query, run = execFileSync) {
+  const output = run('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres',
+    'psql', '-X', '-q', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-At', '-v', 'ON_ERROR_STOP=1'], {
+    input: readOnlySQL(query), encoding: 'utf8', timeout: 45000, maxBuffer: 4 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'],
+  }).trim()
+  return JSON.parse(output)
+}
+
+function identityQuery(reports) {
+  const clauses = []
+  for (const name of ['public-resource-isolation', 'public-resource-files', 'public-indexed-retrieval-isolation']) {
+    clauses.push(`(u.id IN (${idsSQL(reports[name].fixtures.users)}))`)
+  }
+  const tenant = reports['public-tenant-isolation']
+  clauses.push(`(${runWindow('u', tenant)} AND u.username IN (${literal(tenant.prefix + '-alice')},${literal(tenant.prefix + '-bob')}) AND u.email=u.username || '@example.test')`)
+  const access = reports['public-access-control']
+  const username = access.prefix ? `u.username=${literal(access.prefix)}` : "u.username ~ '^access-check-[0-9]+$'"
+  clauses.push(`(${runWindow('u', access)} AND u.tenant_id=${tenantID} AND ${username} AND u.email IN (u.username || '@example.test','updated-' || u.username || '@example.test'))`)
+  return `SELECT COALESCE(json_agg(json_build_object('id',u.id,'username',u.username,'email',u.email,'tenant_id',u.tenant_id,'created_at',u.created_at)), '[]'::json) FROM users u WHERE ${clauses.join(' OR ')};`
+}
+
+function resolveIdentities(reports, rows) {
+  assert.ok(Array.isArray(rows), 'User discovery did not return an array')
+  const groups = [], used = new Set()
+  const matchUser = (report, username, id, tenants, updatedEmail = false) => {
+    const found = rows.filter(row => row.username === username && (!id || row.id === id))
+    assert.equal(found.length, 1, `Expected one retained fixture identity: ${username}`)
+    const row = found[0]
+    assert.match(row.id, uuid)
+    assert.ok(tenants.includes(Number(row.tenant_id)), `Unexpected fixture tenant: ${username}`)
+    assert.ok([`${username}@example.test`, ...(updatedEmail ? [`updated-${username}@example.test`] : [])].includes(row.email), `Unexpected fixture email: ${username}`)
+    assert.ok(Date.parse(row.created_at) >= Date.parse(report.started_at) && Date.parse(row.created_at) <= Date.parse(completed(report)), `Fixture identity outside report window: ${username}`)
+    assert.ok(!used.has(row.id), `Fixture ID reused across reports: ${row.id}`)
+    used.add(row.id)
+    return row.id
+  }
+  for (const [name, labels] of [
+    ['public-resource-isolation', ['alice', 'bob']], ['public-resource-files', ['alice', 'bob']],
+    ['public-indexed-retrieval-isolation', ['owner', 'reader']],
+  ]) {
+    const report = reports[name]
+    assert.equal(report.fixtures.users.length, labels.length)
+    groups.push({name, report, prefix: report.prefix, users: labels.map((label, index) => matchUser(report, `${report.prefix}-${label}`, report.fixtures.users[index], [tenantID]))})
+  }
+  const tenant = reports['public-tenant-isolation']
+  groups.push({name: 'public-tenant-isolation', report: tenant, prefix: tenant.prefix, users: ['alice', 'bob'].map(label => matchUser(tenant, `${tenant.prefix}-${label}`, undefined, [tenantID, tenant.second_tenant]))})
+  const access = reports['public-access-control']
+  const candidates = rows.filter(row => /^access-check-[0-9]+$/.test(row.username) && (!access.prefix || access.prefix === row.username))
+  assert.equal(candidates.length, 1, 'Expected one access-control fixture account in its exact report window')
+  const accessPrefix = candidates[0].username
+  groups.push({name: 'public-access-control', report: access, prefix: accessPrefix, users: [matchUser(access, accessPrefix, undefined, [tenantID], true)]})
+  assert.equal(used.size, rows.length, 'Unexpected identity returned by fixture discovery')
+  return groups
+}
+
+function reconciliationQuery(reports, groups) {
+  const allUsers = unique(groups.flatMap(group => group.users))
+  const reported = key => unique(reportNames.flatMap(name => reports[name].fixtures?.[key] || []))
+  const targetName = (alias, field, group) => `${alias}.${field} ~ ${literal(`^${group.prefix}($|-(invalid|agent|admin-agent|member-agent|human|knowledge|skill|flow|admin-edited|member-flow|shared-flow-task|blocked|viewer-blocked|forbidden|document|alice|bob)$)`)}`
+  const discover = (alias, owner, field = 'name') => groups.map(group => `(${runWindow(alias, group.report)} AND (${owner ? `${alias}.${owner} IN (${idsSQL(group.users)}) OR ` : ''}${targetName(alias, field, group)}))`).join(' OR ')
+  const scoped = (table, key, owner, field = 'name') => `SELECT t.id FROM ${table} t WHERE t.tenant_id=${tenantID} AND (t.id IN (${idsSQL(reported(key))}) OR ${discover('t', owner, field)})`
+  const eventScope = groups.map(group => `(${runWindow('e', group.report)} AND ${targetName('e', 'message', {...group, prefix: '创建' + group.prefix})})`).join(' OR ')
+  const ctes = [
+    `fixture_roles AS (${scoped('workbench_access_roles', 'roles', null)})`,
+    `fixture_agents AS (${scoped('custom_agents', 'agents', 'created_by')})`,
+    `fixture_skills AS (${scoped('tenant_skill_catalog', 'skills', 'created_by')})`,
+    `fixture_humans AS (${scoped('digital_humans', 'humans', 'owner_id')})`,
+    `fixture_kbs AS (${scoped('knowledge_bases', 'kbs', 'creator_id')})`,
+    `fixture_workbench AS (${scoped('workbench_resources', 'workbench', 'owner_id')} UNION SELECT e.resource_id FROM workbench_events e WHERE e.tenant_id=${tenantID} AND e.action='create' AND (${eventScope}))`,
+    `fixture_sessions AS (${scoped('sessions', 'sessions', 'user_id', 'title')})`,
+    `fixture_docs AS (SELECT id FROM knowledges WHERE tenant_id=${tenantID} AND (id IN (${idsSQL(reported('docs'))}) OR knowledge_base_id IN (SELECT id FROM fixture_kbs)))`,
+    `fixture_tags AS (SELECT id FROM knowledge_tags WHERE tenant_id=${tenantID} AND (id IN (${idsSQL(reported('tags'))}) OR knowledge_base_id IN (SELECT id FROM fixture_kbs)))`,
+    `fixture_resource_ids AS (SELECT id FROM fixture_agents UNION SELECT id FROM fixture_skills UNION SELECT id FROM fixture_humans UNION SELECT id FROM fixture_kbs UNION SELECT id FROM fixture_workbench)`,
+  ]
+  // Include recorded IDs even when their resource row was physically removed.
+  for (const [cte, key] of [['fixture_roles', 'roles'], ['fixture_agents', 'agents'], ['fixture_skills', 'skills'], ['fixture_humans', 'humans'], ['fixture_kbs', 'kbs'], ['fixture_workbench', 'workbench'], ['fixture_docs', 'docs'], ['fixture_tags', 'tags'], ['fixture_sessions', 'sessions']]) {
+    const index = ctes.findIndex(value => value.startsWith(`${cte} AS (`))
+    const ids = reported(key)
+    if (ids.length) ctes[index] = ctes[index].slice(0, -1) + ` UNION SELECT unnest(ARRAY[${idsSQL(ids)}]::text[]))`
+  }
+  const inUsers = `IN (${idsSQL(allUsers)})`
+  const fileIDs = idsSQL(reported('resources')), messageIDs = idsSQL(reported('messages'))
+  const membershipTenants = `${tenantID},${reports['public-tenant-isolation'].second_tenant}`
+  const counts = {
+    active_users: `SELECT count(*) FROM users WHERE id ${inUsers} AND deleted_at IS NULL`,
+    active_memberships: `SELECT count(*) FROM tenant_members WHERE user_id ${inUsers} AND deleted_at IS NULL`,
+    roles: 'SELECT count(*) FROM workbench_access_roles WHERE id IN (SELECT id FROM fixture_roles)',
+    assignments: `SELECT count(*) FROM workbench_access_assignments WHERE user_id ${inUsers} OR role_id IN (SELECT id FROM fixture_roles)`,
+    grants: `SELECT count(*) FROM workbench_access_grants WHERE user_id ${inUsers} OR (tenant_id=${tenantID} AND resource_id IN (SELECT id FROM fixture_resource_ids))`,
+    unrevoked_tokens: `SELECT count(*) FROM auth_tokens WHERE user_id ${inUsers} AND is_revoked=false`,
+    active_agents: 'SELECT count(*) FROM custom_agents WHERE id IN (SELECT id FROM fixture_agents) AND deleted_at IS NULL',
+    active_skills: 'SELECT count(*) FROM tenant_skill_catalog WHERE id IN (SELECT id FROM fixture_skills) AND deleted_at IS NULL',
+    active_digital_humans: 'SELECT count(*) FROM digital_humans WHERE id IN (SELECT id FROM fixture_humans) AND deleted_at IS NULL',
+    workbench_resources: 'SELECT count(*) FROM workbench_resources WHERE id IN (SELECT id FROM fixture_workbench)',
+    active_sessions: 'SELECT count(*) FROM sessions WHERE id IN (SELECT id FROM fixture_sessions) AND deleted_at IS NULL',
+    synthetic_messages: `SELECT count(*) FROM messages WHERE id IN (${messageIDs})`,
+    file_resources: `SELECT count(*) FROM resources WHERE id IN (${fileIDs})`,
+    file_bindings: `SELECT count(*) FROM resource_bindings WHERE resource_id IN (${fileIDs})`,
+    file_access_grants: `SELECT count(*) FROM resource_access_grants WHERE resource_id IN (${fileIDs}) AND revoked_at IS NULL`,
+    active_knowledge_bases: 'SELECT count(*) FROM knowledge_bases WHERE id IN (SELECT id FROM fixture_kbs) AND deleted_at IS NULL',
+    active_documents: 'SELECT count(*) FROM knowledges WHERE id IN (SELECT id FROM fixture_docs) AND deleted_at IS NULL',
+    active_chunks: `SELECT count(*) FROM chunks WHERE tenant_id=${tenantID} AND (knowledge_base_id IN (SELECT id FROM fixture_kbs) OR knowledge_id IN (SELECT id FROM fixture_docs)) AND deleted_at IS NULL`,
+    tags: 'SELECT count(*) FROM knowledge_tags WHERE id IN (SELECT id FROM fixture_tags)',
+    tag_relations: 'SELECT count(*) FROM knowledge_tag_relations WHERE tag_id IN (SELECT id FROM fixture_tags) OR knowledge_id IN (SELECT id FROM fixture_docs)',
+    publications: `SELECT count(*) FROM workbench_access_publications WHERE tenant_id=${tenantID} AND resource_id IN (SELECT id FROM fixture_resource_ids)`,
+    secondary_tenant: `SELECT count(*) FROM tenants WHERE id=${reports['public-tenant-isolation'].second_tenant}`,
+    secondary_memberships: `SELECT count(*) FROM tenant_members WHERE tenant_id=${reports['public-tenant-isolation'].second_tenant} AND deleted_at IS NULL`,
+  }
+  for (const table of memoryTables) counts[table] = `SELECT count(*) FROM ${table} WHERE tenant_id IN (${membershipTenants}) AND subject_id IN (${idsSQL(allUsers.map(id => `web_user:${id}`))})`
+  assert.deepEqual(Object.keys(counts), fixtureReconciliationCounts, 'Reconciliation queries must cover the publication contract')
+  const retained = {
+    soft_deleted_users: `SELECT count(*) FROM users WHERE id ${inUsers} AND deleted_at IS NOT NULL`,
+    revoked_tokens: `SELECT count(*) FROM auth_tokens WHERE user_id ${inUsers} AND is_revoked=true`,
+    soft_deleted_sessions: 'SELECT count(*) FROM sessions WHERE id IN (SELECT id FROM fixture_sessions) AND deleted_at IS NOT NULL',
+    soft_deleted_documents: 'SELECT count(*) FROM knowledges WHERE id IN (SELECT id FROM fixture_docs) AND deleted_at IS NOT NULL',
+    soft_deleted_chunks: `SELECT count(*) FROM chunks WHERE tenant_id=${tenantID} AND knowledge_base_id IN (SELECT id FROM fixture_kbs) AND deleted_at IS NOT NULL`,
+    workbench_revisions: `SELECT count(*) FROM workbench_revisions WHERE tenant_id=${tenantID} AND resource_id IN (SELECT id FROM fixture_workbench)`,
+  }
+  const objectSQL = values => `json_build_object(${Object.entries(values).map(([name, query]) => `${literal(name)},(${query})`).join(',')})`
+  const discovered = Object.fromEntries(['roles', 'agents', 'skills', 'humans', 'kbs', 'workbench', 'sessions', 'docs', 'tags'].map(key => [key, `SELECT COALESCE(json_agg(id), '[]'::json) FROM fixture_${key}`]))
+  return {query: `WITH ${ctes.join(',\n')}\nSELECT json_build_object('counts',${objectSQL(counts)},'retained',${objectSQL(retained)},'discovered',${objectSQL(discovered)});`, countNames: Object.keys(counts)}
+}
+
+function reconcile({manifest, reports, executeSQL = executeReadOnly, exists = fs.existsSync, directory = root, now = Date.now()}) {
+  validateReports(manifest, reports, now)
+  const identities = executeSQL(identityQuery(reports))
+  const groups = resolveIdentities(reports, identities)
+  const {query, countNames} = reconciliationQuery(reports, groups)
+  const database = executeSQL(query)
+  const checks = countNames.map(name => {
+    const count = database.counts?.[name]
+    assert.ok(Number.isSafeInteger(count) && count >= 0, `Invalid reconciliation count: ${name}`)
+    return {name, count, passed: count === 0}
+  })
+  const fileDirectory = path.join(directory, 'acceptance', 'files', String(tenantID), 'exports', reports['public-resource-files'].prefix)
+  checks.push({name: 'file_fixture_directory_removed', path: fileDirectory, passed: !exists(fileDirectory)})
+  return {release: manifest.release, origin: manifest.url, checked_at: new Date(now).toISOString(), read_only: true,
+    reports: Object.fromEntries(reportNames.map(name => [name, {started_at: reports[name].started_at, completed_at: completed(reports[name])}])),
+    identities, discovered: database.discovered, retained: database.retained, checks,
+    failures: checks.filter(check => !check.passed).map(check => check.name), passed: checks.every(check => check.passed),
+    scope: 'PostgreSQL fixture records and exact file-test export directory. Soft-deleted history, revoked tokens and audits are retained. Milvus zero-vector cleanup is attested by the fresh indexed-retrieval report, not queried again here.'}
+}
+
+function main() {
+  const release = process.argv[2]
+  assert.match(release || '', /^public-\d{8}-\d{6}$/, 'Usage: node scripts/verify_public_fixture_reconciliation.cjs public-YYYYMMDD-HHMMSS')
+  const reportPath = path.join(root, 'acceptance', 'public-fixture-reconciliation.json')
+  // Invalidate a previous success before reading reports or contacting the database.
+  fs.writeFileSync(reportPath, JSON.stringify({release, started_at: new Date().toISOString(), read_only: true,
+    passed: false, checks: [], failures: ['Reconciliation has not completed.']}, null, 2) + '\n', {mode: 0o600})
+  const manifest = JSON.parse(fs.readFileSync(path.join(root, 'releases', release, 'manifest.json'), 'utf8'))
+  assert.equal(manifest.release, release)
+  const reports = Object.fromEntries(reportNames.map(name => [name, JSON.parse(fs.readFileSync(path.join(root, 'acceptance', `${name}.json`), 'utf8'))]))
+  const report = reconcile({manifest, reports})
+  fs.writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n', {mode: 0o600})
+  console.log(JSON.stringify({release, passed: report.passed, checks: report.checks.length, failures: report.failures, retained: report.retained}))
+  if (!report.passed) process.exitCode = 1
+}
+
+module.exports = {reportNames, memoryTables, validateReports, readOnlySQL, executeReadOnly, identityQuery, resolveIdentities, reconciliationQuery, reconcile}
+if (require.main === module) {
+  try { main() } catch (error) { console.error(error.message); process.exitCode = 1 }
+}