WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

2026-09-22 16:19:06 +08:00
commit f7fc992f57
3929 changed files with 1006607 additions and 0 deletions
@@ -0,0 +1,278 @@
+#!/usr/bin/env node
+'use strict'
+
+const assert = require('node:assert/strict')
+const crypto = require('node:crypto')
+const fs = require('node:fs')
+const path = require('node:path')
+const { execFileSync, spawn } = require('node:child_process')
+
+const acceptance = process.env.MEMORY_SMOKE_DIRECTORY || '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
+const database = {
+  context: process.env.MEMORY_SMOKE_DOCKER_CONTEXT || 'colima-yawan',
+  container: process.env.MEMORY_SMOKE_DB_CONTAINER || 'WeKnora-postgres',
+  name: process.env.MEMORY_SMOKE_DB_NAME || 'yawan_native_acceptance_20260915',
+}
+const literal = value => `'${String(value).replaceAll("'", "''")}'`
+let manifest, manifestPath, admin
+
+function privateJSON(file, value) {
+  const temporary = `${file}.${crypto.randomBytes(4).toString('hex')}.tmp`
+  const fd = fs.openSync(temporary, 'wx', 0o600)
+  try {
+    fs.writeFileSync(fd, `${JSON.stringify(value, null, 2)}\n`)
+    fs.fsyncSync(fd)
+  } finally { fs.closeSync(fd) }
+  fs.renameSync(temporary, file)
+}
+const persist = () => privateJSON(manifestPath, manifest)
+function sql(query) {
+  try {
+    return execFileSync(process.env.MEMORY_SMOKE_DOCKER || '/opt/homebrew/bin/docker', [
+      '--context', database.context, 'exec', '-i', database.container,
+      'psql', '-X', '-q', '-U', 'postgres', '-d', database.name,
+      '-At', '-v', 'ON_ERROR_STOP=1',
+    ], { input: query, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 30000 }).trim()
+  } catch {
+    // SQL stderr may contain request fields. Keep credentials out of reports.
+    throw new Error('Fixture database operation failed; inspect the bounded fixture SQL and database locally')
+  }
+}
+async function api(route, body, identity = admin, tenant = manifest.source_tenant_id) {
+  const response = await fetch(manifest.base + route, {
+    method: body === undefined ? 'GET' : 'POST',
+    headers: { 'Content-Type': 'application/json', ...(identity ? { Authorization: `Bearer ${identity.token}`, 'X-Tenant-ID': String(tenant) } : {}) },
+    body: body === undefined ? undefined : JSON.stringify(body),
+    signal: AbortSignal.timeout(15000),
+  })
+  assert.ok(response.ok, `${route}: HTTP ${response.status}`)
+  return response.json()
+}
+function validateManifest() {
+  assert.equal(manifest.format, 'native-memory-fixture-v1')
+  assert.match(manifest.run, /^memory-smoke-[a-f0-9]{16}$/)
+  assert.deepEqual(manifest.database, database, 'Use the same database connection as the fixture manifest')
+  assert.ok(Number.isSafeInteger(manifest.source_tenant_id) && manifest.source_tenant_id > 0)
+  if (manifest.tenant_id !== undefined) {
+    assert.ok(Number.isSafeInteger(manifest.tenant_id) && manifest.tenant_id > 0 && manifest.tenant_id !== manifest.source_tenant_id)
+  }
+  assert.equal(manifest.users.length, 2)
+  for (const [index, label] of ['owner', 'outsider'].entries()) {
+    assert.equal(manifest.users[index].username, `${manifest.run}-${label}`)
+    assert.equal(manifest.users[index].email, `${manifest.run}-${label}@example.test`)
+  }
+  const url = new URL(manifest.base)
+  assert.ok(['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname), 'Only loopback API origins are permitted')
+}
+
+async function provision() {
+  const adminFile = process.env.MEMORY_SMOKE_ADMIN_AUTH || path.join(acceptance, 'super-admin-login.json')
+  assert.equal(fs.statSync(adminFile).mode & 0o077, 0, 'Admin credentials must be a private file (mode 0600)')
+  const credentials = JSON.parse(fs.readFileSync(adminFile, 'utf8'))
+  admin = await api('/auth/login', { email: credentials.email, password: credentials.password }, null)
+  assert.equal(typeof admin.token, 'string')
+  // The random name and business marker allow exact recovery after interruption.
+  manifest.tenant_id = Number(sql(`INSERT INTO tenants(name,business,status,memory_config)
+    VALUES (${literal(manifest.run)},'native-memory-smoke-fixture','active',
+      '{"enabled":true,"write_mode":"explicit_only","vector_recall":false,"embedding_model_id":""}') RETURNING id;`))
+  assert.ok(Number.isSafeInteger(manifest.tenant_id) && manifest.tenant_id > 0 && manifest.tenant_id !== manifest.source_tenant_id)
+  persist()
+  for (const user of manifest.users) {
+    const password = crypto.randomBytes(18).toString('base64url')
+    const row = (await api('/access/users', { username: user.username, email: user.email, password, role: 'contributor', status: 'active' })).data
+    user.id = row.id
+    persist()
+    const role = user === manifest.users[0] ? 'owner' : 'contributor'
+    sql(`BEGIN;
+      UPDATE users SET tenant_id=${manifest.tenant_id} WHERE id=${literal(user.id)} AND email=${literal(user.email)} AND tenant_id=${manifest.source_tenant_id};
+      UPDATE tenant_members SET tenant_id=${manifest.tenant_id},role=${literal(role)} WHERE user_id=${literal(user.id)} AND tenant_id=${manifest.source_tenant_id};
+      DELETE FROM workbench_access_assignments WHERE user_id=${literal(user.id)} AND tenant_id=${manifest.source_tenant_id};
+      COMMIT;`)
+    const identity = await api('/auth/login', { email: user.email, password }, null)
+    assert.equal(Number(identity.user.tenant_id), manifest.tenant_id, 'Fixture login selected a different workspace')
+    assert.equal(identity.user.is_system_admin, false, 'Fixture user must not be a system administrator')
+    privateJSON(path.join(path.dirname(manifestPath), `${user === manifest.users[0] ? 'owner' : 'outsider'}-auth.json`), { token: identity.token, tenantID: manifest.tenant_id })
+    if (role === 'owner') {
+      await api('/workbench/operations/grants', { user_id: user.id, amount: 1_000_000_000, reason: manifest.run, idempotency_key: manifest.run }, identity, manifest.tenant_id)
+    }
+  }
+  manifest.provisioned_at = new Date().toISOString()
+  persist()
+}
+
+function runSmoke() {
+  const directory = path.dirname(manifestPath)
+  return new Promise((resolve, reject) => {
+    const child = spawn(process.execPath, [path.join(__dirname, 'native_memory_api_smoke.cjs'), '--run'], {
+      env: { ...process.env, MEMORY_SMOKE_BASE: manifest.base,
+        MEMORY_SMOKE_RUN: manifest.run, MEMORY_SMOKE_LEGACY: '1',
+        MEMORY_SMOKE_OWNER_AUTH: path.join(directory, 'owner-auth.json'),
+        MEMORY_SMOKE_OUTSIDER_AUTH: path.join(directory, 'outsider-auth.json'),
+        MEMORY_SMOKE_MANIFEST: path.join(directory, 'smoke.json') },
+      stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
+    })
+    manifest.smoke_pid = child.pid
+    persist()
+    const timer = setTimeout(() => child.kill('SIGTERM'), 300000)
+    child.on('message', message => {
+      if (message?.type !== 'seed-legacy-memory') return
+      try {
+        validateManifest()
+        assert.equal(message.run, manifest.run)
+        assert.match(message.agent_id, /^[a-f0-9-]{36}$/)
+        assert.ok(!manifest.legacy_candidate, 'Only one synthetic legacy candidate is permitted')
+        const ownerID = manifest.users[0].id
+        const id = crypto.randomUUID(), historyID = crypto.randomUUID()
+        const original = `${manifest.run}: original legacy advice`
+        const source = `${manifest.run}/legacy/${historyID}`
+        // Seed only a synthetic accepted history row in this run's empty workspace.
+        // Migration provenance matching itself is verified by PostgreSQL unit tests.
+        sql(`BEGIN;
+          DO $seed$ BEGIN
+            IF NOT EXISTS (SELECT 1 FROM tenants WHERE id=${manifest.tenant_id} AND name=${literal(manifest.run)} AND business='native-memory-smoke-fixture') OR
+               NOT EXISTS (SELECT 1 FROM custom_agents WHERE id=${literal(message.agent_id)} AND tenant_id=${manifest.tenant_id} AND created_by=${literal(ownerID)} AND name=${literal(manifest.run)} AND deleted_at IS NULL AND config->>'memory_enabled'='true')
+            THEN RAISE EXCEPTION 'Legacy fixture scope mismatch'; END IF;
+          END $seed$;
+          INSERT INTO workbench_history(id,tenant_id,owner_id,source_kind,source_id,payload)
+          VALUES(${literal(historyID)},${manifest.tenant_id},${literal(ownerID)},'pendingmemory',${literal(id)},${literal(JSON.stringify({ fixture: manifest.run, content: original }))}::jsonb);
+          INSERT INTO workbench_pending_memories(id,tenant_id,owner_id,agent_id,content,source_ref,status,reviewed_by,legacy_history_id)
+          VALUES(${literal(id)},${manifest.tenant_id},${literal(ownerID)},${literal(message.agent_id)},${literal(original)},${literal(source)},'accepted',${literal(ownerID)},${literal(historyID)});
+          COMMIT;`)
+        manifest.legacy_candidate = { id, history_id: historyID, agent_id: message.agent_id }
+        persist()
+        child.send({ type: 'legacy-memory-seeded', id, history_id: historyID })
+      } catch (error) {
+        manifest.legacy_seed_error = error.message
+        persist()
+        if (child.connected) child.send({ type: 'legacy-memory-seeded', error: error.message })
+      }
+    })
+    child.once('error', error => { clearTimeout(timer); reject(error) })
+    child.once('close', (code, signal) => {
+      clearTimeout(timer)
+      manifest.smoke_exit_code = code
+      manifest.smoke_signal = signal
+      manifest.smoke_finished_at = new Date().toISOString()
+      manifest.passed = code === 0
+      persist()
+      resolve()
+    })
+  })
+}
+
+function cleanup() {
+  validateManifest()
+  if (manifest.smoke_pid && !manifest.smoke_finished_at) {
+    let running = true
+    try { process.kill(manifest.smoke_pid, 0) } catch (error) {
+      if (error.code === 'ESRCH') running = false
+    }
+    assert.equal(running, false, 'Fixture smoke process is still alive; wait for it to finish before cleanup')
+  }
+  const discovered = sql(`SELECT id FROM tenants WHERE name=${literal(manifest.run)} AND business='native-memory-smoke-fixture';`)
+  if (discovered) {
+    const tenant = Number(discovered)
+    assert.ok(Number.isSafeInteger(tenant) && tenant > 0 && tenant !== manifest.source_tenant_id)
+    if (manifest.tenant_id) assert.equal(tenant, manifest.tenant_id)
+    manifest.tenant_id = tenant
+  }
+  const tenant = manifest.tenant_id || 0
+  const identities = manifest.users.map(user => `(username=${literal(user.username)} AND email=${literal(user.email)})`).join(' OR ')
+  const rows = JSON.parse(sql(`SELECT COALESCE(json_agg(json_build_object('id',id,'username',username,'tenant_id',tenant_id)),'[]') FROM users WHERE ${identities};`))
+  for (const row of rows) {
+    const user = manifest.users.find(value => value.username === row.username)
+    if (user.id) assert.equal(user.id, row.id)
+    assert.ok([manifest.source_tenant_id, tenant].includes(Number(row.tenant_id)), 'Fixture user moved to an unexpected workspace')
+    user.id = row.id
+  }
+  persist()
+  const userIDs = manifest.users.filter(user => user.id).map(user => literal(user.id)).join(',') || "''"
+  // Unknown nonempty tables stop cleanup. This prevents silently orphaning new
+  // fixture dependencies when future API changes add another storage surface.
+  const tables = [
+    'workbench_pending_memories', 'workbench_revisions', 'workbench_events', 'workbench_jobs',
+    'workbench_resources', 'workbench_balance_holds', 'workbench_usage', 'workbench_transactions',
+    'workbench_balances', 'workbench_operation_logs', 'workbench_consumption', 'workbench_history',
+    'workbench_usage_quotas', 'workbench_resource_forks', 'workbench_resource_submissions',
+    'workbench_resource_hidden', 'workbench_resource_version_history',
+    'workbench_access_grants', 'workbench_access_publications', 'workbench_access_assignments', 'workbench_access_roles',
+    'memory_item_embeddings', 'memory_tombstones', 'memory_items', 'memory_topic_stats', 'memory_doc_affinity', 'memory_subjects',
+    'custom_agents', 'models', 'audit_logs', 'tenant_members',
+  ]
+  const tableList = tables.map(literal).join(',')
+  const query = `BEGIN;
+    SET LOCAL lock_timeout = '3s';
+    SET LOCAL statement_timeout = '20s';
+    DO $fixture$
+    DECLARE entry record; count_rows bigint;
+    BEGIN
+      IF ${tenant} <> 0 AND EXISTS (SELECT 1 FROM tenants WHERE id=${tenant} AND (name<>${literal(manifest.run)} OR business<>'native-memory-smoke-fixture')) THEN RAISE EXCEPTION 'Fixture tenant identity mismatch'; END IF;
+      IF EXISTS (SELECT 1 FROM users WHERE id IN (${userIDs}) AND (is_system_admin OR can_access_all_tenants OR NOT (${identities}))) THEN RAISE EXCEPTION 'Unexpected user identity'; END IF;
+      IF ${tenant} > 0 AND (EXISTS (SELECT 1 FROM users WHERE tenant_id=${tenant} AND id NOT IN (${userIDs})) OR EXISTS (SELECT 1 FROM tenant_members WHERE tenant_id=${tenant} AND user_id NOT IN (${userIDs}))) THEN RAISE EXCEPTION 'Fixture has other users'; END IF;
+      IF EXISTS (SELECT 1 FROM tenant_members WHERE user_id IN (${userIDs}) AND tenant_id NOT IN (${tenant},${manifest.source_tenant_id})) THEN RAISE EXCEPTION 'Fixture user has foreign memberships'; END IF;
+      IF ${tenant} > 0 AND EXISTS (SELECT 1 FROM workbench_jobs WHERE tenant_id=${tenant} AND status IN ('queued','running')) THEN RAISE EXCEPTION 'Fixture jobs are still active'; END IF;
+      FOR entry IN SELECT table_name FROM information_schema.columns WHERE table_schema='public' AND column_name='tenant_id' AND table_name NOT IN (${tableList},'users') LOOP
+        EXECUTE format('SELECT count(*) FROM %I WHERE tenant_id=$1 AND $1>0',entry.table_name) INTO count_rows USING ${tenant};
+        IF count_rows > 0 THEN RAISE EXCEPTION 'Unknown fixture dependency: %',entry.table_name; END IF;
+      END LOOP;
+      ${tables.map(table => `IF ${tenant} > 0 AND to_regclass('public.${table}') IS NOT NULL THEN DELETE FROM ${table} WHERE tenant_id=${tenant}; END IF;`).join('\n      ')}
+      DELETE FROM auth_tokens WHERE user_id IN (${userIDs});
+      DELETE FROM workbench_access_assignments WHERE user_id IN (${userIDs}) AND tenant_id=${manifest.source_tenant_id};
+      DELETE FROM workbench_access_grants WHERE user_id IN (${userIDs}) AND tenant_id=${manifest.source_tenant_id};
+      DELETE FROM tenant_members WHERE user_id IN (${userIDs}) AND tenant_id=${manifest.source_tenant_id};
+      DELETE FROM users WHERE id IN (${userIDs}) AND (${identities}) AND tenant_id IN (${tenant},${manifest.source_tenant_id});
+      DELETE FROM tenants WHERE id=${tenant} AND name=${literal(manifest.run)} AND business='native-memory-smoke-fixture';
+    END $fixture$;
+    COMMIT;`
+  fs.writeFileSync(path.join(path.dirname(manifestPath), 'fixture-cleanup.sql'), query + '\n', { mode: 0o600 })
+  sql(query)
+  assert.equal(sql(`SELECT (SELECT count(*) FROM tenants WHERE id=${tenant} AND ${tenant}>0) + (SELECT count(*) FROM users WHERE id IN (${userIDs}));`), '0')
+  for (const name of ['owner-auth.json', 'outsider-auth.json']) fs.rmSync(path.join(path.dirname(manifestPath), name), { force: true })
+  manifest.cleaned_at = new Date().toISOString()
+  manifest.cleanup_error = null
+  manifest.cleanup_note = 'Dedicated tenant, fixture users, tokens, balances, model, Agent and fixture data removed. Source workspace administrator audit entries are retained.'
+  persist()
+}
+
+async function main() {
+  if (process.argv[2] === '--cleanup') {
+    assert.ok(process.argv[3], '--cleanup requires the fixture manifest path')
+    manifestPath = path.resolve(process.argv[3])
+    manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'))
+    cleanup()
+    return
+  }
+  if (process.argv[2] !== '--run') {
+    console.log('Use --run to create an isolated fixture, run native_memory_api_smoke.cjs and remove all fixture data; use --cleanup /absolute/path/fixture.json to retry cleanup. See native_memory_api_smoke.md.')
+    return
+  }
+  const run = `memory-smoke-${crypto.randomBytes(8).toString('hex')}`
+  const directory = path.join(acceptance, run)
+  fs.mkdirSync(directory, { mode: 0o700, recursive: true })
+  manifestPath = path.join(directory, 'fixture.json')
+  manifest = { format: 'native-memory-fixture-v1', run, database,
+    base: (process.env.MEMORY_SMOKE_BASE || 'http://127.0.0.1:18080/api/v1').replace(/\/$/, ''),
+    source_tenant_id: Number(process.env.MEMORY_SMOKE_SOURCE_TENANT || 10059),
+    started_at: new Date().toISOString(), passed: false,
+    users: ['owner', 'outsider'].map(label => ({ username: `${run}-${label}`, email: `${run}-${label}@example.test` })) }
+  validateManifest()
+  persist()
+  try {
+    await provision()
+    await runSmoke()
+  } catch (error) {
+    manifest.error = error.message
+    process.exitCode = 1
+    persist()
+  } finally {
+    try { cleanup() } catch (error) { manifest.cleanup_error = error.message; process.exitCode = 1; persist() }
+  }
+  if (!manifest.passed || !manifest.cleaned_at) process.exitCode = 1
+}
+
+main().catch(error => {
+  process.exitCode = 1
+  console.error(error.message)
+}).finally(() => {
+  if (manifest) console.log(JSON.stringify({ passed: manifest.passed, cleaned: !!manifest.cleaned_at, error: manifest.error, cleanup_error: manifest.cleanup_error, manifest: manifestPath }, null, 2))
+})