xinmi/scripts/native_memory_fixture.cjs

#!/usr/bin/env node
'use strict'

const assert = require('node:assert/strict')
const crypto = require('node:crypto')
const fs = require('node:fs')
const path = require('node:path')
const { execFileSync, spawn } = require('node:child_process')

const acceptance = process.env.MEMORY_SMOKE_DIRECTORY || '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
const database = {
  context: process.env.MEMORY_SMOKE_DOCKER_CONTEXT || 'colima-yawan',
  container: process.env.MEMORY_SMOKE_DB_CONTAINER || 'WeKnora-postgres',
  name: process.env.MEMORY_SMOKE_DB_NAME || 'yawan_native_acceptance_20260915',
}
const literal = value => `'${String(value).replaceAll("'", "''")}'`
let manifest, manifestPath, admin

function privateJSON(file, value) {
  const temporary = `${file}.${crypto.randomBytes(4).toString('hex')}.tmp`
  const fd = fs.openSync(temporary, 'wx', 0o600)
  try {
    fs.writeFileSync(fd, `${JSON.stringify(value, null, 2)}\n`)
    fs.fsyncSync(fd)
  } finally { fs.closeSync(fd) }
  fs.renameSync(temporary, file)
}
const persist = () => privateJSON(manifestPath, manifest)
function sql(query) {
  try {
    return execFileSync(process.env.MEMORY_SMOKE_DOCKER || '/opt/homebrew/bin/docker', [
      '--context', database.context, 'exec', '-i', database.container,
      'psql', '-X', '-q', '-U', 'postgres', '-d', database.name,
      '-At', '-v', 'ON_ERROR_STOP=1',
    ], { input: query, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 30000 }).trim()
  } catch {
    // SQL stderr may contain request fields. Keep credentials out of reports.
    throw new Error('Fixture database operation failed; inspect the bounded fixture SQL and database locally')
  }
}
async function api(route, body, identity = admin, tenant = manifest.source_tenant_id) {
  const response = await fetch(manifest.base + route, {
    method: body === undefined ? 'GET' : 'POST',
    headers: { 'Content-Type': 'application/json', ...(identity ? { Authorization: `Bearer ${identity.token}`, 'X-Tenant-ID': String(tenant) } : {}) },
    body: body === undefined ? undefined : JSON.stringify(body),
    signal: AbortSignal.timeout(15000),
  })
  assert.ok(response.ok, `${route}: HTTP ${response.status}`)
  return response.json()
}
function validateManifest() {
  assert.equal(manifest.format, 'native-memory-fixture-v1')
  assert.match(manifest.run, /^memory-smoke-[a-f0-9]{16}$/)
  assert.deepEqual(manifest.database, database, 'Use the same database connection as the fixture manifest')
  assert.ok(Number.isSafeInteger(manifest.source_tenant_id) && manifest.source_tenant_id > 0)
  if (manifest.tenant_id !== undefined) {
    assert.ok(Number.isSafeInteger(manifest.tenant_id) && manifest.tenant_id > 0 && manifest.tenant_id !== manifest.source_tenant_id)
  }
  assert.equal(manifest.users.length, 2)
  for (const [index, label] of ['owner', 'outsider'].entries()) {
    assert.equal(manifest.users[index].username, `${manifest.run}-${label}`)
    assert.equal(manifest.users[index].email, `${manifest.run}-${label}@example.test`)
  }
  const url = new URL(manifest.base)
  assert.ok(['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname), 'Only loopback API origins are permitted')
}

async function provision() {
  const adminFile = process.env.MEMORY_SMOKE_ADMIN_AUTH || path.join(acceptance, 'super-admin-login.json')
  assert.equal(fs.statSync(adminFile).mode & 0o077, 0, 'Admin credentials must be a private file (mode 0600)')
  const credentials = JSON.parse(fs.readFileSync(adminFile, 'utf8'))
  admin = await api('/auth/login', { email: credentials.email, password: credentials.password }, null)
  assert.equal(typeof admin.token, 'string')
  // The random name and business marker allow exact recovery after interruption.
  manifest.tenant_id = Number(sql(`INSERT INTO tenants(name,business,status,memory_config)
    VALUES (${literal(manifest.run)},'native-memory-smoke-fixture','active',
      '{"enabled":true,"write_mode":"explicit_only","vector_recall":false,"embedding_model_id":""}') RETURNING id;`))
  assert.ok(Number.isSafeInteger(manifest.tenant_id) && manifest.tenant_id > 0 && manifest.tenant_id !== manifest.source_tenant_id)
  persist()
  for (const user of manifest.users) {
    const password = crypto.randomBytes(18).toString('base64url')
    const row = (await api('/access/users', { username: user.username, email: user.email, password, role: 'contributor', status: 'active' })).data
    user.id = row.id
    persist()
    const role = user === manifest.users[0] ? 'owner' : 'contributor'
    sql(`BEGIN;
      UPDATE users SET tenant_id=${manifest.tenant_id} WHERE id=${literal(user.id)} AND email=${literal(user.email)} AND tenant_id=${manifest.source_tenant_id};
      UPDATE tenant_members SET tenant_id=${manifest.tenant_id},role=${literal(role)} WHERE user_id=${literal(user.id)} AND tenant_id=${manifest.source_tenant_id};
      DELETE FROM workbench_access_assignments WHERE user_id=${literal(user.id)} AND tenant_id=${manifest.source_tenant_id};
      COMMIT;`)
    const identity = await api('/auth/login', { email: user.email, password }, null)
    assert.equal(Number(identity.user.tenant_id), manifest.tenant_id, 'Fixture login selected a different workspace')
    assert.equal(identity.user.is_system_admin, false, 'Fixture user must not be a system administrator')
    privateJSON(path.join(path.dirname(manifestPath), `${user === manifest.users[0] ? 'owner' : 'outsider'}-auth.json`), { token: identity.token, tenantID: manifest.tenant_id })
    if (role === 'owner') {
      await api('/workbench/operations/grants', { user_id: user.id, amount: 1_000_000_000, reason: manifest.run, idempotency_key: manifest.run }, identity, manifest.tenant_id)
    }
  }
  manifest.provisioned_at = new Date().toISOString()
  persist()
}

function runSmoke() {
  const directory = path.dirname(manifestPath)
  return new Promise((resolve, reject) => {
    const child = spawn(process.execPath, [path.join(__dirname, 'native_memory_api_smoke.cjs'), '--run'], {
      env: { ...process.env, MEMORY_SMOKE_BASE: manifest.base,
        MEMORY_SMOKE_RUN: manifest.run, MEMORY_SMOKE_LEGACY: '1',
        MEMORY_SMOKE_OWNER_AUTH: path.join(directory, 'owner-auth.json'),
        MEMORY_SMOKE_OUTSIDER_AUTH: path.join(directory, 'outsider-auth.json'),
        MEMORY_SMOKE_MANIFEST: path.join(directory, 'smoke.json') },
      stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
    })
    manifest.smoke_pid = child.pid
    persist()
    const timer = setTimeout(() => child.kill('SIGTERM'), 300000)
    child.on('message', message => {
      if (message?.type !== 'seed-legacy-memory') return
      try {
        validateManifest()
        assert.equal(message.run, manifest.run)
        assert.match(message.agent_id, /^[a-f0-9-]{36}$/)
        assert.ok(!manifest.legacy_candidate, 'Only one synthetic legacy candidate is permitted')
        const ownerID = manifest.users[0].id
        const id = crypto.randomUUID(), historyID = crypto.randomUUID()
        const original = `${manifest.run}: original legacy advice`
        const source = `${manifest.run}/legacy/${historyID}`
        // Seed only a synthetic accepted history row in this run's empty workspace.
        // Migration provenance matching itself is verified by PostgreSQL unit tests.
        sql(`BEGIN;
          DO $seed$ BEGIN
            IF NOT EXISTS (SELECT 1 FROM tenants WHERE id=${manifest.tenant_id} AND name=${literal(manifest.run)} AND business='native-memory-smoke-fixture') OR
               NOT EXISTS (SELECT 1 FROM custom_agents WHERE id=${literal(message.agent_id)} AND tenant_id=${manifest.tenant_id} AND created_by=${literal(ownerID)} AND name=${literal(manifest.run)} AND deleted_at IS NULL AND config->>'memory_enabled'='true')
            THEN RAISE EXCEPTION 'Legacy fixture scope mismatch'; END IF;
          END $seed$;
          INSERT INTO workbench_history(id,tenant_id,owner_id,source_kind,source_id,payload)
          VALUES(${literal(historyID)},${manifest.tenant_id},${literal(ownerID)},'pendingmemory',${literal(id)},${literal(JSON.stringify({ fixture: manifest.run, content: original }))}::jsonb);
          INSERT INTO workbench_pending_memories(id,tenant_id,owner_id,agent_id,content,source_ref,status,reviewed_by,legacy_history_id)
          VALUES(${literal(id)},${manifest.tenant_id},${literal(ownerID)},${literal(message.agent_id)},${literal(original)},${literal(source)},'accepted',${literal(ownerID)},${literal(historyID)});
          COMMIT;`)
        manifest.legacy_candidate = { id, history_id: historyID, agent_id: message.agent_id }
        persist()
        child.send({ type: 'legacy-memory-seeded', id, history_id: historyID })
      } catch (error) {
        manifest.legacy_seed_error = error.message
        persist()
        if (child.connected) child.send({ type: 'legacy-memory-seeded', error: error.message })
      }
    })
    child.once('error', error => { clearTimeout(timer); reject(error) })
    child.once('close', (code, signal) => {
      clearTimeout(timer)
      manifest.smoke_exit_code = code
      manifest.smoke_signal = signal
      manifest.smoke_finished_at = new Date().toISOString()
      manifest.passed = code === 0
      persist()
      resolve()
    })
  })
}

function cleanup() {
  validateManifest()
  if (manifest.smoke_pid && !manifest.smoke_finished_at) {
    let running = true
    try { process.kill(manifest.smoke_pid, 0) } catch (error) {
      if (error.code === 'ESRCH') running = false
    }
    assert.equal(running, false, 'Fixture smoke process is still alive; wait for it to finish before cleanup')
  }
  const discovered = sql(`SELECT id FROM tenants WHERE name=${literal(manifest.run)} AND business='native-memory-smoke-fixture';`)
  if (discovered) {
    const tenant = Number(discovered)
    assert.ok(Number.isSafeInteger(tenant) && tenant > 0 && tenant !== manifest.source_tenant_id)
    if (manifest.tenant_id) assert.equal(tenant, manifest.tenant_id)
    manifest.tenant_id = tenant
  }
  const tenant = manifest.tenant_id || 0
  const identities = manifest.users.map(user => `(username=${literal(user.username)} AND email=${literal(user.email)})`).join(' OR ')
  const rows = JSON.parse(sql(`SELECT COALESCE(json_agg(json_build_object('id',id,'username',username,'tenant_id',tenant_id)),'[]') FROM users WHERE ${identities};`))
  for (const row of rows) {
    const user = manifest.users.find(value => value.username === row.username)
    if (user.id) assert.equal(user.id, row.id)
    assert.ok([manifest.source_tenant_id, tenant].includes(Number(row.tenant_id)), 'Fixture user moved to an unexpected workspace')
    user.id = row.id
  }
  persist()
  const userIDs = manifest.users.filter(user => user.id).map(user => literal(user.id)).join(',') || "''"
  // Unknown nonempty tables stop cleanup. This prevents silently orphaning new
  // fixture dependencies when future API changes add another storage surface.
  const tables = [
    'workbench_pending_memories', 'workbench_revisions', 'workbench_events', 'workbench_jobs',
    'workbench_resources', 'workbench_balance_holds', 'workbench_usage', 'workbench_transactions',
    'workbench_balances', 'workbench_operation_logs', 'workbench_consumption', 'workbench_history',
    'workbench_usage_quotas', 'workbench_resource_forks', 'workbench_resource_submissions',
    'workbench_resource_hidden', 'workbench_resource_version_history',
    'workbench_access_grants', 'workbench_access_publications', 'workbench_access_assignments', 'workbench_access_roles',
    'memory_item_embeddings', 'memory_tombstones', 'memory_items', 'memory_topic_stats', 'memory_doc_affinity', 'memory_subjects',
    'custom_agents', 'models', 'audit_logs', 'tenant_members',
  ]
  const tableList = tables.map(literal).join(',')
  const query = `BEGIN;
    SET LOCAL lock_timeout = '3s';
    SET LOCAL statement_timeout = '20s';
    DO $fixture$
    DECLARE entry record; count_rows bigint;
    BEGIN
      IF ${tenant} <> 0 AND EXISTS (SELECT 1 FROM tenants WHERE id=${tenant} AND (name<>${literal(manifest.run)} OR business<>'native-memory-smoke-fixture')) THEN RAISE EXCEPTION 'Fixture tenant identity mismatch'; END IF;
      IF EXISTS (SELECT 1 FROM users WHERE id IN (${userIDs}) AND (is_system_admin OR can_access_all_tenants OR NOT (${identities}))) THEN RAISE EXCEPTION 'Unexpected user identity'; END IF;
      IF ${tenant} > 0 AND (EXISTS (SELECT 1 FROM users WHERE tenant_id=${tenant} AND id NOT IN (${userIDs})) OR EXISTS (SELECT 1 FROM tenant_members WHERE tenant_id=${tenant} AND user_id NOT IN (${userIDs}))) THEN RAISE EXCEPTION 'Fixture has other users'; END IF;
      IF EXISTS (SELECT 1 FROM tenant_members WHERE user_id IN (${userIDs}) AND tenant_id NOT IN (${tenant},${manifest.source_tenant_id})) THEN RAISE EXCEPTION 'Fixture user has foreign memberships'; END IF;
      IF ${tenant} > 0 AND EXISTS (SELECT 1 FROM workbench_jobs WHERE tenant_id=${tenant} AND status IN ('queued','running')) THEN RAISE EXCEPTION 'Fixture jobs are still active'; END IF;
      FOR entry IN SELECT table_name FROM information_schema.columns WHERE table_schema='public' AND column_name='tenant_id' AND table_name NOT IN (${tableList},'users') LOOP
        EXECUTE format('SELECT count(*) FROM %I WHERE tenant_id=$1 AND $1>0',entry.table_name) INTO count_rows USING ${tenant};
        IF count_rows > 0 THEN RAISE EXCEPTION 'Unknown fixture dependency: %',entry.table_name; END IF;
      END LOOP;
      ${tables.map(table => `IF ${tenant} > 0 AND to_regclass('public.${table}') IS NOT NULL THEN DELETE FROM ${table} WHERE tenant_id=${tenant}; END IF;`).join('\n      ')}
      DELETE FROM auth_tokens WHERE user_id IN (${userIDs});
      DELETE FROM workbench_access_assignments WHERE user_id IN (${userIDs}) AND tenant_id=${manifest.source_tenant_id};
      DELETE FROM workbench_access_grants WHERE user_id IN (${userIDs}) AND tenant_id=${manifest.source_tenant_id};
      DELETE FROM tenant_members WHERE user_id IN (${userIDs}) AND tenant_id=${manifest.source_tenant_id};
      DELETE FROM users WHERE id IN (${userIDs}) AND (${identities}) AND tenant_id IN (${tenant},${manifest.source_tenant_id});
      DELETE FROM tenants WHERE id=${tenant} AND name=${literal(manifest.run)} AND business='native-memory-smoke-fixture';
    END $fixture$;
    COMMIT;`
  fs.writeFileSync(path.join(path.dirname(manifestPath), 'fixture-cleanup.sql'), query + '\n', { mode: 0o600 })
  sql(query)
  assert.equal(sql(`SELECT (SELECT count(*) FROM tenants WHERE id=${tenant} AND ${tenant}>0) + (SELECT count(*) FROM users WHERE id IN (${userIDs}));`), '0')
  for (const name of ['owner-auth.json', 'outsider-auth.json']) fs.rmSync(path.join(path.dirname(manifestPath), name), { force: true })
  manifest.cleaned_at = new Date().toISOString()
  manifest.cleanup_error = null
  manifest.cleanup_note = 'Dedicated tenant, fixture users, tokens, balances, model, Agent and fixture data removed. Source workspace administrator audit entries are retained.'
  persist()
}

async function main() {
  if (process.argv[2] === '--cleanup') {
    assert.ok(process.argv[3], '--cleanup requires the fixture manifest path')
    manifestPath = path.resolve(process.argv[3])
    manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'))
    cleanup()
    return
  }
  if (process.argv[2] !== '--run') {
    console.log('Use --run to create an isolated fixture, run native_memory_api_smoke.cjs and remove all fixture data; use --cleanup /absolute/path/fixture.json to retry cleanup. See native_memory_api_smoke.md.')
    return
  }
  const run = `memory-smoke-${crypto.randomBytes(8).toString('hex')}`
  const directory = path.join(acceptance, run)
  fs.mkdirSync(directory, { mode: 0o700, recursive: true })
  manifestPath = path.join(directory, 'fixture.json')
  manifest = { format: 'native-memory-fixture-v1', run, database,
    base: (process.env.MEMORY_SMOKE_BASE || 'http://127.0.0.1:18080/api/v1').replace(/\/$/, ''),
    source_tenant_id: Number(process.env.MEMORY_SMOKE_SOURCE_TENANT || 10059),
    started_at: new Date().toISOString(), passed: false,
    users: ['owner', 'outsider'].map(label => ({ username: `${run}-${label}`, email: `${run}-${label}@example.test` })) }
  validateManifest()
  persist()
  try {
    await provision()
    await runSmoke()
  } catch (error) {
    manifest.error = error.message
    process.exitCode = 1
    persist()
  } finally {
    try { cleanup() } catch (error) { manifest.cleanup_error = error.message; process.exitCode = 1; persist() }
  }
  if (!manifest.passed || !manifest.cleaned_at) process.exitCode = 1
}

main().catch(error => {
  process.exitCode = 1
  console.error(error.message)
}).finally(() => {
  if (manifest) console.log(JSON.stringify({ passed: manifest.passed, cleaned: !!manifest.cleaned_at, error: manifest.error, cleanup_error: manifest.cleanup_error, manifest: manifestPath }, null, 2))
})