xinmi/scripts/verify_fixture_cleanup.test.cjs

const assert = require('node:assert/strict')
const test = require('node:test')
const fs = require('node:fs')
const path = require('node:path')
const vm = require('node:vm')
const {cleanupMemoryFixture, memoryTables} = require('./verify_resource_isolation_live.cjs')
const {validateResponse, trackCreatedResource, finishVerification} = require('./verify_access_control.cjs')
const {cleanupMessageFixture} = require('./verify_resource_files_live.cjs')

const prefix = 'resource-isolation-1758297600000'
const fixture = Object.freeze({
  id: '5b4af2da-04eb-4e76-b54d-cec380559428',
  username: `${prefix}-alice`,
  email: `${prefix}-alice@example.test`,
})

test('memory cleanup restricts all six tables to the newly created account and verifies zero remaining rows', () => {
  const statements = []
  const subject = cleanupMemoryFixture(fixture, prefix, statement => {
    statements.push(statement)
    return statements.length === 1 ? '' : '0|0|0|0|0|0'
  })
  assert.equal(subject, `web_user:${fixture.id}`)
  assert.equal(statements.length, 2)
  assert.match(statements[0], /^BEGIN;/)
  assert.match(statements[0], /IF NOT EXISTS \(SELECT 1 FROM users/)
  for (const field of ['id', 'username', 'email']) assert.ok(statements[0].includes(`${field}='${fixture[field]}'`))
  assert.ok(statements[0].indexOf('RAISE EXCEPTION') < statements[0].indexOf('DELETE FROM'))
  assert.match(statements[0], /END; \$fixture_cleanup\$;/)
  assert.equal((statements[0].match(/DELETE FROM /g) || []).length, 6)
  for (const table of ['memory_item_embeddings', 'memory_items', 'memory_tombstones', 'memory_topic_stats', 'memory_doc_affinity', 'memory_subjects']) {
    const scoped = `${table} WHERE tenant_id=10059 AND subject_id='web_user:${fixture.id}'`
    assert.ok(statements[0].includes(`DELETE FROM ${scoped};`))
    assert.ok(statements[1].includes(`SELECT count(*) FROM ${scoped}`))
  }
  assert.match(statements[0], /COMMIT;$/)
  assert.equal(memoryTables.length, 6)
})

test('memory cleanup rejects business accounts, other runs, malformed IDs and mismatched email before any SQL', () => {
  const otherPrefix = 'resource-isolation-1758297500000'
  const rejected = [
    [{...fixture, username: 'admin'}, prefix],
    [{...fixture, email: 'business@example.com'}, prefix],
    [{...fixture, id: "x'; DELETE FROM users; --"}, prefix],
    [fixture, otherPrefix],
    [fixture, 'resource-isolation-%'],
  ]
  for (const [value, runPrefix] of rejected) {
    let calls = 0
    assert.throws(() => cleanupMemoryFixture(value, runPrefix, () => { calls++ }))
    assert.equal(calls, 0)
  }
})

test('memory cleanup surfaces owner guard errors and residual rows as failures', () => {
  const statements = []
  assert.throws(() => cleanupMemoryFixture(fixture, prefix, statement => {
    statements.push(statement)
    throw new Error('Memory cleanup account does not match this verification run')
  }), /does not match/)
  assert.equal(statements.length, 1)
  let calls = 0
  assert.throws(() => cleanupMemoryFixture(fixture, prefix, () => ++calls === 1 ? '' : '0|1|0|0|0|0'), /left rows/)
  assert.equal(calls, 2)
})

test('browser shutdown and cleanup failures still allow every cleanup action and a failed report', async () => {
  const events = [], failures = [], cleanup = [
    async () => { events.push('first') },
    async () => { events.push('second'); throw new Error('delete unavailable') },
    async () => { events.push('third') },
  ]
  let report
  await finishVerification({
    browser: {close: async () => { events.push('close'); throw new Error('browser disconnected') }},
    cleanup, failures,
    writeReport: () => { events.push('report'); report = {passed: failures.length === 0, failures: [...failures]} },
  })
  assert.deepEqual(events, ['close', 'third', 'second', 'first', 'report'])
  assert.equal(cleanup.length, 3)
  assert.deepEqual(report, {passed: false, failures: ['browser close: browser disconnected', 'cleanup: delete unavailable']})
})

test('failure before browser startup still cleans fixtures and writes the report', async () => {
  const events = [], failures = ['login failed']
  await finishVerification({
    cleanup: [() => events.push('fixture')], failures,
    writeReport: () => events.push(failures.length ? 'failed report' : 'passed report'),
  })
  assert.deepEqual(events, ['fixture', 'failed report'])
})

test('unexpected successful creation is registered before status assertion, including duplicate and fork envelopes', async () => {
  const cases = [
    ['/access/roles', {id: 'new-role'}, 'new-role', 400],
    ['/workbench/resources/content-tasks', {id: 'new-task'}, 'new-task', 403],
    ['/agents', {id: 'new-agent'}, 'new-agent', 403],
    ['/knowledge-bases', {target_id: 'new-kb'}, 'new-kb', 403],
    ['/knowledge-bases', {knowledge_base: {id: 'new-nested-kb'}}, 'new-nested-kb', 403],
    ['/knowledge', {id: 'new-document'}, 'new-document', 403],
    ['/workbench/resources/flows', {id: 'fork-metadata', personal_id: 'new-flow'}, 'new-flow', 410],
  ]
  for (const [collection, data, expectedID, expectedStatus] of cases) {
    const cleanup = [], removed = [], statements = []
    const track = trackCreatedResource(collection, ['original-resource'], cleanup, route => removed.push(route), statement => statements.push(statement))
    assert.throws(() => validateResponse(collection, 'POST', {data}, 201, expectedStatus, track), /HTTP 201/)
    assert.equal(cleanup.length, collection==='/workbench/resources/flows' ? 2 : 1)
    await finishVerification({cleanup, failures: [], writeReport: () => {}})
    assert.deepEqual(removed, [`${collection}/${expectedID}?detach=true`])
    if(collection==='/workbench/resources/flows'){
      assert.equal(statements.length, 1)
      assert.ok(statements[0].includes("WHERE tenant_id=10059 AND kind='flows' AND id='fork-metadata' AND personal_id='new-flow'"))
      assert.ok(statements[0].includes("WHERE tenant_id=10059 AND kind='flows' AND resource_id='new-flow'"))
    }else assert.deepEqual(statements, [])
  }
})

test('expected denial never registers an existing resource from an error response', () => {
  const cleanup = []
  validateResponse('/agents/original/copy', 'POST', {data: {id: 'original'}}, 403, [403,404], trackCreatedResource('/agents', [], cleanup))
  assert.equal(cleanup.length, 0)
})

test('unexpected creation refuses to delete the original or fork metadata instead of the created resource', () => {
  for (const [collection, result] of [
    ['/agents', {data: {id: 'original'}}],
    ['/knowledge-bases', {data: {target_id: 'original'}}],
    ['/workbench/resources/flows', {data: {id: 'fork-metadata'}}],
  ]) {
    const cleanup = []
    assert.throws(() => validateResponse(collection, 'POST', result, 200, 403, trackCreatedResource(collection, ['original'], cleanup)), /new resource ID/)
    assert.equal(cleanup.length, 0)
  }
})

test('resource isolation main cleans both fixture memory scopes after a failure following memory creation', {timeout: 3000}, async () => {
  const events = [], userIDs = [fixture.id, 'fcf8b9eb-2af9-4267-9f8c-b288849f8310']
  let createdUsers = 0, createdAgents = 0, memoryCreated = false, complete
  const reported = new Promise(resolve => { complete = resolve })
  const verificationModule = {exports: {}}
  const fakeFS = {
    readFileSync: () => JSON.stringify({email: 'fake-admin@example.test', password: 'fake-test-password'}),
    writeFileSync: (_file, body) => complete(JSON.parse(body)),
  }
  const fakeRequire = name => {
    if(name==='node:fs')return fakeFS
    if(name==='node:child_process')return {execFileSync: (_command, args) => {
      const statement = args.at(-1)
      events.push({type: 'sql', statement})
      return statement.startsWith('SELECT ') ? '0|0|0|0|0|0' : ''
    }}
    return require(name)
  }
  fakeRequire.main = verificationModule
  const context = vm.createContext({
    module: verificationModule, require: fakeRequire,
    process: {env: {}}, console: {log() {}, error() {}},
    FormData, Blob, AbortSignal,
    fetch: async (url, request) => {
      const route = new URL(url).pathname.replace('/api/v1', '')
      const method = request.method, body = request.body instanceof FormData ? {} : JSON.parse(request.body || '{}')
      events.push({type: 'http', route, method})
      let status = 200, data = {}
      if(route==='/auth/login'){
        const index = body.email.includes('-alice@') ? 0 : body.email.includes('-bob@') ? 1 : -1
        data = {token: 'fake-token', user: {id: index<0 ? 'fake-admin' : userIDs[index]}}
      }else if(route==='/access/roles'&&method==='POST')data = {data: {id: 'fake-role', version: 1}}
      else if(route==='/access/users'&&method==='POST')data = {data: {...body, id: userIDs[createdUsers++]}}
      else if(route==='/knowledge-bases'&&method==='POST')data = {data: {id: 'fake-kb', creator_id: userIDs[0]}}
      else if(route==='/knowledge-bases'&&method==='GET')data = {data: []}
      else if(route==='/agents'&&method==='POST')data = {data: {id: `fake-agent-${++createdAgents}`}}
      else if(route.endsWith('/knowledge/manual'))data = {data: {id: 'fake-doc'}}
      else if(route.includes('/suggested-questions'))data = {data: []}
      else if(route==='/skills/instructions/import')data = {data: {id: 'fake-skill'}}
      else if(route==='/memory/items'&&method==='POST'){
        memoryCreated = true
        data = {data: {id: 'fake-memory'}}
      }else if(route==='/memory/items'&&method==='GET')throw new Error('injected failure after memory creation')
      else if(method!=='DELETE'&&!route.startsWith('/access/'))status = 403
      return {status, text: async () => JSON.stringify(data)}
    },
  })
  vm.runInContext(fs.readFileSync(path.join(__dirname, 'verify_resource_isolation_live.cjs'), 'utf8'), context)
  const report = await reported
  assert.ok(memoryCreated)
  assert.equal(report.passed, false)
  assert.ok(report.failures.includes('injected failure after memory creation'))
  assert.deepEqual(report.fixtures.memory_subjects, userIDs.map(id => `web_user:${id}`))
  for(const id of userIDs){
    const cleanupIndex = events.findIndex(event => event.type==='sql' && event.statement.includes('DELETE FROM memory_items') && event.statement.includes(`subject_id='web_user:${id}'`))
    const userDeleteIndex = events.findIndex(event => event.type==='http' && event.route===`/access/users/${id}` && event.method==='DELETE')
    assert.ok(cleanupIndex>=0&&userDeleteIndex>cleanupIndex)
    assert.ok(report.checks.some(check => check.name===`memory_cleanup_${id}`&&check.passed))
  }
  assert.ok(!events.some(event => event.type==='sql'&&event.statement.includes("subject_id='web_user:fake-admin'")))
})

const filePrefix = 'file-isolation-1758297600000'
const messageFixture = Object.freeze({
  tenant_id: 10059,
  session_id: 'a75ddc61-81e2-42f1-a870-656aac0a0931',
  user_id: fixture.id,
  username: `${filePrefix}-alice`,
  email: `${filePrefix}-alice@example.test`,
  title: filePrefix,
  messages: [
    {id: '40c638f6-eafc-4305-85a6-993d4c4250e5', content: 'Disposable attachment fixture.'},
    {id: '9a38599e-4d21-485b-8bfc-8bf24fa19b3f', content: 'Unrelated message.'},
  ],
})

test('message cleanup guards exact tenant, account, session and both synthetic message contents', () => {
  const statements = []
  cleanupMessageFixture(messageFixture, filePrefix, statement => {
    statements.push(statement)
    return statements.length === 1 ? '' : '0'
  })
  assert.equal(statements.length, 2)
  const transaction = statements[0]
  assert.match(transaction, /^BEGIN;/)
  assert.ok(transaction.includes(`SELECT 1 FROM users WHERE id='${messageFixture.user_id}' AND tenant_id=10059 AND username='${messageFixture.username}' AND email='${messageFixture.email}'`))
  const sessionScope = `id='${messageFixture.session_id}' AND tenant_id=10059 AND user_id='${messageFixture.user_id}' AND title='${filePrefix}'`
  assert.ok(transaction.includes(`IF NOT EXISTS (SELECT 1 FROM sessions WHERE ${sessionScope})`))
  for (const message of messageFixture.messages) {
    const scope = `id='${message.id}' AND session_id='${messageFixture.session_id}' AND role='assistant' AND content='${message.content}'`
    assert.ok(transaction.includes(`IF EXISTS (SELECT 1 FROM messages WHERE id='${message.id}') AND NOT EXISTS (SELECT 1 FROM messages WHERE ${scope})`))
    assert.ok(transaction.includes(`DELETE FROM messages WHERE ${scope} AND EXISTS (SELECT 1 FROM sessions WHERE ${sessionScope});`))
    assert.ok(statements[1].includes(`'${message.id}'`))
  }
  assert.ok(transaction.lastIndexOf('RAISE EXCEPTION') < transaction.indexOf('DELETE FROM'))
  assert.equal((transaction.match(/DELETE FROM /g) || []).length, 2)
  assert.match(transaction, /END; \$message_cleanup\$;/)
  assert.match(transaction, /COMMIT;$/)
})

test('message cleanup rejects business or mismatched fixture metadata without SQL', () => {
  const rejected = [
    [{...messageFixture, tenant_id: 10060}, filePrefix],
    [{...messageFixture, username: 'admin'}, filePrefix],
    [{...messageFixture, email: 'business@example.com'}, filePrefix],
    [{...messageFixture, title: 'Business conversation'}, filePrefix],
    [{...messageFixture, session_id: "x'; DELETE FROM sessions; --"}, filePrefix],
    [{...messageFixture, user_id: 'admin'}, filePrefix],
    [{...messageFixture, messages: [{...messageFixture.messages[0], content: 'Real customer message'}, messageFixture.messages[1]]}, filePrefix],
    [{...messageFixture, messages: [messageFixture.messages[0], {...messageFixture.messages[1], id: messageFixture.messages[0].id}]}, filePrefix],
    [{...messageFixture, messages: [messageFixture.messages[0]]}, filePrefix],
    [messageFixture, 'file-isolation-1758297500000'],
    [messageFixture, 'file-isolation-%'],
  ]
  for (const [value, runPrefix] of rejected) {
    let calls = 0
    assert.throws(() => cleanupMessageFixture(value, runPrefix, () => { calls++ }))
    assert.equal(calls, 0)
  }
})

test('message cleanup propagates database guard failures and checks residual rows', () => {
  let calls = 0
  assert.throws(() => cleanupMessageFixture(messageFixture, filePrefix, () => {
    calls++
    throw new Error('Message cleanup session does not match this verification run')
  }), /does not match/)
  assert.equal(calls, 1)
  calls = 0
  assert.throws(() => cleanupMessageFixture(messageFixture, filePrefix, () => ++calls === 1 ? '' : '1'), /left rows/)
  assert.equal(calls, 2)
})

for (const failurePoint of ['message insert', 'message file check']) test(`file isolation cleans messages and reports exact fixture IDs after ${failurePoint} failure`, {timeout: 3000}, async () => {
  const events = [], userIDs = [fixture.id, 'fcf8b9eb-2af9-4267-9f8c-b288849f8310']
  let createdUsers = 0, createdKBs = 0, grant = false, observedPrefix, complete
  const reported = new Promise(resolve => { complete = resolve })
  const verificationModule = {exports: {}}
  const resourceFiles = new Map(), physicalFiles = new Map()
  const fakeFS = {
    readFileSync: () => JSON.stringify({email: 'fake-admin@example.test', password: 'fake-test-password'}),
    mkdirSync() {},
    rmSync: file => events.push({type: 'rm', file}),
    writeFileSync: (file, body) => {
      if (file.endsWith('.json')) complete(JSON.parse(body))
      else events.push({type: 'file', file, body})
    },
  }
  const fakeRequire = name => {
    if (name === 'node:fs') return fakeFS
    if (name === 'node:child_process') return {execFileSync: (_command, _args, options) => {
      const statement = options.input
      events.push({type: 'sql', statement})
      if (statement.startsWith('INSERT INTO messages') && failurePoint === 'message insert') throw new Error('injected message insert failure')
      if (statement.startsWith('INSERT INTO resources')) {
        const values = statement.match(/VALUES \('[^']+','([^']+)',10059,'local','([^']+)'/)
        const label = values[2].split('/').at(-1).replace('.txt', '')
        const file = {label, marker: `${observedPrefix} ${label}`}
        resourceFiles.set(`resource://${values[1]}`, file)
        physicalFiles.set(values[2], file)
      }
      return statement.startsWith('SELECT count(*)') ? '0' : ''
    }}
    return require(name)
  }
  fakeRequire.main = verificationModule
  const context = vm.createContext({
    module: verificationModule, require: fakeRequire,
    process: {env: {}}, console: {log() {}, error() {}}, AbortSignal, Buffer,
    fetch: async (url, request) => {
      const parsed = new URL(url), route = parsed.pathname.replace('/api/v1', '')
      const method = request.method, body = JSON.parse(request.body || '{}')
      const identity = request.headers.Authorization?.replace('Bearer ', '')
      events.push({type: 'http', route, method})
      let status = 200, data = {}, responseText
      if (route === '/auth/login') {
        const index = body.email.includes('-alice@') ? 0 : body.email.includes('-bob@') ? 1 : -1
        data = {token: index < 0 ? 'admin' : index === 0 ? 'alice' : 'bob', user: {id: index < 0 ? 'fake-admin' : userIDs[index]}}
      } else if (route === '/access/roles' && method === 'POST') {
        observedPrefix = body.name
        data = {data: {id: 'fake-role'}}
      } else if (route === '/access/users' && method === 'POST') data = {data: {...body, id: userIDs[createdUsers++]}}
      else if (route === '/knowledge-bases' && method === 'POST') data = {data: {id: `fake-kb-${++createdKBs}`}}
      else if (route.endsWith('/knowledge/manual')) data = {data: {id: 'fake-doc'}}
      else if (route === '/access/grants') grant = body.action === 'grant'
      else if (route === '/sessions' && method === 'POST') data = {data: {id: messageFixture.session_id}}
      else if (parsed.searchParams.has('file_path')) {
        const ref = parsed.searchParams.get('file_path')
        const file = resourceFiles.get(ref) || physicalFiles.get(ref)
        assert.ok(file)
        if (file.label === 'message' && failurePoint === 'message file check') throw new Error('injected message file check failure')
        const allowed = identity === 'admin' || identity === 'alice' || (identity === 'bob' && grant && file.label === 'knowledge')
        status = allowed ? 200 : 403
        responseText = allowed ? file.marker : '{}'
      } else if (route.startsWith('/sessions') && identity === 'bob') status = 404
      else if (method === 'DELETE') { status = 204; responseText = '' }
      return {status, text: async () => responseText ?? JSON.stringify(data), headers: {get: () => 'private, no-store'}}
    },
  })
  vm.runInContext(fs.readFileSync(path.join(__dirname, 'verify_resource_files_live.cjs'), 'utf8'), context)
  const report = await reported
  assert.equal(report.passed, false)
  assert.ok(report.failures.includes(`injected ${failurePoint} failure`), JSON.stringify(report.failures))
  assert.deepEqual(report.fixtures.users, userIDs)
  assert.deepEqual(report.fixtures.sessions, [messageFixture.session_id])
  assert.equal(report.fixtures.messages.length, 2)
  assert.equal(report.fixtures.message_scopes.length, 1)
  assert.equal(report.fixtures.message_scopes[0].session_id, messageFixture.session_id)
  assert.equal(report.fixtures.message_scopes[0].user_id, userIDs[0])
  assert.deepEqual(report.fixtures.message_scopes[0].messages.map(message => message.id), report.fixtures.messages)
  const cleanupIndex = events.findIndex(event => event.type === 'sql' && event.statement.includes('DELETE FROM messages'))
  const sessionDeleteIndex = events.findIndex(event => event.type === 'http' && event.method === 'DELETE' && event.route === `/sessions/${messageFixture.session_id}` && events.indexOf(event) > cleanupIndex)
  const userDeleteIndex = events.findIndex(event => event.type === 'http' && event.method === 'DELETE' && event.route === `/access/users/${userIDs[0]}`)
  assert.ok(cleanupIndex >= 0 && sessionDeleteIndex > cleanupIndex && userDeleteIndex > sessionDeleteIndex)
  const statement = events[cleanupIndex].statement
  for (const id of report.fixtures.messages) assert.ok(statement.includes(`id='${id}'`))
  assert.ok(report.checks.some(check => check.name === `message_cleanup_${messageFixture.session_id}` && check.passed))
})