xinmi/scripts/verify_resumable_uploads_live.cjs

// Disposable acceptance fixtures only. Never run against production.
// Optional: VERIFY_UPLOAD_20GIB=1 streams 20 GiB from a reused 4 MiB buffer.
// Raw bytes and credentials are never written to the report. The 20 GiB file
// intentionally exercises the explicit binary-parser bound after transport.
const fs = require('node:fs')
const crypto = require('node:crypto')
const assert = require('node:assert/strict')
const directory = process.env.ACCEPTANCE_DIR || '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
const base = process.env.ACCEPTANCE_API || 'http://127.0.0.1:18080/api/v1'
if (!/^http:\/\/(127\.0\.0\.1|localhost):18080\/api\/v1$/.test(base)) throw new Error('Only isolated local acceptance API is allowed')
const tenant = '10059', prefix = `resumable-check-${Date.now()}`
const checks = [], uploads = [], humans = [], kbs = [], users = [], roles = []
let admin
const hash = buffer => crypto.createHash('sha256').update(buffer).digest('hex')
const check = (name, value) => { assert.ok(value, name); checks.push({ name, passed: true }); console.log(name) }
async function api(path, method = 'GET', body, identity = admin, expected = 200, headers = {}) {
  const binary = Buffer.isBuffer(body)
  const response = await fetch(base + path, { method, headers: {
    'Content-Type': binary ? 'application/octet-stream' : 'application/json',
    ...(identity ? { Authorization: `Bearer ${identity.token}`, 'X-Tenant-ID': tenant } : {}), ...headers,
  }, body: body === undefined ? undefined : binary ? body : JSON.stringify(body), signal: AbortSignal.timeout(120000) })
  const raw = await response.text(); let result
  try { result = JSON.parse(raw) } catch { result = { message: raw.slice(0, 300) } }
  assert.ok((Array.isArray(expected) ? expected : [expected]).includes(response.status), `${method} ${path} HTTP ${response.status}: ${result.message || result.error?.message || ''}`)
  return result
}
async function chunk(upload, buffer, offset, identity = admin, expected = 200, checksum = hash(buffer)) {
  return api(`/uploads/${upload.id}/chunks`, 'PUT', buffer, identity, expected, { 'Upload-Offset': String(offset), 'Upload-Checksum': checksum })
}
async function wait(id) {
  const deadline = Date.now() + 240000
  while (Date.now() < deadline) {
    const row = (await api(`/uploads/${id}`)).data
    if (['completed', 'failed'].includes(row.state)) return row
    await new Promise(resolve => setTimeout(resolve, 1000))
  }
  throw new Error(`Upload ${id} timed out`)
}
async function create(kind, target, name, buffer, size = buffer.length, extra = {}) {
  const row = (await api('/uploads', 'POST', { kind, target_id: target, name, size, ...(size === buffer.length ? { sha256: hash(buffer) } : {}), ...extra }, admin, 201)).data
  uploads.push(row.id); return row
}
async function main() {
  const credentials = JSON.parse(fs.readFileSync(`${directory}/super-admin-login.json`, 'utf8'))
  admin = await api('/auth/login', 'POST', { email: credentials.email, password: credentials.password }, null)
  const role = (await api('/access/roles', 'POST', { name: prefix, permissions: ['knowledge.read', 'knowledge.write', 'digital-humans.read', 'digital-humans.write'] })).data; roles.push(role.id)
  const password = crypto.randomBytes(18).toString('base64url'), email = `${prefix}@example.test`
  const user = (await api('/access/users', 'POST', { username: prefix, email, password, role: 'contributor', status: 'active', custom_role_id: role.id })).data; users.push(user.id)
  const other = await api('/auth/login', 'POST', { email, password }, null)
  const human = (await api('/digital-humans', 'POST', { name: prefix, persona: {} }, admin, 201)).data; humans.push(human.id)
  const buffer = Buffer.alloc(4 * 1024 * 1024, 'a'), tail = Buffer.from('\nEND native resumable evidence\n')
  const row = await create('digital-human', human.id, 'material.txt', buffer, buffer.length + tail.length)
  await chunk(row, buffer, 0, admin, 400, '0'.repeat(64))
  check('checksum_mismatch_keeps_offset_zero', (await api(`/uploads/${row.id}`)).data.offset === 0)
  await chunk(row, buffer, 0, other, 404)
  await api(`/uploads/${row.id}`, 'GET', undefined, other, 404)
  await api(`/uploads/${row.id}/finalize`, 'POST', {}, other, 404)
  await api(`/uploads/${row.id}`, 'DELETE', undefined, other, 404)
  check('other_user_cannot_read_write_finalize_or_cancel', !(await api('/uploads', 'GET', undefined, other)).data.some(item => item.id === row.id))
  await chunk(row, buffer, 0)
  check('durable_offset_after_first_chunk', (await api(`/uploads/${row.id}`)).data.offset === buffer.length)
  await chunk(row, buffer, 0)
  check('idempotent_chunk_replay', (await api(`/uploads/${row.id}`)).data.offset === buffer.length)
  await api(`/uploads/${row.id}/finalize`, 'POST', {}, admin, 409)
  await chunk(row, tail, buffer.length)
  await api(`/uploads/${row.id}/finalize`, 'POST', {}, admin, 202)
  await api(`/uploads/${row.id}/finalize`, 'POST', {}, admin, 202)
  const parsed = await wait(row.id)
  check('material_async_bounded_evidence', parsed.state === 'completed' && parsed.result.selection === 'sampled' && [...parsed.result.source.text].length <= 12000 && parsed.result.source.text.includes('END native'))
  await api(`/digital-humans/${human.id}/distill`, 'POST', { upload_ids: [row.id], revision: human.revision }, other, [400, 404])
  check('material_cannot_be_consumed_by_other_user', true)
  const kb = (await api('/knowledge-bases', 'POST', { name: prefix, type: 'document', embedding_model_id: 'a2c3ce32-6192-46c8-8072-910abfe8afed', storage_backend_id: 'edcd8dc2-4e93-4bd4-839c-d835619fc0a6', indexing_strategy: { vector_enabled: true, keyword_enabled: true, wiki_enabled: false, graph_enabled: false } }, admin, [200, 201])).data; kbs.push(kb.id)
  const text = Buffer.from('Resumable native indexing acceptance. The evidence number is tangerine 846. The entire source is preserved and indexed.')
  const knowledge = await create('knowledge', kb.id, 'resumable.txt', text)
  await chunk(knowledge, text, 0)
  await api(`/uploads/${knowledge.id}/finalize`, 'POST', {}, admin, 202)
  const indexed = await wait(knowledge.id)
  check('knowledge_async_native_indexing', indexed.state === 'completed' && indexed.result.knowledge_ids.length === 1)
  const document = (await api(`/knowledge/${indexed.result.knowledge_ids[0]}`)).data
  check('knowledge_native_completed', document.parse_status === 'completed')
  const chunks = (await api(`/chunks/${document.id}?page_size=10`)).data
  check('knowledge_native_chunk_evidence', chunks.some(chunk => chunk.content.includes('tangerine 846')))
  await api('/uploads', 'POST', { kind: 'digital-human', target_id: human.id, name: 'too-large.txt', size: 20 * 1024 ** 3 + 1 }, admin, 400)
  check('rejects_transport_over_20gib', true)
  if (process.env.VERIFY_UPLOAD_20GIB === '1') {
    const total = 20 * 1024 ** 3, chunkData = Buffer.alloc(4 * 1024 * 1024), checksum = hash(chunkData)
    const large = await create('digital-human', human.id, 'transport-20gib.pdf', chunkData, total)
    const started = Date.now()
    for (let offset = 0; offset < total; offset += chunkData.length) {
      await chunk(large, chunkData, offset, admin, 200, checksum)
      if (offset === 0) {
        const reloaded = (await api(`/uploads/${large.id}`)).data
        assert.equal(reloaded.offset, chunkData.length)
        await chunk(large, chunkData, 0, admin, 200, checksum)
      }
      if ((offset + chunkData.length) % (1024 ** 3) === 0) console.log(`20 GiB transport: ${(offset + chunkData.length) / 1024 ** 3}/20 GiB`)
    }
    check('20gib_transport_streamed_with_resume', (await api(`/uploads/${large.id}`)).data.offset === total)
    await api(`/uploads/${large.id}/finalize`, 'POST', {}, admin, 202)
    const result = await wait(large.id)
    check('20gib_checksum_verified_without_full_memory_parse', result.sha256.length === 64 && result.state === 'failed' && result.error.includes('20 MiB'))
    checks.push({ name: '20gib_elapsed_seconds', value: Math.round((Date.now() - started) / 1000) })
    await api(`/uploads/${large.id}`, 'DELETE', undefined, admin)
  }
}
async function cleanup() {
  for (const id of uploads) try { await api(`/uploads/${id}`, 'DELETE', undefined, admin, [200, 404, 409]) } catch {}
  for (const id of kbs) try { await api(`/knowledge-bases/${id}`, 'DELETE', undefined, admin, [200, 202, 404]) } catch {}
  for (const id of humans) try { await api(`/digital-humans/${id}`, 'DELETE', undefined, admin, [200, 404]) } catch {}
  for (const id of users) try { await api(`/access/users/${id}`, 'DELETE', undefined, admin, [200, 404]) } catch {}
  for (const id of roles) try { await api(`/access/roles/${id}`, 'DELETE', undefined, admin, [200, 404]) } catch {}
}
main().catch(error => { checks.push({ name: 'failure', message: error.message }); console.error(error.message); process.exitCode = 1 }).finally(async () => {
  await cleanup()
  const report = { prefix, checks, fixtures: { uploads, humans, kbs, users, roles }, finished_at: new Date().toISOString() }
  fs.writeFileSync(`${directory}/${process.env.VERIFY_UPLOAD_REPORT || 'resumable-upload-verification.json'}`, JSON.stringify(report, null, 2))
})