92 lines
8.4 KiB
JavaScript
92 lines
8.4 KiB
JavaScript
|
// Disposable fixtures in the isolated native acceptance database only.
|
||
|
|
const fs = require('node:fs')
|
||
|
|
const path = require('node:path')
|
||
|
|
const crypto = require('node:crypto')
|
||
|
|
const assert = require('node:assert/strict')
|
||
|
|
const { execFileSync } = require('node:child_process')
|
||
|
|
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
|
||
|
|
const base = 'http://127.0.0.1:18080/api/v1', tenant = '10059'
|
||
|
|
const prefix = `resumable-recovery-${Date.now()}`, checks = [], fixtures = { uploads: [], kbs: [], users: [], roles: [], documents: [] }
|
||
|
|
let admin
|
||
|
|
const hash = (bytes, algorithm = 'sha256') => crypto.createHash(algorithm).update(bytes).digest('hex')
|
||
|
|
const literal = value => `'${String(value).replaceAll("'", "''")}'`
|
||
|
|
const sql = statement => execFileSync('docker', ['--context', 'colima-yawan', 'exec', 'WeKnora-postgres', 'psql', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-v', 'ON_ERROR_STOP=1', '-Atc', statement], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim()
|
||
|
|
const check = (name, value) => { assert.ok(value, name); checks.push({ name, passed: true }); console.log(name) }
|
||
|
|
const uuid5 = name => {
|
||
|
|
const bytes = crypto.createHash('sha1').update(Buffer.from('6ba7b8129dad11d180b400c04fd430c8', 'hex')).update(name).digest().subarray(0, 16)
|
||
|
|
bytes[6] = (bytes[6] & 15) | 0x50; bytes[8] = (bytes[8] & 63) | 0x80
|
||
|
|
return bytes.toString('hex').replace(/^(.{8})(.{4})(.{4})(.{4})(.{12})$/, '$1-$2-$3-$4-$5')
|
||
|
|
}
|
||
|
|
async function api(route, method = 'GET', body, identity = admin, expected = 200, headers = {}) {
|
||
|
|
const binary = Buffer.isBuffer(body)
|
||
|
|
const response = await fetch(base + route, { method, headers: { 'Content-Type': binary ? 'application/octet-stream' : 'application/json', ...(identity ? { Authorization: `Bearer ${identity.token}`, 'X-Tenant-ID': tenant } : {}), ...headers }, body: body === undefined ? undefined : binary ? body : JSON.stringify(body), signal: AbortSignal.timeout(120000) })
|
||
|
|
const raw = await response.text(); let result
|
||
|
|
try { result = JSON.parse(raw) } catch { result = { message: raw.slice(0, 300) } }
|
||
|
|
assert.ok((Array.isArray(expected) ? expected : [expected]).includes(response.status), `${method} ${route}: HTTP ${response.status}; ${result.message || result.error?.message || ''}`)
|
||
|
|
return result
|
||
|
|
}
|
||
|
|
async function wait(id) {
|
||
|
|
for (let deadline = Date.now() + 180000; Date.now() < deadline;) {
|
||
|
|
const row = (await api(`/uploads/${id}`)).data
|
||
|
|
if (['completed', 'failed'].includes(row.state)) return row
|
||
|
|
await new Promise(resolve => setTimeout(resolve, 1000))
|
||
|
|
}
|
||
|
|
throw new Error('Native recovery timed out')
|
||
|
|
}
|
||
|
|
async function createUpload(kb, name, bytes, identity = admin) {
|
||
|
|
const row = (await api('/uploads', 'POST', { kind: 'knowledge', target_id: kb.id, name, size: bytes.length, sha256: hash(bytes) }, identity, 201)).data
|
||
|
|
fixtures.uploads.push({ id: row.id, identity })
|
||
|
|
await api(`/uploads/${row.id}/chunks`, 'PUT', bytes, identity, 200, { 'Upload-Offset': '0', 'Upload-Checksum': hash(bytes) })
|
||
|
|
return row
|
||
|
|
}
|
||
|
|
async function main() {
|
||
|
|
const credentials = JSON.parse(fs.readFileSync(`${directory}/super-admin-login.json`, 'utf8'))
|
||
|
|
admin = await api('/auth/login', 'POST', credentials, null)
|
||
|
|
const kbForm = { name: prefix, type: 'document', embedding_model_id: 'a2c3ce32-6192-46c8-8072-910abfe8afed', storage_backend_id: 'edcd8dc2-4e93-4bd4-839c-d835619fc0a6', indexing_strategy: { vector_enabled: true, keyword_enabled: true, wiki_enabled: false, graph_enabled: false } }
|
||
|
|
const kb = (await api('/knowledge-bases', 'POST', kbForm, admin, [200, 201])).data; fixtures.kbs.push(kb.id)
|
||
|
|
for (const status of ['pending', 'failed']) {
|
||
|
|
const bytes = Buffer.from(`Native crash recovery evidence ${status}: persimmon 492. This stored source must be indexed exactly once.`)
|
||
|
|
const upload = await createUpload(kb, `recovery-${status}.txt`, bytes)
|
||
|
|
const digestRows = (await api(`/uploads/${upload.id}/chunks?offset=0`)).data
|
||
|
|
check(`${status}_prefix_digest_matches`, digestRows.length === 1 && digestRows[0].sha256 === hash(bytes))
|
||
|
|
const id = uuid5(`resumable:${tenant}:${kb.id}:${upload.id}:0`); fixtures.documents.push(id)
|
||
|
|
// Reproduce the persisted native row and file just before queue delivery.
|
||
|
|
const relative = `${tenant}/${id}/recovery.txt`
|
||
|
|
fs.mkdirSync(path.dirname(`${directory}/files/${relative}`), { recursive: true })
|
||
|
|
fs.writeFileSync(`${directory}/files/${relative}`, bytes)
|
||
|
|
sql(`INSERT INTO knowledges (id,tenant_id,knowledge_base_id,type,channel,source,title,file_name,file_type,file_size,file_hash,file_path,parse_status,enable_status,embedding_model_id,metadata,created_at,updated_at) VALUES (${literal(id)},10059,${literal(kb.id)},'file','web','',${literal(`recovery-${status}.txt`)},${literal(`recovery-${status}.txt`)},'txt',${bytes.length},${literal(hash(bytes, 'md5'))},${literal(`local://${relative}`)},${literal(status)},'disabled',${literal(kbForm.embedding_model_id)},${literal(JSON.stringify({ resumable_upload_id: upload.id, resumable_upload_offset: '0' }))}::jsonb,now(),now())`)
|
||
|
|
await api(`/uploads/${upload.id}/finalize`, 'POST', {}, admin, 202)
|
||
|
|
const complete = await wait(upload.id)
|
||
|
|
check(`${status}_native_recovery_completes`, complete.state === 'completed')
|
||
|
|
check(`${status}_reuses_deterministic_document`, complete.result.knowledge_ids.length === 1 && complete.result.knowledge_ids[0] === id)
|
||
|
|
const chunks = (await api(`/chunks/${id}?page_size=20`)).data
|
||
|
|
check(`${status}_native_chunk_evidence`, chunks.some(row => row.content.includes('persimmon 492')))
|
||
|
|
check(`${status}_no_duplicate_native_document`, sql(`SELECT count(*) FROM knowledges WHERE metadata->>'resumable_upload_id'=${literal(upload.id)} AND deleted_at IS NULL`) === '1')
|
||
|
|
}
|
||
|
|
const role = (await api('/access/roles', 'POST', { name: prefix, permissions: ['knowledge.read', 'knowledge.write'] })).data; fixtures.roles.push(role.id)
|
||
|
|
const password = crypto.randomBytes(18).toString('base64url'), email = `${prefix}@example.test`
|
||
|
|
const user = (await api('/access/users', 'POST', { username: prefix, email, password, role: 'contributor', status: 'active', custom_role_id: role.id })).data; fixtures.users.push(user.id)
|
||
|
|
const member = await api('/auth/login', 'POST', { email, password }, null)
|
||
|
|
const memberKB = (await api('/knowledge-bases', 'POST', { ...kbForm, name: prefix + '-member' }, member, [200, 201])).data; fixtures.kbs.push(memberKB.id)
|
||
|
|
const upload = await createUpload(memberKB, 'revoke.txt', Buffer.from('revocation material'), member)
|
||
|
|
await api(`/access/roles/${role.id}`, 'PUT', { ...role, permissions: ['knowledge.read'] })
|
||
|
|
await api(`/uploads/${upload.id}`, 'GET', undefined, member, 403)
|
||
|
|
await api(`/uploads/${upload.id}/chunks?offset=0`, 'GET', undefined, member, 403)
|
||
|
|
await api(`/uploads/${upload.id}/chunks`, 'PUT', Buffer.from('revocation material'), member, 403, { 'Upload-Offset': '0', 'Upload-Checksum': hash(Buffer.from('revocation material')) })
|
||
|
|
await api(`/uploads/${upload.id}/finalize`, 'POST', {}, member, 403)
|
||
|
|
check('revocation_hides_list_and_blocks_read_write_finalize', !(await api('/uploads', 'GET', undefined, member)).data.some(row => row.id === upload.id))
|
||
|
|
await api(`/uploads/${upload.id}`, 'DELETE', undefined, member)
|
||
|
|
check('revoked_owner_can_release_staging', sql(`SELECT count(*) FROM resumable_upload_chunks WHERE upload_id=${literal(upload.id)}`) === '0' && !fs.existsSync(`${directory}/resumable-uploads/${upload.id}.upload`))
|
||
|
|
}
|
||
|
|
async function cleanup() {
|
||
|
|
for (const row of fixtures.uploads) try { await api(`/uploads/${row.id}`, 'DELETE', undefined, row.identity, [200, 404, 409]) } catch {}
|
||
|
|
for (const id of fixtures.kbs) try { await api(`/knowledge-bases/${id}`, 'DELETE', undefined, admin, [200, 202, 404]) } catch {}
|
||
|
|
for (const id of fixtures.users) try { await api(`/access/users/${id}`, 'DELETE', undefined, admin, [200, 404]) } catch {}
|
||
|
|
for (const id of fixtures.roles) try { await api(`/access/roles/${id}`, 'DELETE', undefined, admin, [200, 404]) } catch {}
|
||
|
|
for (const id of fixtures.documents) fs.rmSync(`${directory}/files/${tenant}/${id}`, { recursive: true, force: true })
|
||
|
|
}
|
||
|
|
main().catch(error => { checks.push({ name: 'failure', message: error.message }); console.error(error.message); process.exitCode = 1 }).finally(async () => {
|
||
|
|
await cleanup()
|
||
|
|
fs.writeFileSync(`${directory}/resumable-recovery-verification.json`, JSON.stringify({ prefix, checks, fixtures: { ...fixtures, uploads: fixtures.uploads.map(row => row.id) }, finished_at: new Date().toISOString() }, null, 2))
|
||
|
|
})
|