239 lines
14 KiB
JavaScript
239 lines
14 KiB
JavaScript
|
const startedAt = new Date().toISOString()
|
||
|
|
const assert = require('node:assert/strict')
|
||
|
|
const fs = require('node:fs')
|
||
|
|
const crypto = require('node:crypto')
|
||
|
|
const { execFileSync } = require('node:child_process')
|
||
|
|
const { cleanupIndexedFixtureTags } = require('./indexed_fixture_tag_cleanup.cjs')
|
||
|
|
|
||
|
|
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
|
||
|
|
const base = process.env.ISOLATION_API_URL || 'https://agent.pducv.cn/api/v1'
|
||
|
|
const prefix = `indexed-isolation-${Date.now()}`
|
||
|
|
const embeddingModelID = 'a2c3ce32-6192-46c8-8072-910abfe8afed'
|
||
|
|
const collection = 'yawan_acceptance_hybrid_512'
|
||
|
|
const checks = [], failures = [], users = [], roles = [], kbs = [], docs = [], tags = []
|
||
|
|
const cleanup = [], timings = {}
|
||
|
|
const permittedRerankModelID = process.env.INDEXED_RERANK_MODEL_ID || ''
|
||
|
|
let retrievalModel = null
|
||
|
|
let scopedSearchRequests = 0
|
||
|
|
let admin
|
||
|
|
|
||
|
|
function check(name, passed, details = {}) {
|
||
|
|
checks.push({name, passed: !!passed, ...details})
|
||
|
|
console.log(`${passed ? 'PASS' : 'FAIL'} ${name}`)
|
||
|
|
if (!passed) failures.push(name)
|
||
|
|
}
|
||
|
|
|
||
|
|
async function raw(route, method = 'GET', body, identity = admin) {
|
||
|
|
if (route === '/knowledge-search') {
|
||
|
|
assert.ok(++scopedSearchRequests <= 16, 'Indexed fixture exceeded its scoped-search request budget')
|
||
|
|
assert.ok(typeof body?.query === 'string' && body.query.length <= 512, 'Indexed fixture query must remain small')
|
||
|
|
}
|
||
|
|
const form = body instanceof FormData
|
||
|
|
const start = Date.now()
|
||
|
|
const response = await fetch(base + route, {
|
||
|
|
method,
|
||
|
|
headers: {
|
||
|
|
...(form ? {} : {'Content-Type': 'application/json'}),
|
||
|
|
...(identity ? {Authorization: `Bearer ${identity.token}`, 'X-Tenant-ID': '10059'} : {}),
|
||
|
|
},
|
||
|
|
body: body === undefined ? undefined : form ? body : JSON.stringify(body),
|
||
|
|
signal: AbortSignal.timeout(45000),
|
||
|
|
})
|
||
|
|
let data
|
||
|
|
try { data = await response.json() } catch { data = null }
|
||
|
|
return {status: response.status, data, duration_ms: Date.now() - start}
|
||
|
|
}
|
||
|
|
|
||
|
|
async function api(route, method = 'GET', body, identity = admin) {
|
||
|
|
const response = await raw(route, method, body, identity)
|
||
|
|
assert.ok(response.status >= 200 && response.status < 300, `${method} ${route}: HTTP ${response.status}`)
|
||
|
|
return response.data
|
||
|
|
}
|
||
|
|
|
||
|
|
async function until(name, fn, timeout = 180000) {
|
||
|
|
const start = Date.now()
|
||
|
|
while (Date.now() - start < timeout) {
|
||
|
|
const result = await fn()
|
||
|
|
if (result) return result
|
||
|
|
await new Promise(resolve => setTimeout(resolve, 1000))
|
||
|
|
}
|
||
|
|
throw new Error(`${name} timed out after ${timeout / 1000}s`)
|
||
|
|
}
|
||
|
|
|
||
|
|
async function vectorRows(kbID) {
|
||
|
|
const response = await fetch('http://127.0.0.1:19530/v2/vectordb/entities/query', {
|
||
|
|
method: 'POST', headers: {'Content-Type': 'application/json'},
|
||
|
|
body: JSON.stringify({collectionName: collection, filter: `knowledge_base_id == "${kbID}"`,
|
||
|
|
outputFields: ['id', 'knowledge_id', 'chunk_id', 'knowledge_base_id'], limit: 100,
|
||
|
|
consistencyLevel: 'Strong'}),
|
||
|
|
signal: AbortSignal.timeout(15000),
|
||
|
|
})
|
||
|
|
assert.equal(response.status, 200, 'Milvus query HTTP status')
|
||
|
|
const result = await response.json()
|
||
|
|
assert.equal(result.code, 0, 'Milvus query result code')
|
||
|
|
assert.ok(Array.isArray(result.data), 'Milvus query returns rows')
|
||
|
|
return result.data
|
||
|
|
}
|
||
|
|
|
||
|
|
const literal = value => `'${String(value).replaceAll("'", "''")}'`
|
||
|
|
function sql(statement) {
|
||
|
|
return execFileSync('docker', ['--context', 'colima-yawan', 'exec', 'WeKnora-postgres',
|
||
|
|
'psql', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-v', 'ON_ERROR_STOP=1',
|
||
|
|
'-Atc', statement], {encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe']}).trim()
|
||
|
|
}
|
||
|
|
|
||
|
|
async function main() {
|
||
|
|
const credentials = JSON.parse(fs.readFileSync(`${directory}/super-admin-login.json`, 'utf8'))
|
||
|
|
admin = await api('/auth/login', 'POST', {email: credentials.email, password: credentials.password}, null)
|
||
|
|
const models = (await api('/models')).data
|
||
|
|
const embedding = models.find(model => model.id === embeddingModelID)
|
||
|
|
assert.ok(embedding, 'Local embedding model exists')
|
||
|
|
assert.ok(['localhost', '127.0.0.1', 'host.docker.internal'].includes(new URL(embedding.parameters.base_url).hostname), 'Embedding endpoint is local')
|
||
|
|
const configuredRerankID = sql("SELECT COALESCE(retrieval_config->>'rerank_model_id','') FROM tenants WHERE id=10059")
|
||
|
|
const selectedRerank = configuredRerankID
|
||
|
|
? models.find(model => model.id === configuredRerankID)
|
||
|
|
: models.find(model => model.type === 'Rerank')
|
||
|
|
assert.ok(!configuredRerankID || selectedRerank?.type === 'Rerank', 'Configured rerank model must exist')
|
||
|
|
if (selectedRerank) {
|
||
|
|
assert.equal(selectedRerank.id, permittedRerankModelID, 'Explicit INDEXED_RERANK_MODEL_ID is required for configured rerank calls')
|
||
|
|
retrievalModel = { id: selectedRerank.id, type: selectedRerank.type, provider_host: new URL(selectedRerank.parameters.base_url).hostname }
|
||
|
|
} else {
|
||
|
|
assert.equal(permittedRerankModelID, '', 'Requested rerank model is not selected by the live retrieval configuration')
|
||
|
|
}
|
||
|
|
check(selectedRerank ? 'local_embedding_and_explicit_bounded_rerank_no_generation' : 'local_embedding_only_no_generation_or_rerank', true)
|
||
|
|
|
||
|
|
const role = (await api('/access/roles', 'POST', {name: prefix,
|
||
|
|
permissions: ['knowledge.read', 'knowledge.write', 'chat.read', 'chat.write']})).data
|
||
|
|
roles.push(role.id)
|
||
|
|
async function member(label) {
|
||
|
|
const password = crypto.randomBytes(18).toString('base64url')
|
||
|
|
const email = `${prefix}-${label}@example.test`
|
||
|
|
const user = (await api('/access/users', 'POST', {username: `${prefix}-${label}`, email, password,
|
||
|
|
role: 'contributor', status: 'active', custom_role_id: role.id})).data
|
||
|
|
users.push(user.id)
|
||
|
|
return api('/auth/login', 'POST', {email, password}, null)
|
||
|
|
}
|
||
|
|
const alice = await member('owner'), bob = await member('reader')
|
||
|
|
const kb = (await api('/knowledge-bases', 'POST', {name: prefix, type: 'document',
|
||
|
|
embedding_model_id: embeddingModelID, storage_backend_id: 'edcd8dc2-4e93-4bd4-839c-d835619fc0a6',
|
||
|
|
summary_model_id: '', chunking_config: {chunk_size: 512, chunk_overlap: 0},
|
||
|
|
indexing_strategy: {vector_enabled: true, keyword_enabled: true, wiki_enabled: false, graph_enabled: false},
|
||
|
|
question_generation_config: {enabled: false}, auto_tag_config: {enabled: false}}, alice)).data
|
||
|
|
kbs.push(kb.id)
|
||
|
|
check('indexed_knowledge_owned_by_fixture_user', kb.creator_id === alice.user.id)
|
||
|
|
const tag = (await api(`/knowledge-bases/${kb.id}/tags`, 'POST', {
|
||
|
|
name: `${prefix}-category`, color: '#64748b', sort_order: 0,
|
||
|
|
}, alice)).data
|
||
|
|
tags.push(tag.id)
|
||
|
|
const emptyTag = (await api(`/knowledge-bases/${kb.id}/tags`, 'POST', {
|
||
|
|
name: `${prefix}-empty-category`, color: '#64748b', sort_order: 1,
|
||
|
|
}, alice)).data
|
||
|
|
tags.push(emptyTag.id)
|
||
|
|
const marker = `archive-cobalt-${crypto.randomBytes(8).toString('hex')}`
|
||
|
|
const content = `Indexed retrieval isolation acceptance. The archive code is ${marker}. This small private document verifies indexed owner retrieval and explicit resource grants. The archive code identifies this disposable fixture only.`
|
||
|
|
const form = new FormData()
|
||
|
|
form.set('file', new Blob([content], {type: 'text/plain'}), `${prefix}.txt`)
|
||
|
|
form.set('tag_ids', tag.id)
|
||
|
|
const indexStart = Date.now()
|
||
|
|
const doc = (await api(`/knowledge-bases/${kb.id}/knowledge/file`, 'POST', form, alice)).data
|
||
|
|
docs.push(doc.id)
|
||
|
|
const indexed = await until('native ingestion', async () => {
|
||
|
|
const current = (await api(`/knowledge/${doc.id}`, 'GET', undefined, alice)).data
|
||
|
|
assert.ok(!['failed', 'cancelled'].includes(current.parse_status), `Ingestion status ${current.parse_status}`)
|
||
|
|
return current.parse_status === 'completed' ? current : null
|
||
|
|
})
|
||
|
|
timings.indexing_ms = Date.now() - indexStart
|
||
|
|
check('real_tag_bound_to_uploaded_document', Array.isArray(indexed.tags) && indexed.tags.some(item => item.id === tag.id))
|
||
|
|
const chunks = (await api(`/chunks/${doc.id}?page_size=100`, 'GET', undefined, alice)).data
|
||
|
|
const fixtureChunks = chunks.filter(chunk => chunk.content?.includes(marker))
|
||
|
|
check('api_upload_parsed_into_real_chunks', fixtureChunks.length > 0, {chunk_count: fixtureChunks.length})
|
||
|
|
assert.ok(fixtureChunks.length, 'Fixture chunk required for positive and negative controls')
|
||
|
|
const milvus = await until('native vector index', async () => {
|
||
|
|
const rows = await vectorRows(kb.id)
|
||
|
|
return rows.some(row => row.knowledge_id === doc.id && fixtureChunks.some(chunk => chunk.id === row.chunk_id)) ? rows : null
|
||
|
|
})
|
||
|
|
check('real_milvus_index_contains_fixture_chunk', true, {vector_count: milvus.length, collection})
|
||
|
|
|
||
|
|
const searchBody = {query_text: content, match_count: 5, vector_threshold: 0, keyword_threshold: 0,
|
||
|
|
disable_keywords_match: true, disable_vector_match: false, skip_context_enrichment: true}
|
||
|
|
function hit(response) {
|
||
|
|
return response.status === 200 && Array.isArray(response.data?.data) && response.data.data.some(row =>
|
||
|
|
row.knowledge_id === doc.id && row.content?.includes(marker))
|
||
|
|
}
|
||
|
|
async function directVector(label, identity, expectedVisible) {
|
||
|
|
const response = await raw(`/knowledge-bases/${kb.id}/hybrid-search`, 'POST', searchBody, identity)
|
||
|
|
const denied = [403, 404].includes(response.status) ||
|
||
|
|
(response.status === 200 && Array.isArray(response.data?.data) && response.data.data.length === 0)
|
||
|
|
check(`${label}_direct_vector_${expectedVisible ? 'hits' : 'denied'}`, expectedVisible ? hit(response) : denied,
|
||
|
|
{status: response.status, hit_count: response.data?.data?.length ?? null, fixture_hit: hit(response), duration_ms: response.duration_ms})
|
||
|
|
}
|
||
|
|
const scopes = [
|
||
|
|
['kb', {knowledge_base_ids: [kb.id]}],
|
||
|
|
['document', {knowledge_ids: [doc.id]}],
|
||
|
|
['tag', {knowledge_base_ids: [kb.id], tag_ids: [tag.id]}],
|
||
|
|
]
|
||
|
|
async function scopedSearch(label, identity, expectedVisible) {
|
||
|
|
for (const [scope, payload] of scopes) {
|
||
|
|
const response = await raw('/knowledge-search', 'POST', {query: content, ...payload}, identity)
|
||
|
|
const denied = [403, 404].includes(response.status) ||
|
||
|
|
(response.status === 200 && Array.isArray(response.data?.data) && response.data.data.length === 0)
|
||
|
|
check(`${label}_${scope}_${expectedVisible ? 'hits' : 'denied'}`, expectedVisible ? hit(response) : denied,
|
||
|
|
{status: response.status, hit_count: response.data?.data?.length ?? null, fixture_hit: hit(response), duration_ms: response.duration_ms})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
await directVector('owner', alice, true)
|
||
|
|
await scopedSearch('owner', alice, true)
|
||
|
|
const unrelatedTag = await raw('/knowledge-search', 'POST', {
|
||
|
|
query: content, knowledge_base_ids: [kb.id], tag_ids: [emptyTag.id],
|
||
|
|
}, alice)
|
||
|
|
check('owner_unrelated_tag_returns_no_fixture', unrelatedTag.status === 200 &&
|
||
|
|
Array.isArray(unrelatedTag.data?.data) && unrelatedTag.data.data.length === 0,
|
||
|
|
{status: unrelatedTag.status, hit_count: unrelatedTag.data?.data?.length ?? null})
|
||
|
|
await directVector('before_grant', bob, false)
|
||
|
|
await scopedSearch('before_grant', bob, false)
|
||
|
|
await api('/access/grants', 'POST', {resources: [{kind: 'knowledge', id: kb.id}], user_ids: [bob.user.id], action: 'grant'})
|
||
|
|
await directVector('after_grant', bob, true)
|
||
|
|
await scopedSearch('after_grant', bob, true)
|
||
|
|
await api('/access/grants', 'POST', {resources: [{kind: 'knowledge', id: kb.id}], user_ids: [bob.user.id], action: 'revoke'})
|
||
|
|
await directVector('after_revoke', bob, false)
|
||
|
|
await scopedSearch('after_revoke', bob, false)
|
||
|
|
await directVector('owner_after_revoke', alice, true)
|
||
|
|
await scopedSearch('owner_after_revoke', alice, true)
|
||
|
|
}
|
||
|
|
|
||
|
|
async function clean(name, operation) {
|
||
|
|
try { await operation(); cleanup.push({name, passed: true}); console.log(`CLEAN ${name}`) }
|
||
|
|
catch (error) { cleanup.push({name, passed: false, error: error.message}); failures.push(`cleanup ${name}: ${error.message}`) }
|
||
|
|
}
|
||
|
|
|
||
|
|
main().catch(error => {failures.push(error.message); console.error(error.message)}).finally(async () => {
|
||
|
|
if (admin) {
|
||
|
|
for (const id of kbs) await clean(`knowledge ${id}`, async () => {
|
||
|
|
const response = await raw(`/knowledge-bases/${id}`, 'DELETE')
|
||
|
|
assert.ok([200, 202, 204, 404].includes(response.status), `Delete KB HTTP ${response.status}`)
|
||
|
|
await until('knowledge deletion', async () => (await raw(`/knowledge-bases/${id}`)).status === 404)
|
||
|
|
await until('vector deletion', async () => (await vectorRows(id)).length === 0)
|
||
|
|
// Only publication rows belonging to this run's generated resources are removed.
|
||
|
|
sql(`DELETE FROM workbench_access_publications WHERE tenant_id=10059 AND resource_id=${literal(id)}`)
|
||
|
|
assert.equal(sql(`SELECT count(*) FROM knowledges WHERE tenant_id=10059 AND knowledge_base_id=${literal(id)} AND deleted_at IS NULL`), '0')
|
||
|
|
check('whole_knowledge_base_delete_removes_bound_and_empty_categories',
|
||
|
|
sql(`SELECT count(*) FROM knowledge_tags WHERE tenant_id=10059 AND knowledge_base_id=${literal(id)}`) === '0')
|
||
|
|
check('whole_knowledge_base_delete_removes_category_relations',
|
||
|
|
tags.length > 0 && sql(`SELECT count(*) FROM knowledge_tag_relations WHERE tag_id IN (${tags.map(literal).join(',')})`) === '0')
|
||
|
|
})
|
||
|
|
if (kbs.length) await clean('indexed fixture categories', () => cleanupIndexedFixtureTags({
|
||
|
|
tenant_id: 10059, started_at: startedAt, time: new Date().toISOString(), base, prefix,
|
||
|
|
fixtures: {users, roles, kbs, docs, tags}, cleanup,
|
||
|
|
}))
|
||
|
|
for (const id of users) await clean(`user ${id}`, () => api(`/access/users/${id}`, 'DELETE'))
|
||
|
|
for (const id of roles) await clean(`role ${id}`, () => api(`/access/roles/${id}`, 'DELETE'))
|
||
|
|
}
|
||
|
|
const report = {tenant_id: 10059, started_at: startedAt, time: new Date().toISOString(), base, prefix, fixtures: {users, roles, kbs, docs, tags},
|
||
|
|
scope: 'Two non-admin users in tenant 10059; real local embedding and Milvus indexed fixture; KB/document/tag search scopes; explicit grant and revoke. No cross-tenant or generated-answer claim.',
|
||
|
|
retrieval_model: retrievalModel, scoped_search_requests: scopedSearchRequests,
|
||
|
|
timings, checks, cleanup, failures, passed: failures.length === 0}
|
||
|
|
fs.writeFileSync(`${directory}/public-indexed-retrieval-isolation.json`, JSON.stringify(report, null, 2))
|
||
|
|
console.log(JSON.stringify({passed: report.passed, checks: checks.length, failures, cleanup: cleanup.length}))
|
||
|
|
if (failures.length) process.exitCode = 1
|
||
|
|
})
|