268 lines
18 KiB
JavaScript
268 lines
18 KiB
JavaScript
|
const assert = require('node:assert/strict')
|
||
|
|
const crypto = require('node:crypto')
|
||
|
|
const fs = require('node:fs')
|
||
|
|
const http = require('node:http')
|
||
|
|
const { execFileSync } = require('node:child_process')
|
||
|
|
|
||
|
|
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
|
||
|
|
const base = (process.env.VERIFY_API_ORIGIN || 'http://127.0.0.1:18080') + '/api/v1'
|
||
|
|
const origin = process.env.VERIFY_FRONTEND_ORIGIN || 'http://127.0.0.1:15173'
|
||
|
|
const tenant = 10059
|
||
|
|
const prefix = `content-review-qc-${Date.now()}`
|
||
|
|
const checks = [], failures = [], calls = []
|
||
|
|
const privateMarker = 'PRIVATE_REVIEW_FIXTURE_PERSONA'
|
||
|
|
const acceptedMarker = 'ACCEPTED_REVIEW_FIXTURE_PROSE'
|
||
|
|
const skippedMarker = 'SKIPPED_REVIEW_FIXTURE_PROSE'
|
||
|
|
const fixedSummary = 'LOCAL_DETERMINISTIC_REVIEW_PASSED'
|
||
|
|
const qualitySettings = { rules: 'LOCAL_RULES_CHECK_TRANSITIONS', style: 'LOCAL_STYLE_BEGINNER_FRIENDLY' }
|
||
|
|
const initialBalance = 1000000
|
||
|
|
const resourceID = crypto.randomUUID(), absentAgentID = crypto.randomUUID()
|
||
|
|
let admin, reviewer, role, user, model, job, browser, seeded = false
|
||
|
|
let activeRequests = 0
|
||
|
|
const literal = value => `'${String(value).replaceAll("'", "''")}'`
|
||
|
|
const sql = query => execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres',
|
||
|
|
'psql', '-X', '-q', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-At', '-v', 'ON_ERROR_STOP=1'],
|
||
|
|
{ input: query, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim()
|
||
|
|
const check = (name, value) => { checks.push({ name, passed: !!value }); assert.ok(value, name) }
|
||
|
|
const pause = ms => new Promise(resolve => setTimeout(resolve, ms))
|
||
|
|
|
||
|
|
async function runTask(action, version, parameters = {}) {
|
||
|
|
job = (await api(`/workbench/resources/content-tasks/${resourceID}/actions/${action}`, 'POST', { version, ...parameters }, reviewer, 202)).data
|
||
|
|
return awaitTask(action)
|
||
|
|
}
|
||
|
|
|
||
|
|
async function awaitTask(action) {
|
||
|
|
const deadline = Date.now() + 45000
|
||
|
|
while (Date.now() < deadline) {
|
||
|
|
job = (await api(`/workbench/jobs/${job.id}`, 'GET', undefined, reviewer)).data
|
||
|
|
if (job.status === 'succeeded') break
|
||
|
|
assert.ok(['queued', 'running'].includes(job.status), `${action} job ${job.status}: ${job.error || ''}`)
|
||
|
|
await pause(400)
|
||
|
|
}
|
||
|
|
check(`${action}_queue_job_succeeded`, job.status === 'succeeded' && job.progress === 100)
|
||
|
|
return (await api(`/workbench/resources/content-tasks/${resourceID}`, 'GET', undefined, reviewer)).data
|
||
|
|
}
|
||
|
|
|
||
|
|
async function configureThroughBrowser() {
|
||
|
|
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
|
||
|
|
browser = await chromium.launch({ headless: true, channel: 'chrome' })
|
||
|
|
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } })
|
||
|
|
await context.addInitScript(({ identity, tenant }) => {
|
||
|
|
const values = { weknora_user: JSON.stringify({ ...identity.user, tenant_id: String(tenant) }),
|
||
|
|
weknora_tenant: JSON.stringify({ ...identity.active_tenant, id: String(tenant) }),
|
||
|
|
weknora_token: identity.token, weknora_refresh_token: identity.refresh_token,
|
||
|
|
weknora_memberships: JSON.stringify(identity.memberships) }
|
||
|
|
for (const [key, value] of Object.entries(values)) localStorage.setItem(key, value)
|
||
|
|
localStorage.setItem('weknora:new-user-guide-done:v1', '1')
|
||
|
|
}, { identity: reviewer, tenant })
|
||
|
|
const page = await context.newPage(), browserErrors = [], mutations = []
|
||
|
|
page.on('pageerror', error => browserErrors.push(error.message))
|
||
|
|
page.on('response', response => {
|
||
|
|
if (response.status() >= 400 && new URL(response.url()).pathname.startsWith('/api/')) {
|
||
|
|
browserErrors.push(`${response.status()} ${new URL(response.url()).pathname}`)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
page.on('request', request => {
|
||
|
|
const pathname = new URL(request.url()).pathname
|
||
|
|
if (pathname.startsWith('/api/v1/workbench/resources/') && ['POST', 'PUT', 'PATCH', 'DELETE'].includes(request.method())) {
|
||
|
|
mutations.push({ method: request.method(), path: pathname })
|
||
|
|
}
|
||
|
|
})
|
||
|
|
await page.goto(`${origin}/platform/workbench/content?tab=tasks&resource=${resourceID}`)
|
||
|
|
await page.getByRole('heading', { name: prefix, exact: true }).waitFor()
|
||
|
|
check('reviewer_has_no_task_creation_or_configuration_controls',
|
||
|
|
await page.getByRole('button', { name: '创建内容任务', exact: true }).count() === 0 &&
|
||
|
|
await page.getByRole('button', { name: '编辑配置', exact: true }).count() === 0)
|
||
|
|
await page.getByRole('button', { name: '质检规则与风格', exact: true }).click()
|
||
|
|
const drawer = page.locator('.t-drawer--open').filter({ hasText: '质检规则与风格' })
|
||
|
|
await drawer.waitFor({ state: 'visible' })
|
||
|
|
const rules = drawer.locator('[aria-label="质检规则"] textarea')
|
||
|
|
const style = drawer.locator('[aria-label="表达风格"] textarea')
|
||
|
|
await rules.fill(qualitySettings.rules)
|
||
|
|
await style.fill(qualitySettings.style)
|
||
|
|
await page.screenshot({ path: `${directory}/content-review-qc-live-rules-desktop.png`, animations: 'disabled' })
|
||
|
|
await page.setViewportSize({ width: 390, height: 844 })
|
||
|
|
await page.screenshot({ path: `${directory}/content-review-qc-live-rules-mobile.png`, animations: 'disabled' })
|
||
|
|
check('quality_rules_drawer_mobile_has_no_horizontal_overflow', await drawer.evaluate(element =>
|
||
|
|
document.documentElement.scrollWidth <= innerWidth + 2 && Array.from(element.querySelectorAll('textarea')).every(input => {
|
||
|
|
const bounds = input.getBoundingClientRect()
|
||
|
|
return bounds.width > 0 && bounds.left >= 0 && bounds.right <= innerWidth + 2
|
||
|
|
})))
|
||
|
|
const actionPath = `/api/v1/workbench/resources/content-tasks/${resourceID}/actions/review-settings`
|
||
|
|
const responsePromise = page.waitForResponse(response => response.request().method() === 'POST' && new URL(response.url()).pathname === actionPath)
|
||
|
|
await drawer.getByRole('button', { name: '保存设置', exact: true }).click()
|
||
|
|
const response = await responsePromise
|
||
|
|
check('quality_rules_drawer_submits_authorized_review_action', response.status() === 202)
|
||
|
|
job = (await response.json()).data
|
||
|
|
const configured = await awaitTask('review-settings')
|
||
|
|
await drawer.waitFor({ state: 'hidden' })
|
||
|
|
await page.reload()
|
||
|
|
await page.getByRole('heading', { name: prefix, exact: true }).waitFor()
|
||
|
|
await page.getByRole('button', { name: '质检规则与风格', exact: true }).click()
|
||
|
|
await rules.waitFor({ state: 'visible' })
|
||
|
|
check('quality_rules_drawer_reloads_saved_rules_and_style',
|
||
|
|
await rules.inputValue() === qualitySettings.rules && await style.inputValue() === qualitySettings.style)
|
||
|
|
check('quality_rules_drawer_does_not_use_generic_resource_edit',
|
||
|
|
mutations.length === 1 && mutations[0].method === 'POST' && mutations[0].path === actionPath)
|
||
|
|
check('quality_rules_drawer_has_no_failed_requests_or_page_errors: ' + browserErrors.join(', '), browserErrors.length === 0)
|
||
|
|
await drawer.getByRole('button', { name: '取消', exact: true }).click()
|
||
|
|
await context.close()
|
||
|
|
await browser.close()
|
||
|
|
browser = undefined
|
||
|
|
return configured
|
||
|
|
}
|
||
|
|
|
||
|
|
async function api(route, method = 'GET', body, identity = admin, expected = 200) {
|
||
|
|
const response = await fetch(base + route, { method, headers: { 'Content-Type': 'application/json',
|
||
|
|
...(identity ? { Authorization: `Bearer ${identity.token}`, 'X-Tenant-ID': String(tenant) } : {}) },
|
||
|
|
body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(30000) })
|
||
|
|
const data = await response.json()
|
||
|
|
assert.ok((Array.isArray(expected) ? expected : [expected]).includes(response.status),
|
||
|
|
`${method} ${route}: HTTP ${response.status}; ${data.message || data.error?.message || ''}`)
|
||
|
|
return data
|
||
|
|
}
|
||
|
|
|
||
|
|
const server = http.createServer(async (request, response) => {
|
||
|
|
activeRequests++
|
||
|
|
try {
|
||
|
|
assert.equal(request.method, 'POST')
|
||
|
|
assert.equal(request.url, '/v1/chat/completions')
|
||
|
|
let body = ''
|
||
|
|
for await (const part of request) body += part
|
||
|
|
const input = JSON.parse(body)
|
||
|
|
calls.push(input)
|
||
|
|
assert.ok(!input.stream, 'quality review uses non-stream completion')
|
||
|
|
response.writeHead(200, { 'Content-Type': 'application/json' })
|
||
|
|
response.end(JSON.stringify({ id: prefix, object: 'chat.completion', created: Math.floor(Date.now() / 1000),
|
||
|
|
model: input.model,
|
||
|
|
choices: [{ index: 0, message: { role: 'assistant', content: JSON.stringify({ passed: true, summary: fixedSummary, issues: [] }) }, finish_reason: 'stop' }],
|
||
|
|
usage: { prompt_tokens: 31, completion_tokens: 19, total_tokens: 50 } }))
|
||
|
|
} catch (error) {
|
||
|
|
failures.push('local model fixture: ' + error.message)
|
||
|
|
response.writeHead(500, { 'Content-Type': 'application/json' })
|
||
|
|
response.end(JSON.stringify({ error: { message: 'local fixture failed' } }))
|
||
|
|
} finally {
|
||
|
|
activeRequests--
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
async function main() {
|
||
|
|
const credentials = JSON.parse(fs.readFileSync(`${directory}/super-admin-login.json`, 'utf8'))
|
||
|
|
admin = await api('/auth/login', 'POST', { email: credentials.email, password: credentials.password }, null)
|
||
|
|
await new Promise(resolve => server.listen(0, '0.0.0.0', resolve))
|
||
|
|
model = (await api('/models', 'POST', { name: prefix, type: 'KnowledgeQA', source: 'remote', parameters: {
|
||
|
|
base_url: `http://host.docker.internal:${server.address().port}/v1`, api_key: 'local-fixture-only', provider: 'generic',
|
||
|
|
context_window: 32768, max_output_tokens: 512,
|
||
|
|
} }, admin, 201)).data
|
||
|
|
role = (await api('/access/roles', 'POST', { name: prefix, permissions: ['content.read', 'content.review'] })).data
|
||
|
|
check('reviewer_role_uses_explicit_permission_version', role.permission_version === 2)
|
||
|
|
const password = crypto.randomBytes(18).toString('base64url')
|
||
|
|
const form = { username: prefix, email: `${prefix}@example.test`, password, role: 'contributor', status: 'active', custom_role_id: role.id }
|
||
|
|
user = (await api('/access/users', 'POST', form)).data
|
||
|
|
reviewer = await api('/auth/login', 'POST', { email: form.email, password }, null)
|
||
|
|
const permissions = (await api('/access/me', 'GET', undefined, reviewer)).data.permissions
|
||
|
|
check('reviewer_has_exactly_content_read_and_review', JSON.stringify([...permissions].sort()) === JSON.stringify(['content.read', 'content.review']))
|
||
|
|
await api('/agents', 'GET', undefined, reviewer, 403)
|
||
|
|
check('reviewer_cannot_read_agents', true)
|
||
|
|
|
||
|
|
// A missing original Agent and flow must not prevent reviewing recorded prose.
|
||
|
|
const data = { agent_id: absentAgentID, flow_id: crypto.randomUUID(), duration_minutes: 135,
|
||
|
|
segments: [
|
||
|
|
{ id: 'accepted', title: 'Accepted fixture', agent_id: absentAgentID, content: acceptedMarker, status: 'accepted' },
|
||
|
|
{ id: 'skipped', title: 'Skipped fixture', agent_id: absentAgentID, content: skippedMarker, status: 'skipped' },
|
||
|
|
], agent_snapshots: { [absentAgentID]: { name: 'Unavailable private writer', model_id: model.id,
|
||
|
|
system_prompt: privateMarker, instructions: `${privateMarker}_SKILLS`, skill_ids: [crypto.randomUUID()],
|
||
|
|
kb_selection_mode: 'selected', knowledge_bases: [crypto.randomUUID()], memory_enabled: true,
|
||
|
|
context_window: 32768, output_limit: 512,
|
||
|
|
fusions: [{ id: crypto.randomUUID(), name: 'Private fusion', system_prompt: `${privateMarker}_FUSION` }],
|
||
|
|
fusion_memories: [{ content: `${privateMarker}_MEMORY` }], captured_at: new Date().toISOString(),
|
||
|
|
} } }
|
||
|
|
sql(`BEGIN;
|
||
|
|
INSERT INTO workbench_resources(id,tenant_id,owner_id,owner_name,kind,name,status,version,data,created_at,updated_at)
|
||
|
|
VALUES(${literal(resourceID)},${tenant},${literal(user.id)},${literal(prefix)},'content-tasks',${literal(prefix)},'draft',1,${literal(JSON.stringify(data))}::jsonb,now(),now());
|
||
|
|
INSERT INTO workbench_balances(tenant_id,user_id,balance,reserved,updated_at)
|
||
|
|
VALUES(${tenant},${literal(user.id)},${initialBalance},0,now());
|
||
|
|
COMMIT;`)
|
||
|
|
seeded = true
|
||
|
|
check('original_agent_is_absent', sql(`SELECT count(*) FROM custom_agents WHERE id=${literal(absentAgentID)}`) === '0')
|
||
|
|
await api(`/workbench/resources/content-tasks/${resourceID}`, 'GET', undefined, reviewer)
|
||
|
|
const configured = await configureThroughBrowser()
|
||
|
|
check('review_settings_saved_without_generation_access', configured.version === 2 && JSON.stringify(configured.data.quality_settings) === JSON.stringify(qualitySettings) && calls.length === 0)
|
||
|
|
check('duration_survives_review_settings_and_reload', configured.data.duration_minutes === 135)
|
||
|
|
const task = await runTask('review', configured.version)
|
||
|
|
check('quality_result_saved_and_manuscript_hash_recorded', task.status === 'reviewed' && task.version === 3 &&
|
||
|
|
task.data.quality?.passed === true && task.data.quality.summary.includes(fixedSummary) &&
|
||
|
|
/^[a-f0-9]{64}$/.test(task.data.quality.manuscript_hash) && /^[a-f0-9]{64}$/.test(task.data.quality.settings_hash) && task.data.review.includes(fixedSummary))
|
||
|
|
check('only_one_local_model_call', calls.length === 1)
|
||
|
|
check('review_prompt_has_builtin_rules_and_accepted_prose_only', calls[0].messages.length === 2 &&
|
||
|
|
calls[0].messages[0].role === 'system' && calls[0].messages[1].role === 'user' &&
|
||
|
|
calls[0].messages[1].content.includes(acceptedMarker) && !JSON.stringify(calls[0]).includes(skippedMarker) &&
|
||
|
|
!JSON.stringify(calls[0]).includes(privateMarker))
|
||
|
|
check('review_receives_saved_rules_and_style', calls[0].messages[1].content.includes(qualitySettings.rules) && calls[0].messages[1].content.includes(qualitySettings.style))
|
||
|
|
check('review_receives_duration_as_whole_task_target', calls[0].messages[1].content.includes('135 分钟') &&
|
||
|
|
calls[0].messages[1].content.includes('不得把局部窗口视作完整任务') && task.data.quality.duration_minutes === 135)
|
||
|
|
const usage = JSON.parse(sql(`SELECT coalesce(json_agg(u),'[]') FROM workbench_usage u WHERE model_id=${literal(model.id)}`))
|
||
|
|
check('usage_belongs_to_reviewer_and_review_job', usage.length === 1 && usage[0].owner_id === user.id &&
|
||
|
|
usage[0].owner_id !== admin.user.id && usage[0].job_id === job.id && usage[0].agent_id === absentAgentID &&
|
||
|
|
usage[0].status === 'succeeded' && usage[0].input_tokens === 31 && usage[0].output_tokens === 19 && usage[0].points === 500)
|
||
|
|
const balance = JSON.parse(sql(`SELECT row_to_json(b) FROM workbench_balances b WHERE tenant_id=${tenant} AND user_id=${literal(user.id)}`))
|
||
|
|
check('reviewer_balance_settled_and_reservation_released', balance.balance === initialBalance - 500 && balance.reserved === 0)
|
||
|
|
sql(`UPDATE workbench_balances SET balance=0 WHERE tenant_id=${tenant} AND user_id=${literal(user.id)}`)
|
||
|
|
const composed = await runTask('compose', task.version)
|
||
|
|
check('compose_preserves_accepted_text_without_model_or_balance', composed.status === 'completed' && composed.data.composed === acceptedMarker && calls.length === 1)
|
||
|
|
check('composition_preserves_duration', composed.data.duration_minutes === 135)
|
||
|
|
const changed = await runTask('review-settings', composed.version, { quality_settings: { ...qualitySettings, rules: 'CHANGED_RULES' } })
|
||
|
|
check('changed_settings_clear_approval_and_final_but_keep_accepted_prose', !changed.data.quality && !changed.data.composed && !changed.data.review && changed.data.segments[0].status === 'accepted' && changed.data.segments[0].content === acceptedMarker)
|
||
|
|
check('non_ai_actions_create_no_additional_usage', sql(`SELECT count(*) FROM workbench_usage WHERE model_id=${literal(model.id)}`) === '1')
|
||
|
|
await api('/workbench/operations/records/usage', 'GET', undefined, reviewer, 403)
|
||
|
|
check('review_does_not_grant_admin_usage_visibility', true)
|
||
|
|
}
|
||
|
|
|
||
|
|
async function cleanup() {
|
||
|
|
if (job && ['queued', 'running'].includes(job.status)) {
|
||
|
|
await api(`/workbench/jobs/${job.id}/cancel`, 'POST', {}, reviewer, [200, 409])
|
||
|
|
await pause(1500)
|
||
|
|
}
|
||
|
|
if (user) {
|
||
|
|
const deadline = Date.now() + 15000
|
||
|
|
while (Date.now() < deadline) {
|
||
|
|
const unsettled = sql(`SELECT count(*) FROM workbench_balance_holds WHERE tenant_id=${tenant} AND user_id=${literal(user.id)} AND status='reserved'`)
|
||
|
|
if (unsettled === '0' && activeRequests === 0) break
|
||
|
|
await pause(300)
|
||
|
|
}
|
||
|
|
assert.equal(sql(`SELECT count(*) FROM workbench_balance_holds WHERE tenant_id=${tenant} AND user_id=${literal(user.id)} AND status='reserved'`), '0', 'fixture cleanup requires settled model calls')
|
||
|
|
}
|
||
|
|
if (seeded) {
|
||
|
|
assert.equal(sql(`SELECT count(*) FROM workbench_jobs WHERE resource_id=${literal(resourceID)} AND status IN ('queued','running')`), '0', 'fixture cleanup requires inactive jobs')
|
||
|
|
sql(`BEGIN;
|
||
|
|
DELETE FROM workbench_events WHERE tenant_id=${tenant} AND resource_id=${literal(resourceID)};
|
||
|
|
DELETE FROM workbench_revisions WHERE tenant_id=${tenant} AND resource_id=${literal(resourceID)};
|
||
|
|
DELETE FROM workbench_jobs WHERE tenant_id=${tenant} AND resource_id=${literal(resourceID)};
|
||
|
|
DELETE FROM workbench_resources WHERE tenant_id=${tenant} AND id=${literal(resourceID)} AND owner_id=${literal(user.id)};
|
||
|
|
DELETE FROM workbench_transactions WHERE tenant_id=${tenant} AND user_id=${literal(user.id)};
|
||
|
|
DELETE FROM workbench_usage WHERE tenant_id=${tenant} AND owner_id=${literal(user.id)};
|
||
|
|
DELETE FROM workbench_balance_holds WHERE tenant_id=${tenant} AND user_id=${literal(user.id)};
|
||
|
|
DELETE FROM workbench_balances WHERE tenant_id=${tenant} AND user_id=${literal(user.id)};
|
||
|
|
COMMIT;`)
|
||
|
|
}
|
||
|
|
if (model) sql(`DELETE FROM models WHERE id=${literal(model.id)} AND name=${literal(prefix)}`)
|
||
|
|
if (user) await api(`/access/users/${user.id}`, 'DELETE')
|
||
|
|
if (role) await api(`/access/roles/${role.id}`, 'DELETE')
|
||
|
|
check('disposable_resource_model_and_role_cleaned', sql(`SELECT
|
||
|
|
(SELECT count(*) FROM workbench_resources WHERE id=${literal(resourceID)}) +
|
||
|
|
(SELECT count(*) FROM models WHERE name=${literal(prefix)}) +
|
||
|
|
(SELECT count(*) FROM workbench_access_roles WHERE name=${literal(prefix)})`) === '0')
|
||
|
|
}
|
||
|
|
|
||
|
|
main().catch(error => { failures.push(error.message); process.exitCode = 1 }).finally(async () => {
|
||
|
|
if (browser) try { await browser.close() } catch (error) { failures.push('browser close: ' + error.message); process.exitCode = 1 }
|
||
|
|
try { await cleanup() } catch (error) { failures.push('cleanup: ' + error.message); process.exitCode = 1 }
|
||
|
|
if (server.listening) await new Promise(resolve => server.close(resolve))
|
||
|
|
const report = { time: new Date().toISOString(), checks, failures, passed: failures.length === 0,
|
||
|
|
origin, api_origin: new URL(base).origin,
|
||
|
|
model_calls: calls.length, model: 'local deterministic HTTP fixture; no paid model calls',
|
||
|
|
reviewer_permissions: ['content.read', 'content.review'], review_job_id: job?.id }
|
||
|
|
fs.writeFileSync(`${directory}/content-review-qc-live.json`, JSON.stringify(report, null, 2))
|
||
|
|
console.log(JSON.stringify(report, null, 2))
|
||
|
|
})
|