WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes
Anydoc / anydoc parser engine (push) Canceled after 0s
App / Format, vet, test, and build (push) Canceled after 0s
CLI / test (macos-latest / go 1.26) (push) Canceled after 0s
CLI / test (ubuntu-latest / go 1.26) (push) Canceled after 0s
CLI / test (windows-latest / go 1.26) (push) Canceled after 0s
Build and Push Docker Image / build-ui (push) Canceled after 0s
Build and Push Docker Image / build-docreader (push) Canceled after 0s
Build and Push Docker Image / build-sandbox (push) Canceled after 0s
Build and Push Docker Image / build-app (amd64, linux/amd64, ubuntu-latest) (push) Canceled after 0s
Build and Push Docker Image / build-app (arm64, linux/arm64, ubuntu-24.04-arm) (push) Canceled after 0s
DocReader / Python tests and Go client tests (push) Canceled after 0s
Build & Publish (dsh-weknora) / Typecheck, test, pack (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (pinned) (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (latest, advisory) (push) Canceled after 0s
Frontend / Test, type-check, and build (push) Canceled after 0s
Go Lint Cache / warm lint cache (.) (push) Canceled after 0s
Go Lint Cache / warm lint cache (cli) (push) Canceled after 0s
Go Lint Cache / warm lint cache (client) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.10) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.11) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.12) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.13) (push) Canceled after 0s
Build and Push Docker Image / merge (push) Canceled after 0s
Build & Publish (dsh-weknora) / Publish to npm (push) Canceled after 0s
Build & Publish (mcp-server) / Build sdist + wheel (push) Canceled after 0s
Build & Publish (mcp-server) / Publish to PyPI (push) Canceled after 0s

This commit is contained in:
yawan
2026-09-22 16:19:06 +08:00
commit f7fc992f57
3929 changed files with 1006607 additions and 0 deletions
+3
View File
@@ -0,0 +1,3 @@
FROM yawan-weknora-app:native-final2
COPY WeKnora-native-acceptance-next /app/WeKnora
COPY migrations /app/migrations
@@ -0,0 +1,4 @@
*
!WeKnora-native-acceptance-next
!migrations/
!migrations/**
+231
View File
@@ -0,0 +1,231 @@
#!/usr/bin/env python3
"""Back up, rehearse and import operational history into local acceptance."""
from __future__ import annotations
import argparse
import contextlib
import datetime
import hashlib
import json
import os
import shutil
import sqlite3
import subprocess
import tempfile
from pathlib import Path
from apply_weknora_identities import sql_run as mapping_sql
from weknora_asset_etl import LEDGER as ASSET_LEDGER
from weknora_business_etl import LEDGER as BUSINESS_LEDGER
from weknora_category_etl import save_json
from weknora_etl import consistent_snapshot
from weknora_identity_etl import IDENTITY_LEDGER, TARGET_DATABASE, TARGET_TENANT
from weknora_operations_etl import LEDGER, TABLES, operations_plan, operations_sql, safe_operations_summary
PRESERVED = ("workbench_resources", "workbench_jobs", "workbench_revisions", "workbench_balance_holds")
DOCKER = ["docker", "--context", "colima-yawan", "exec", "-i", "WeKnora-postgres"]
def sql_run(query):
result = subprocess.run(DOCKER + ["psql", "-X", "-qAt", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", TARGET_DATABASE],
input=query, text=True, capture_output=True, timeout=600)
if result.returncode:
# Do not expose source text included in SQL error contexts.
lines = [line for line in result.stderr.splitlines() if line.startswith("ERROR:")]
error = lines[0][:180] if lines and "Operations " in lines[0] else "database rejected operations transaction"
raise RuntimeError(error)
return result.stdout.strip()
def digest(path):
h = hashlib.sha256()
with path.open("rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024), b""):
h.update(chunk)
return h.hexdigest()
def quiescent(path):
if any(Path(str(path) + suffix).exists() for suffix in ("-wal", "-journal")):
raise RuntimeError("Operations fallback requires no source WAL or journal")
check = subprocess.run(["lsof", "-t", str(path)], capture_output=True, text=True, timeout=15)
if check.returncode != 1 or check.stdout.strip():
raise RuntimeError("Operations fallback requires no process holding the source database")
@contextlib.contextmanager
def source_snapshot(path):
"""Use SQLite backup; compensate for missing WAL sidecars on a quiet source.
Some SQLite builds cannot read a WAL-mode file read-only when sidecars are
absent. Only a byte-identical, verified quiet copy is opened read-write to
initialize its temporary sidecars. The original is never opened for writing.
"""
path = path.resolve()
manager = consistent_snapshot(path)
try:
db, info = manager.__enter__()
except sqlite3.OperationalError as error:
if "unable to open database file" not in str(error):
raise
else:
try:
yield db, info
finally:
manager.__exit__(None, None, None)
return
quiescent(path)
before = path.stat()
with tempfile.TemporaryDirectory(prefix="operations-frozen-source-") as root:
copied = Path(root) / "source.sqlite"
shutil.copyfile(path, copied)
fingerprint = digest(copied)
if digest(path) != fingerprint or before.st_mtime_ns != path.stat().st_mtime_ns:
raise RuntimeError("Operations source changed during quiet copy")
quiescent(path)
with sqlite3.connect(copied) as initializer:
initializer.execute("SELECT count(*) FROM sqlite_master").fetchone()
with consistent_snapshot(copied) as (db, info):
info.update(snapshot_method="verified_quiescent_copy_then_sqlite_backup", source_bytes=before.st_size,
source_mtime_ns=before.st_mtime_ns, original_file_sha256=fingerprint)
yield db, info
quiescent(path)
if (before.st_size, before.st_mtime_ns) != (path.stat().st_size, path.stat().st_mtime_ns) or digest(path) != fingerprint:
raise RuntimeError("Operations source changed during snapshot read")
def build_plan(db, mapping_file):
ids = json.loads(mapping_sql(f"SELECT json_object_agg(source_user_id,target_user_id) FROM {IDENTITY_LEDGER} WHERE source_system='yawan-workbench' AND source_organization_id=1 AND target_tenant_id={TARGET_TENANT};"))
agents = json.loads(mapping_sql(f"SELECT json_object_agg(source_id,target_id) FROM {ASSET_LEDGER} WHERE source_system='yawan-workbench' AND source_organization_id=1 AND source_kind='agent' AND target_tenant_id={TARGET_TENANT};"))
business = json.loads(mapping_sql(f"SELECT json_object_agg(source_kind||':'||source_id,target_id) FROM {BUSINESS_LEDGER} WHERE source_system='yawan-workbench' AND source_organization_id=1 AND target_tenant_id={TARGET_TENANT};"))
model_rows = json.loads(mapping_sql(f"SELECT coalesce(json_agg(json_build_object('provider_id',source_provider_id,'snapshot',source_snapshot,'target_id',target_model_id)),'[]') FROM yawan_legacy_model_mappings WHERE source_system='yawan-workbench' AND source_organization_id=1 AND target_tenant_id={TARGET_TENANT};"))
models = {}
for row in model_rows:
name = row["snapshot"].get("model_name", row["snapshot"].get("model", ""))
if name:
models[f"{row['provider_id']}:{name}"] = row["target_id"]
state = json.loads(mapping_file.read_text())
if state.get("tenant_id") != TARGET_TENANT:
raise ValueError("Operations KB mapping tenant differs")
return operations_plan(db, ids, agents, {k: v["id"] for k, v in state["knowledge_bases"].items()}, business, models)
def target_snapshot():
tables = (*TABLES, "workbench_history", *PRESERVED)
entries = ["'database',current_database()"]
for table in tables:
key = "tenant_id::text||':'||user_id" if table == "workbench_balances" else "id"
entries.append(f"'{table}',(SELECT coalesce(json_agg(json_build_object('id',{key},'hash',md5(to_jsonb(r)::text)) ORDER BY {key}),'[]') FROM {table} r)")
return json.loads(sql_run("SELECT json_build_object(" + ",".join(entries) + ");"))
def backup(path):
if path.exists():
raise RuntimeError("Operations backup exists")
args = DOCKER + ["pg_dump", "-U", "postgres", "-d", TARGET_DATABASE, "--format=custom"]
for table in (*TABLES, "workbench_history", *PRESERVED):
args.extend(["--table", table])
if mapping_sql(f"SELECT to_regclass('{LEDGER}') IS NOT NULL;") == "t":
args.extend(["--table", LEDGER])
with path.open("xb") as output:
os.chmod(path, 0o600)
result = subprocess.run(args, stdout=output, stderr=subprocess.PIPE, timeout=120)
if result.returncode or not path.stat().st_size:
raise RuntimeError("Operations target backup failed")
os.chmod(path, 0o400)
return {"path": str(path), "bytes": path.stat().st_size, "sha256": digest(path)}
def export_plan(path, plan):
with path.open("x") as out:
os.chmod(path, 0o600)
json.dump(plan, out, ensure_ascii=True, sort_keys=True, separators=(",", ":"))
os.chmod(path, 0o400)
return {"path": str(path), "bytes": path.stat().st_size, "sha256": digest(path), "mode": "0400"}
def transaction(query, commit=False):
sql = "BEGIN ISOLATION LEVEL SERIALIZABLE; SET LOCAL statement_timeout='180s'; SET LOCAL lock_timeout='5s';\n"
for i, table in enumerate((*TABLES, "workbench_history", *PRESERVED)):
sql += f"CREATE TEMP TABLE _ops_before_{i} ON COMMIT DROP AS SELECT * FROM {table};\n"
sql += query
for i, table in enumerate((*TABLES, "workbench_history", *PRESERVED)):
if table == "workbench_balances":
join = "n.tenant_id=b.tenant_id AND n.user_id=b.user_id"
else:
join = "n.id=b.id"
assertion = f"EXISTS (SELECT 1 FROM _ops_before_{i} b LEFT JOIN {table} n ON {join} WHERE to_jsonb(n) IS DISTINCT FROM to_jsonb(b))"
if table in PRESERVED:
assertion += f" OR (SELECT count(*) FROM {table})<>(SELECT count(*) FROM _ops_before_{i})"
sql += f"DO $preserve$ BEGIN IF {assertion} THEN RAISE EXCEPTION 'Operations existing data changed in {table}'; END IF; END $preserve$;\n"
return sql + ("COMMIT;" if commit else "ROLLBACK;")
def verify(plan, before, after):
checks = {"acceptance_only": before["database"] == after["database"] == TARGET_DATABASE}
for table in (*TABLES, "workbench_history", *PRESERVED):
old = {r["id"]: r["hash"] for r in before[table]}
new = {r["id"]: r["hash"] for r in after[table]}
checks[table + "_existing_unchanged"] = all(new.get(k) == v for k, v in old.items())
if table in PRESERVED:
checks[table + "_no_new"] = old == new
else:
if table == "workbench_history":
expected = {i["history_id"] for i in plan["items"]}
else:
expected = {f"{TARGET_TENANT}:{i['target_key']}" if table == "workbench_balances" else i["target_key"]
for i in plan["items"] if i["target_table"] == table}
checks[table + "_planned_additions_only"] = set(new) - set(old) == expected - set(old)
checks[table + "_all_planned_present"] = expected <= set(new)
return {"passed": all(checks.values()), "checks": checks}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("command", choices=("plan", "apply", "repeat"))
parser.add_argument("--sqlite", type=Path)
parser.add_argument("--kb-mapping", type=Path)
parser.add_argument("--export", type=Path)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
if args.out.exists():
parser.error("Operations checkpoint already exists")
if args.command == "repeat":
plan = json.loads(args.export.read_text())
info = {"sha256": plan["source_snapshot_sha256"], "source_mutated": False, "snapshot_method": "frozen_verified_export"}
else:
with source_snapshot(args.sqlite) as (db, info):
plan = build_plan(db, args.kb_mapping)
plan["source_snapshot_sha256"] = info["sha256"]
if info["source_mutated"]:
raise RuntimeError("Operations source snapshot changed")
query = operations_sql(plan)
result = {"checked_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "summary": safe_operations_summary(plan),
"snapshot": info, "committed": False}
if args.command != "repeat":
result["private_export"] = export_plan(args.out.with_suffix(".private.json"), plan)
save_json(args.out.with_suffix(".manifest.json"), result)
result["backup"] = backup(args.out.with_suffix(".backup.dump"))
before = target_snapshot()
save_json(args.out.with_suffix(".before.json"), before)
result["rehearsal"] = [json.loads(s) for s in sql_run(transaction(query)).splitlines() if s.startswith("{")]
result["rollback_unchanged"] = before == target_snapshot()
if not result["rollback_unchanged"]:
raise RuntimeError("Operations rollback rehearsal changed target")
if args.command in ("apply", "repeat"):
save_json(args.out.with_suffix(".attempt.json"), result)
result["applied"] = [json.loads(s) for s in sql_run(transaction(query, True)).splitlines() if s.startswith("{")]
result["committed"] = True
save_json(args.out.with_suffix(".committed.json"), result)
after = target_snapshot()
save_json(args.out.with_suffix(".after.json"), after)
result["verification"] = verify(plan, before, after)
if args.command == "repeat":
result["repeat_no_change"] = before == after and all(r.get("inserted_history") == 0 and r.get("inserted_native") == 0 for r in result["applied"])
save_json(args.out, result)
print(json.dumps(result, ensure_ascii=False))
return 0 if result.get("verification", {}).get("passed", True) and result.get("repeat_no_change", True) else 1
if __name__ == "__main__":
raise SystemExit(main())
+131
View File
@@ -0,0 +1,131 @@
#!/usr/bin/env python3
"""Apply a reviewed access plan only to the local acceptance database."""
from __future__ import annotations
import argparse
import datetime
import json
from pathlib import Path
from apply_weknora_identities import sql_run
from weknora_access_etl import ACCESS_TABLES, LEDGER, SOURCE, canonical, migration_sql
from weknora_category_etl import save_json
from weknora_identity_etl import TARGET_DATABASE, TARGET_TENANT
def snapshot():
fields = ["'database',current_database()",
"""'users',(SELECT coalesce(json_agg(json_build_object('id',id,'fingerprint',md5(to_jsonb(u)::text)) ORDER BY id),'[]') FROM users u)""",
"""'memberships',(SELECT coalesce(json_agg(json_build_object('id',id,'fingerprint',md5(to_jsonb(m)::text)) ORDER BY id),'[]') FROM tenant_members m)"""]
fields += [f"'{table}',(SELECT coalesce(json_agg(to_jsonb(a) ORDER BY to_jsonb(a)::text),'[]') FROM {table} a)" for table in ACCESS_TABLES]
result = json.loads(sql_run("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; SELECT json_build_object(" + ",".join(fields) + "); COMMIT;"))
exists = sql_run(f"SELECT to_regclass('{LEDGER}') IS NOT NULL;") == "t"
result[LEDGER] = json.loads(sql_run(f"SELECT coalesce(json_agg(to_jsonb(a) ORDER BY entity_key),'[]') FROM {LEDGER} a;")) if exists else []
return result
def verification(plan, before, after):
checks = {"acceptance_scope": before["database"] == after["database"] == TARGET_DATABASE,
"all_users_unchanged": before["users"] == after["users"],
"all_memberships_unchanged": before["memberships"] == after["memberships"],
"all_explicit_grants_unchanged": before[ACCESS_TABLES[3]] == after[ACCESS_TABLES[3]]}
expected = {
ACCESS_TABLES[0]: {r["id"] for r in plan["roles"]},
ACCESS_TABLES[1]: {(TARGET_TENANT, a["user_id"]) for a in plan["assignments"]},
ACCESS_TABLES[2]: {(TARGET_TENANT, p["kind"], p["resource_id"]) for p in plan["publications"]},
}
def key(table, row):
if table == ACCESS_TABLES[0]:
return row["id"]
return (row["tenant_id"], row["user_id"]) if table == ACCESS_TABLES[1] else (row["tenant_id"], row["kind"], row["resource_id"])
for table in expected:
old, new = ({key(table, r): r for r in state[table]} for state in (before, after))
checks[table + "_existing_unchanged"] = all(new.get(k) == r for k, r in old.items())
checks[table + "_planned_only"] = set(new) - set(old) == expected[table] - set(old)
saved = {r["entity_key"]: r["source_spec"] for r in after[LEDGER]
if r["source_system"] == SOURCE and r["target_tenant_id"] == TARGET_TENANT}
planned = {"role/" + r["source_role"]: r for r in plan["roles"]}
planned.update({"assignment/" + a["user_id"]: a for a in plan["assignments"]})
planned.update({"publication/" + p["kind"] + "/" + p["resource_id"]: p for p in plan["publications"]})
checks["durable_provenance"] = all(saved.get(k) == v for k, v in planned.items())
return {"passed": all(checks.values()), "checks": checks}
def rollback_sql(plan, before, after, commit=False):
migration_sql(plan) # Reuse the scope and plan-integrity validation.
inserted = {table: [r for r in after[table] if r not in before[table]] for table in (*ACCESS_TABLES, LEDGER)}
payload = "'" + canonical(inserted).replace("'", "''") + "'::jsonb"
parts = [f"""-- Removes only unchanged rows inserted by this reviewed application.
-- Default rehearsal. Emit COMMIT only after reviewing the current native edits.
BEGIN ISOLATION LEVEL SERIALIZABLE;
SET LOCAL statement_timeout='30s'; SET LOCAL lock_timeout='5s';
DO $scope$ BEGIN
IF current_database()<>'{TARGET_DATABASE}' THEN RAISE EXCEPTION 'Wrong rollback database'; END IF;
END $scope$;
SELECT pg_advisory_xact_lock({TARGET_TENANT},11905);
LOCK TABLE users,tenant_members,workbench_access_roles,workbench_access_assignments,
workbench_access_publications,workbench_access_grants,{LEDGER} IN SHARE ROW EXCLUSIVE MODE;
DO $suspended$ BEGIN
IF EXISTS(SELECT 1 FROM users u JOIN yawan_legacy_identity_mappings l ON l.target_user_id=u.id
JOIN tenant_members m ON m.user_id=u.id
WHERE l.target_tenant_id={TARGET_TENANT} AND l.source_user_id<>1
AND (u.is_active OR m.status<>'suspended' OR m.role<>'contributor' OR m.tenant_id<>{TARGET_TENANT}))
THEN RAISE EXCEPTION 'Account activation changed; review rollback policy first'; END IF;
END $suspended$;
CREATE TEMP TABLE _access_rollback_payload ON COMMIT DROP AS SELECT {payload} AS data;"""]
for table in (*ACCESS_TABLES, LEDGER):
parts.append(f"""DO $unchanged$ BEGIN
IF EXISTS(SELECT 1 FROM _access_rollback_payload p, jsonb_array_elements(p.data->'{table}') expected
WHERE NOT EXISTS(SELECT 1 FROM {table} actual WHERE to_jsonb(actual)=expected))
THEN RAISE EXCEPTION 'Native edits or missing rows in {table}; rollback aborted'; END IF;
END $unchanged$;""")
parts.append("""DO $references$ BEGIN
IF EXISTS(SELECT 1 FROM workbench_access_grants g JOIN workbench_access_publications p USING(tenant_id,kind,resource_id)
WHERE to_jsonb(p) IN (SELECT value FROM _access_rollback_payload, jsonb_array_elements(data->'workbench_access_publications')))
OR EXISTS(SELECT 1 FROM workbench_access_assignments a JOIN workbench_access_roles r ON r.id=a.role_id
WHERE to_jsonb(r) IN (SELECT value FROM _access_rollback_payload, jsonb_array_elements(data->'workbench_access_roles'))
AND to_jsonb(a) NOT IN (SELECT value FROM _access_rollback_payload, jsonb_array_elements(data->'workbench_access_assignments')))
THEN RAISE EXCEPTION 'New references to migrated access rows; rollback aborted'; END IF;
END $references$;""")
for table in (ACCESS_TABLES[1], ACCESS_TABLES[2], ACCESS_TABLES[0], LEDGER):
parts.append(f"DELETE FROM {table} actual WHERE to_jsonb(actual) IN (SELECT value FROM _access_rollback_payload,jsonb_array_elements(data->'{table}'));")
parts.append("COMMIT;" if commit else "ROLLBACK;")
return "\n".join(parts) + "\n"
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("command", choices=("plan", "apply"))
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
suffixes = (".before.json", ".after.json", ".attempt.json", ".committed.json", ".rollback.sql")
if args.out.exists() or any(args.out.with_suffix(s).exists() for s in suffixes):
parser.error("Checkpoint exists; choose a new output name")
plan = json.loads(args.manifest.read_text())
sql = migration_sql(plan)
before = snapshot()
save_json(args.out.with_suffix(".before.json"), before)
rehearsal = [json.loads(line) for line in sql_run(sql).splitlines() if line.startswith("{")]
if snapshot() != before:
raise RuntimeError("Rehearsal changed state or concurrent changes detected")
result = {"checked_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "database": TARGET_DATABASE,
"tenant_id": TARGET_TENANT, "manifest": str(args.manifest), "plan_sha256": plan["plan_sha256"],
"rehearsal": rehearsal, "committed": False}
if args.command == "apply":
save_json(args.out.with_suffix(".attempt.json"), {**result, "commit_requested": True,
"recovery": "Reconcile durable access ledger and checkpoints; never recreate records blindly"})
changes = [json.loads(line) for line in sql_run(migration_sql(plan, True)).splitlines() if line.startswith("{")]
result.update(committed=True, changes=changes)
save_json(args.out.with_suffix(".committed.json"), result)
after = snapshot()
save_json(args.out.with_suffix(".after.json"), after)
result["verification"] = verification(plan, before, after)
args.out.with_suffix(".rollback.sql").write_text(rollback_sql(plan, before, after), encoding="utf-8")
save_json(args.out, result)
print(json.dumps(result))
return 0 if result.get("verification", {}).get("passed", True) else 1
if __name__ == "__main__":
raise SystemExit(main())
+142
View File
@@ -0,0 +1,142 @@
#!/usr/bin/env python3
"""Checkpoint Agent/skill migration without writing private bodies to reports."""
from __future__ import annotations
import argparse
import datetime
import json
from pathlib import Path
from apply_weknora_identities import sql_run
from weknora_asset_etl import LEDGER, BINDING_LEDGER, asset_plan, asset_sql, safe_asset_summary
from weknora_category_etl import save_json
from weknora_etl import consistent_snapshot
from weknora_identity_etl import IDENTITY_LEDGER, TARGET_DATABASE, TARGET_TENANT
from weknora_model_etl import model_plan
def target_snapshot():
return json.loads(sql_run("""SELECT json_build_object(
'database',current_database(),
'agents',(SELECT coalesce(json_agg(json_build_object('id',id,'tenant_id',tenant_id,
'fingerprint',md5(to_jsonb(a)::text)) ORDER BY id),'[]') FROM custom_agents a),
'skills',(SELECT coalesce(json_agg(json_build_object('id',id,'tenant_id',tenant_id,
'fingerprint',md5(to_jsonb(s)::text)) ORDER BY id),'[]') FROM tenant_skill_catalog s),
'models',(SELECT coalesce(json_agg(json_build_object('id',id,
'fingerprint',md5(to_jsonb(m)::text)) ORDER BY id),'[]') FROM models m),
'knowledge_count',(SELECT count(*) FROM knowledges),
'chunk_count',(SELECT count(*) FROM chunks));"""))
def build_plan(db, mapping_file):
identities = json.loads(sql_run(f"SELECT json_object_agg(source_user_id,target_user_id) FROM {IDENTITY_LEDGER} "
f"WHERE source_system='yawan-workbench' AND source_organization_id=1 AND target_tenant_id={TARGET_TENANT};"))
state = json.loads(mapping_file.read_text())
if state.get('tenant_id') != TARGET_TENANT:
raise ValueError('Knowledge mapping tenant mismatch')
kbs = {key: item['id'] for key, item in state['knowledge_bases'].items()}
return asset_plan(db, identities, kbs, model_plan(db, TARGET_TENANT))
def verification_sql():
return f"""SELECT json_build_object(
'agents',(SELECT count(*) FROM {LEDGER} WHERE target_tenant_id={TARGET_TENANT} AND source_kind='agent'),
'skills',(SELECT count(*) FROM {LEDGER} WHERE target_tenant_id={TARGET_TENANT} AND source_kind='skill'),
'bindings',(SELECT count(*) FROM {BINDING_LEDGER} WHERE target_tenant_id={TARGET_TENANT}),
'effective_bindings',(SELECT coalesce(sum(jsonb_array_length(coalesce(a.config->'instruction_skill_ids','[]'))),0)
FROM custom_agents a JOIN {LEDGER} l ON l.source_kind='agent' AND l.target_id=a.id WHERE l.target_tenant_id={TARGET_TENANT}),
'disabled_agents',(SELECT count(*) FROM custom_agents a JOIN {LEDGER} l ON l.source_kind='agent' AND l.target_id=a.id
WHERE l.target_tenant_id={TARGET_TENANT} AND NOT a.enabled),
'private_agents',(SELECT count(*) FROM custom_agents a JOIN {LEDGER} l ON l.source_kind='agent' AND l.target_id=a.id
WHERE l.target_tenant_id={TARGET_TENANT} AND a.visibility='private'),
'private_skills',(SELECT count(*) FROM tenant_skill_catalog s JOIN {LEDGER} l ON l.source_kind='skill' AND l.target_id=s.id
WHERE l.target_tenant_id={TARGET_TENANT} AND s.visibility='private'),
'rules_match',(SELECT bool_and(a.config->>'system_prompt'=coalesce(l.source_snapshot->>'rules',''))
FROM custom_agents a JOIN {LEDGER} l ON l.source_kind='agent' AND l.target_id=a.id WHERE l.target_tenant_id={TARGET_TENANT}),
'skill_bodies_match',(SELECT bool_and(s.instructions=l.source_snapshot->>'instructions')
FROM tenant_skill_catalog s JOIN {LEDGER} l ON l.source_kind='skill' AND l.target_id=s.id WHERE l.target_tenant_id={TARGET_TENANT}));"""
def transaction(query, commit=False):
return f"""BEGIN ISOLATION LEVEL SERIALIZABLE;
SET LOCAL statement_timeout='30s';
SET LOCAL lock_timeout='5s';
CREATE TEMP TABLE _agents_before ON COMMIT DROP AS SELECT * FROM custom_agents;
CREATE TEMP TABLE _skills_before ON COMMIT DROP AS SELECT * FROM tenant_skill_catalog;
{query}
DO $preservation$
BEGIN
IF EXISTS (SELECT 1 FROM _agents_before b LEFT JOIN custom_agents a ON a.id=b.id
WHERE a.id IS NULL OR to_jsonb(a) IS DISTINCT FROM to_jsonb(b))
OR EXISTS (SELECT 1 FROM _skills_before b LEFT JOIN tenant_skill_catalog s ON s.id=b.id
WHERE s.id IS NULL OR to_jsonb(s) IS DISTINCT FROM to_jsonb(b))
THEN RAISE EXCEPTION 'Asset import changed an existing row'; END IF;
END $preservation$;
SELECT json_build_object(
'inserted_agents',(SELECT count(*) FROM custom_agents a WHERE NOT EXISTS (SELECT 1 FROM _agents_before b WHERE b.id=a.id)),
'inserted_skills',(SELECT count(*) FROM tenant_skill_catalog s WHERE NOT EXISTS (SELECT 1 FROM _skills_before b WHERE b.id=s.id)),
'existing_assets_unchanged',true);
{verification_sql()}
{'COMMIT' if commit else 'ROLLBACK'};
"""
def verify(plan, before, after, results):
checks = {'acceptance_scope': before['database'] == after['database'] == TARGET_DATABASE}
for kind in ('agents', 'skills'):
old, new = ({v['id']: v for v in state[kind]} for state in (before, after))
expected = {item['target']['id'] for item in plan[kind]}
checks[kind + '_existing_unchanged'] = all(new.get(k) == v for k, v in old.items())
checks[kind + '_planned_only'] = set(new) - set(old) == expected - set(old)
checks[kind + '_all_present'] = expected <= set(new)
checks['knowledge_and_models_unchanged'] = all(before[k] == after[k] for k in ('models', 'knowledge_count', 'chunk_count'))
summary = next(item for item in results if 'effective_bindings' in item)
checks['durable_mapping_counts'] = all(summary[k] == len(plan[k]) for k in ('agents', 'skills', 'bindings'))
checks['effective_binding_count'] = summary['effective_bindings'] == sum(len(a['target']['config']['instruction_skill_ids']) for a in plan['agents'])
checks['disabled_and_private_preserved'] = summary['disabled_agents'] == 1 and summary['private_agents'] == 2 and summary['private_skills'] == 2
# Native edits may differ on future repeats; first import must preserve all bodies.
if not any(x['target']['id'] in {a['id'] for a in before['agents']} for x in plan['agents']):
checks['source_bodies_preserved'] = summary['rules_match'] and summary['skill_bodies_match']
return {'passed': all(checks.values()), 'checks': checks}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('command', choices=('plan', 'apply'))
parser.add_argument('--sqlite', type=Path, required=True)
parser.add_argument('--kb-mapping', type=Path, required=True)
parser.add_argument('--out', type=Path, required=True)
args = parser.parse_args()
suffixes = ('.before.json', '.manifest.json', '.attempt.json', '.committed.json', '.after.json')
if any(p.exists() for p in (args.out, *(args.out.with_suffix(s) for s in suffixes))):
parser.error('Checkpoint exists; use a new output name')
with consistent_snapshot(args.sqlite) as (db, info):
plan = build_plan(db, args.kb_mapping)
if info['source_mutated']:
raise RuntimeError('Source changed while taking the snapshot')
plan['source_snapshot_sha256'] = info['sha256']
query = asset_sql(plan)
before = target_snapshot()
save_json(args.out.with_suffix('.before.json'), before)
save_json(args.out.with_suffix('.manifest.json'), {**safe_asset_summary(plan), 'source_snapshot_sha256': info['sha256']})
rehearsal = [json.loads(line) for line in sql_run(transaction(query)).splitlines() if line.startswith('{')]
result = {'checked_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
'database': TARGET_DATABASE, 'tenant_id': TARGET_TENANT,
'source_snapshot_sha256': info['sha256'], 'source_mutated': info['source_mutated'],
'rehearsal': rehearsal, 'committed': False}
if args.command == 'apply':
save_json(args.out.with_suffix('.attempt.json'), {**result, 'commit_requested': True,
'recovery': 'Reconcile asset ledger and committed receipt; do not delete or recreate assets automatically'})
results = [json.loads(line) for line in sql_run(transaction(query, True)).splitlines() if line.startswith('{')]
result.update(committed=True, changes=results)
save_json(args.out.with_suffix('.committed.json'), result)
after = target_snapshot()
save_json(args.out.with_suffix('.after.json'), after)
result['verification'] = verify(plan, before, after, results)
save_json(args.out, result)
print(json.dumps(result))
return 0 if result.get('verification', {}).get('passed', True) else 1
if __name__ == '__main__':
raise SystemExit(main())
+133
View File
@@ -0,0 +1,133 @@
#!/usr/bin/env python3
"""Checkpoint historical workbench imports into the pinned acceptance database."""
from __future__ import annotations
import argparse
import datetime
import json
from pathlib import Path
from apply_weknora_identities import sql_run
from weknora_asset_etl import LEDGER as ASSET_LEDGER
from weknora_business_etl import LEDGER, business_plan, business_sql, safe_business_summary
from weknora_category_etl import save_json
from weknora_etl import consistent_snapshot
from weknora_identity_etl import IDENTITY_LEDGER, TARGET_DATABASE, TARGET_TENANT
PRESERVED_TABLES = ('workbench_jobs', 'workbench_events', 'workbench_revisions',
'custom_agents', 'tenant_skill_catalog', 'models')
def build_plan(db, mapping_file, artifact_root):
identities = json.loads(sql_run(f"SELECT json_object_agg(source_user_id,target_user_id) FROM {IDENTITY_LEDGER} "
f"WHERE source_system='yawan-workbench' AND source_organization_id=1 AND target_tenant_id={TARGET_TENANT};"))
agents = json.loads(sql_run(f"SELECT json_object_agg(source_id,target_id) FROM {ASSET_LEDGER} "
f"WHERE source_system='yawan-workbench' AND source_organization_id=1 AND source_kind='agent' AND target_tenant_id={TARGET_TENANT};"))
state = json.loads(mapping_file.read_text())
if state.get('tenant_id') != TARGET_TENANT:
raise ValueError('Knowledge mapping tenant mismatch')
kbs = {key: item['id'] for key, item in state['knowledge_bases'].items()}
return business_plan(db, identities, agents, kbs, artifact_root)
def target_snapshot():
parts = ["'database',current_database()", "'knowledge_count',(SELECT count(*) FROM knowledges)",
"'chunk_count',(SELECT count(*) FROM chunks)"]
for table in ('workbench_resources', *PRESERVED_TABLES):
parts.append(f"'{table}',(SELECT coalesce(json_agg(json_build_object('id',id,'hash',md5(to_jsonb(r)::text)) ORDER BY id),'[]') FROM {table} r)")
return json.loads(sql_run('SELECT json_build_object(' + ','.join(parts) + ');'))
def verification_sql():
return f"""SELECT json_build_object(
'ledger_counts',(SELECT json_object_agg(source_kind,n) FROM (SELECT source_kind,count(*) n
FROM {LEDGER} WHERE target_tenant_id={TARGET_TENANT} GROUP BY source_kind) q),
'resources',(SELECT count(*) FROM {LEDGER} l JOIN workbench_resources r ON r.id=l.target_id
WHERE l.target_tenant_id={TARGET_TENANT} AND l.source_kind IN ('canvas','trainingmap','campaign')),
'protected',(SELECT count(*) FROM {LEDGER} l JOIN workbench_resources r ON r.id=l.target_id
WHERE l.target_tenant_id={TARGET_TENANT} AND r.data->'legacy_import'->>'read_only'='true'),
'segments',(SELECT coalesce(sum(jsonb_array_length(r.data->'segments')),0)
FROM {LEDGER} l JOIN workbench_resources r ON r.id=l.target_id
WHERE l.target_tenant_id={TARGET_TENANT} AND l.source_kind='campaign'),
'initial_data_match',(SELECT bool_and(r.data=l.native_snapshot->'data' AND r.owner_id=l.native_snapshot->>'owner_id'
AND r.status=l.native_snapshot->>'status' AND r.version=(l.native_snapshot->>'version')::integer)
FROM {LEDGER} l JOIN workbench_resources r ON r.id=l.target_id WHERE l.target_tenant_id={TARGET_TENANT}));"""
def transaction(query, commit=False):
sql = "BEGIN ISOLATION LEVEL SERIALIZABLE; SET LOCAL statement_timeout='45s'; SET LOCAL lock_timeout='5s';\n"
tables = ('workbench_resources', *PRESERVED_TABLES)
for i, table in enumerate(tables):
sql += f'CREATE TEMP TABLE _business_before_{i} ON COMMIT DROP AS SELECT * FROM {table};\n'
sql += query
assertions = ["EXISTS (SELECT 1 FROM _business_before_0 b LEFT JOIN workbench_resources r ON r.id=b.id WHERE r.id IS NULL OR to_jsonb(r) IS DISTINCT FROM to_jsonb(b))"]
for i, table in enumerate(tables[1:], 1):
assertions.extend((f'EXISTS (SELECT * FROM {table} EXCEPT SELECT * FROM _business_before_{i})',
f'EXISTS (SELECT * FROM _business_before_{i} EXCEPT SELECT * FROM {table})'))
sql += "DO $preserve$ BEGIN IF " + ' OR '.join(assertions) + " THEN RAISE EXCEPTION 'Business import changed existing data or execution state'; END IF; END $preserve$;\n"
sql += "SELECT json_build_object('inserted_resources',(SELECT count(*) FROM workbench_resources r WHERE NOT EXISTS (SELECT 1 FROM _business_before_0 b WHERE b.id=r.id)), 'existing_data_unchanged',true);\n"
return sql + verification_sql() + ('COMMIT;' if commit else 'ROLLBACK;')
def verify(plan, before, after, results):
old = {r['id']: r for r in before['workbench_resources']}
new = {r['id']: r for r in after['workbench_resources']}
planned = {r['target']['id'] for r in plan['resources']}
summary = next(r for r in results if 'ledger_counts' in r)
counts = safe_business_summary(plan)['counts']
checks = {
'acceptance_scope': before['database'] == after['database'] == TARGET_DATABASE,
'existing_resources_unchanged': all(new.get(k) == v for k, v in old.items()),
'only_planned_resources_added': set(new) - set(old) == planned - set(old),
'all_resources_present': planned <= set(new),
'no_jobs_events_revisions_or_other_assets_changed': all(before[k] == after[k] for k in PRESERVED_TABLES),
'knowledge_unchanged': all(before[k] == after[k] for k in ('knowledge_count', 'chunk_count')),
'all_history_counts_match': all(summary['ledger_counts'].get(k, 0) == v for k, v in counts.items()),
'all_segments_present': summary['segments'] == counts['segment'],
'historical_execution_protected': summary['protected'] == len(plan['resources']),
}
if not planned.intersection(old):
checks['initial_content_status_and_owner_match'] = summary['initial_data_match']
return {'passed': all(checks.values()), 'checks': checks}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('command', choices=('plan', 'apply'))
parser.add_argument('--sqlite', type=Path, required=True)
parser.add_argument('--kb-mapping', type=Path, required=True)
parser.add_argument('--artifact-root', type=Path, required=True)
parser.add_argument('--out', type=Path, required=True)
args = parser.parse_args()
if any(p.exists() for p in [args.out, *(args.out.with_suffix(s) for s in ('.before.json', '.manifest.json', '.attempt.json', '.committed.json', '.after.json'))]):
parser.error('Checkpoint exists; choose a fresh output name')
with consistent_snapshot(args.sqlite) as (db, info):
plan = build_plan(db, args.kb_mapping, args.artifact_root)
if info['source_mutated']:
raise RuntimeError('Source changed during snapshot')
plan['source_snapshot_sha256'] = info['sha256']
query = business_sql(plan)
before = target_snapshot()
save_json(args.out.with_suffix('.before.json'), before)
save_json(args.out.with_suffix('.manifest.json'), {**safe_business_summary(plan), 'source_snapshot_sha256': info['sha256']})
rehearsal = [json.loads(line) for line in sql_run(transaction(query)).splitlines() if line.startswith('{')]
if before != target_snapshot():
raise RuntimeError('Target changed during rollback rehearsal')
result = {'checked_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'database': TARGET_DATABASE,
'tenant_id': TARGET_TENANT, 'source_mutated': info['source_mutated'],
'source_snapshot_sha256': info['sha256'], 'rehearsal': rehearsal, 'committed': False}
if args.command == 'apply':
save_json(args.out.with_suffix('.attempt.json'), {**result, 'commit_requested': True})
results = [json.loads(line) for line in sql_run(transaction(query, True)).splitlines() if line.startswith('{')]
result.update(committed=True, changes=results)
save_json(args.out.with_suffix('.committed.json'), result)
after = target_snapshot()
save_json(args.out.with_suffix('.after.json'), after)
result['verification'] = verify(plan, before, after, results)
save_json(args.out, result)
print(json.dumps(result))
return 0 if result.get('verification', {}).get('passed', True) else 1
if __name__ == '__main__':
raise SystemExit(main())
@@ -0,0 +1,65 @@
#!/usr/bin/env python3
"""Add the deleted historical owner's disabled identity with rollback rehearsal."""
import argparse
import json
from pathlib import Path
from apply_weknora_identities import sql_run, target_snapshot
from weknora_category_etl import save_json
from weknora_etl import consistent_snapshot
from weknora_historical_identity_etl import historical_identity_plan, historical_identity_sql
from weknora_identity_etl import TARGET_TENANT
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--sqlite', type=Path, required=True)
parser.add_argument('--out', type=Path, required=True)
args = parser.parse_args()
if any(p.exists() for p in [args.out, args.out.with_suffix('.before.json'), args.out.with_suffix('.attempt.json')]):
parser.error('Checkpoint exists')
with consistent_snapshot(args.sqlite) as (db, info):
plan = historical_identity_plan(db, TARGET_TENANT)
if info['source_mutated']:
raise RuntimeError('Source changed during snapshot')
plan['source_snapshot_sha256'] = info['sha256']
query = historical_identity_sql(plan)
before = target_snapshot()
save_json(args.out.with_suffix('.before.json'), before)
sql = """BEGIN ISOLATION LEVEL SERIALIZABLE;
SET LOCAL statement_timeout='30s'; SET LOCAL lock_timeout='5s';
SET LOCAL log_statement='none'; SET LOCAL log_min_error_statement='panic';
CREATE TEMP TABLE _old_users ON COMMIT DROP AS SELECT * FROM users;
CREATE TEMP TABLE _old_members ON COMMIT DROP AS SELECT * FROM tenant_members;
""" + query + """
DO $verify$ BEGIN
IF EXISTS(SELECT 1 FROM _old_users b LEFT JOIN users a ON a.id=b.id WHERE a.id IS NULL OR to_jsonb(a) IS DISTINCT FROM to_jsonb(b))
OR EXISTS(SELECT 1 FROM _old_members b LEFT JOIN tenant_members a ON a.id=b.id WHERE a.id IS NULL OR to_jsonb(a) IS DISTINCT FROM to_jsonb(b))
THEN RAISE EXCEPTION 'Historical identity changed existing users'; END IF;
END $verify$;
"""
sql_run(sql + 'ROLLBACK;')
if before != target_snapshot():
raise RuntimeError('Rollback rehearsal changed identities')
save_json(args.out.with_suffix('.attempt.json'), {'source_snapshot_sha256': info['sha256'], 'commit_requested': True})
sql_run(sql + 'COMMIT;')
after = target_snapshot()
uid = plan['users'][0]['target_id']
user = next(u for u in after['users'] if u['id'] == uid)
members = [m for m in after['memberships'] if m['user_id'] == uid]
tokens = [t for t in after['token_counts'] if t['user_id'] == uid]
checks = {'source_not_mutated': not info['source_mutated'], 'login_disabled': not user['is_active'],
'not_admin': not user['is_system_admin'] and not user['can_access_all_tenants'],
'membership_suspended': len(members) == 1 and members[0]['tenant_id'] == TARGET_TENANT and members[0]['status'] == 'suspended',
'no_tokens': not tokens}
sql_run(sql + 'COMMIT;')
checks['repeat_no_change'] = after == target_snapshot()
result = {'committed': True, 'source_user_id': 2, 'target_id': uid,
'passed': all(checks.values()), 'checks': checks}
save_json(args.out, result)
print(json.dumps(result))
return 0 if result['passed'] else 1
if __name__ == '__main__':
raise SystemExit(main())
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""Checkpointed identity import restricted to the local acceptance database."""
from __future__ import annotations
import argparse
import datetime
import json
import subprocess
from pathlib import Path
from weknora_category_etl import save_json
from weknora_etl import consistent_snapshot
from weknora_identity_etl import (ADMIN_TARGET_ID, IDENTITY_LEDGER, ORGANIZATION_LEDGER,
TARGET_DATABASE, TARGET_TENANT, identity_plan, identity_sql)
def sql_run(sql):
result = subprocess.run([
"docker", "--context", "colima-yawan", "exec", "-i", "WeKnora-postgres",
"psql", "-X", "-qAt", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", TARGET_DATABASE,
], input=sql, text=True, capture_output=True, timeout=60)
if result.returncode:
# PostgreSQL can include SQL payloads in errors. Do not print them.
raise RuntimeError("Identity transaction not confirmed; reconcile ledger and checkpoints before retrying")
return result.stdout.strip()
def target_snapshot():
snapshot = json.loads(sql_run(f"""
BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY;
SELECT json_build_object(
'database',current_database(),
'tenant',(SELECT json_build_object('id',id,'status',status) FROM tenants WHERE id={TARGET_TENANT}),
'users',(SELECT coalesce(json_agg(row_to_json(x) ORDER BY id),'[]') FROM
(SELECT id,username,email,avatar,tenant_id,is_active,is_system_admin,can_access_all_tenants,
created_at,updated_at,deleted_at FROM users) x),
'memberships',(SELECT coalesce(json_agg(row_to_json(x) ORDER BY id),'[]') FROM tenant_members x),
'token_counts',(SELECT coalesce(json_agg(row_to_json(x) ORDER BY user_id),'[]') FROM
(SELECT user_id,count(*) AS count FROM auth_tokens GROUP BY user_id) x)
);
COMMIT;
"""))
snapshot["ledgers"] = {}
for table in (ORGANIZATION_LEDGER, IDENTITY_LEDGER):
exists = sql_run(f"SELECT to_regclass('public.{table}') IS NOT NULL;") == "t"
order = "source_system,source_organization_id" + (",source_user_id" if table == IDENTITY_LEDGER else "")
snapshot["ledgers"][table] = json.loads(sql_run(f"SELECT coalesce(json_agg(x ORDER BY {order}),'[]') FROM {table} x;")) if exists else []
return snapshot
def transaction(manifest, commit=False):
# Save complete rows only inside PostgreSQL; credentials never enter reports.
return f"""
BEGIN ISOLATION LEVEL SERIALIZABLE;
SET LOCAL statement_timeout = '30s';
SET LOCAL lock_timeout = '5s';
CREATE TEMP TABLE _identity_before_users ON COMMIT DROP AS SELECT * FROM users;
CREATE TEMP TABLE _identity_before_members ON COMMIT DROP AS SELECT * FROM tenant_members;
{identity_sql(manifest)}
DO $identity_preservation$
BEGIN
IF EXISTS (SELECT 1 FROM _identity_before_users b LEFT JOIN users u ON u.id=b.id
WHERE u.id IS NULL OR to_jsonb(u) IS DISTINCT FROM to_jsonb(b))
THEN RAISE EXCEPTION 'Identity import changed an existing user'; END IF;
IF EXISTS (SELECT 1 FROM _identity_before_members b LEFT JOIN tenant_members m ON m.id=b.id
WHERE m.id IS NULL OR to_jsonb(m) IS DISTINCT FROM to_jsonb(b))
THEN RAISE EXCEPTION 'Identity import changed an existing membership'; END IF;
END $identity_preservation$;
SELECT json_build_object(
'inserted_users',(SELECT count(*) FROM users u WHERE NOT EXISTS (SELECT 1 FROM _identity_before_users b WHERE b.id=u.id)),
'inserted_memberships',(SELECT count(*) FROM tenant_members m WHERE NOT EXISTS (SELECT 1 FROM _identity_before_members b WHERE b.id=m.id)),
'identity_mappings',(SELECT count(*) FROM {IDENTITY_LEDGER} WHERE target_tenant_id={TARGET_TENANT}),
'organization_mappings',(SELECT count(*) FROM {ORGANIZATION_LEDGER} WHERE target_tenant_id={TARGET_TENANT}),
'existing_users_unchanged',true,'existing_memberships_unchanged',true
);
{'COMMIT' if commit else 'ROLLBACK'};
"""
def verify_states(manifest, before, after):
checks = []
def check(name, passed):
checks.append({"check": name, "passed": bool(passed)})
check("acceptance_scope", before["database"] == after["database"] == TARGET_DATABASE
and before["tenant"] == after["tenant"] == {"id": TARGET_TENANT, "status": "active"})
old_users = {u["id"]: u for u in before["users"]}
users = {u["id"]: u for u in after["users"]}
old_members = {m["id"]: m for m in before["memberships"]}
members = {m["id"]: m for m in after["memberships"]}
check("existing_users_unchanged", all(users.get(key) == value for key, value in old_users.items()))
check("existing_memberships_unchanged", all(members.get(key) == value for key, value in old_members.items()))
check("existing_admin_unchanged", ADMIN_TARGET_ID in old_users and users.get(ADMIN_TARGET_ID) == old_users[ADMIN_TARGET_ID])
expected_ids = {u["target_id"] for u in manifest["users"] if u["action"] == "create_suspended"}
check("only_planned_users_added", set(users) - set(old_users) == expected_ids - set(old_users))
added_members = [m for key, m in members.items() if key not in old_members]
check("only_planned_memberships_added", len(added_members) == len(expected_ids - set(old_users))
and {m["user_id"] for m in added_members} == expected_ids - set(old_users))
token_counts = {t["user_id"]: t["count"] for t in after["token_counts"]}
for item in manifest["users"]:
if item["action"] != "create_suspended":
continue
uid = item["target_id"]
user = users.get(uid, {})
scoped = [m for m in members.values() if m["user_id"] == uid and m["deleted_at"] is None]
expected_contact = old_users.get(uid, {"username": item["target_username"], "email": item["target_email"]})
check(f"user_{item['source_user']['id']}_identity", user.get("username") == expected_contact["username"]
and user.get("email") == expected_contact["email"] and user.get("tenant_id") == TARGET_TENANT
and user.get("deleted_at") is None and user.get("is_system_admin") is False
and user.get("can_access_all_tenants") is False)
check(f"user_{item['source_user']['id']}_isolated", len(scoped) == 1
and scoped[0]["tenant_id"] == TARGET_TENANT and scoped[0]["role"] == "contributor")
# Exact reruns preserve later activation and password-reset decisions.
if uid not in old_users:
check(f"user_{item['source_user']['id']}_login_disabled", user.get("is_active") is False
and len(scoped) == 1 and scoped[0]["status"] == "suspended" and token_counts.get(uid, 0) == 0)
skipped_names = {u["source_user"]["username"].casefold() for u in manifest["skipped"]}
check("disabled_source_accounts_not_added", all(u["username"].casefold() not in skipped_names
for uid, u in users.items() if uid not in old_users))
identities = {r["source_user_id"]: r for r in after["ledgers"][IDENTITY_LEDGER] if r["target_tenant_id"] == TARGET_TENANT}
organizations = [r for r in after["ledgers"][ORGANIZATION_LEDGER] if r["target_tenant_id"] == TARGET_TENANT]
check("durable_identity_mapping", set(identities) == {u["source_user"]["id"] for u in manifest["users"]}
and all(identities[u["source_user"]["id"]]["target_user_id"] == u["target_id"]
and identities[u["source_user"]["id"]]["source_snapshot"] == u["source_user"] for u in manifest["users"]))
check("durable_organization_and_roles", len(organizations) == 1
and organizations[0]["source_snapshot"] == manifest["source_organization"]
and organizations[0]["source_role_definitions"] == manifest["source_role_definitions"])
return {"passed": all(c["passed"] for c in checks), "checks": checks}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("command", choices=("plan", "apply"))
parser.add_argument("--sqlite", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
suffixes = (".before.json", ".manifest.json", ".attempt.json", ".committed.json", ".after.json")
if any(p.exists() for p in (args.out, *(args.out.with_suffix(s) for s in suffixes))):
parser.error("Output checkpoint exists; use a new output name")
with consistent_snapshot(args.sqlite) as (snapshot, info):
manifest = identity_plan(snapshot, TARGET_TENANT)
if info["source_mutated"]:
raise RuntimeError("Source changed while taking snapshot; retry during a stable migration window")
manifest["source_snapshot"] = info
manifest["source_snapshot_sha256"] = info["sha256"]
before = target_snapshot()
save_json(args.out.with_suffix(".before.json"), before)
save_json(args.out.with_suffix(".manifest.json"), manifest)
rehearsal = [json.loads(line) for line in sql_run(transaction(manifest)).splitlines() if line.startswith("{")]
result = {"checked_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"tenant_id": TARGET_TENANT, "database": TARGET_DATABASE,
"source_snapshot_sha256": info["sha256"], "source_mutated": info["source_mutated"],
"mapped_admins": 1, "planned_suspended_users": 2, "skipped_disabled_users": len(manifest["skipped"]),
"rehearsal": rehearsal, "committed": False}
if args.command == "apply":
save_json(args.out.with_suffix(".attempt.json"), {**result, "commit_requested": True,
"recovery": "Inspect identity ledger and use a new output name to reconcile; do not delete users automatically"})
result["changes"] = [json.loads(line) for line in sql_run(transaction(manifest, True)).splitlines() if line.startswith("{")]
result["committed"] = True
save_json(args.out.with_suffix(".committed.json"), result)
after = target_snapshot()
save_json(args.out.with_suffix(".after.json"), after)
result["verification"] = verify_states(manifest, before, after)
save_json(args.out, result)
print(json.dumps(result, ensure_ascii=False))
return 0 if result.get("verification", {}).get("passed", True) else 1
if __name__ == "__main__":
raise SystemExit(main())
+142
View File
@@ -0,0 +1,142 @@
#!/usr/bin/env python3
"""Checkpointed model import into the isolated acceptance database."""
from __future__ import annotations
import argparse
import datetime
import json
import subprocess
from pathlib import Path
from apply_weknora_identities import sql_run
from weknora_category_etl import save_json
from weknora_etl import consistent_snapshot
from weknora_identity_etl import TARGET_DATABASE, TARGET_TENANT
from weknora_model_etl import LEDGER, model_plan, model_sql, model_verification_sql, read_credentials
def resolve_source_credential(value, provider_id):
if not value.startswith('keychain:'):
return value
expected = f'keychain:yawen-workbench.providers:provider-{provider_id}'
if value != expected:
raise ValueError('Source credential reference does not match the reviewed provider')
result = subprocess.run(['/usr/bin/security', 'find-generic-password', '-s', value[9:], '-w'],
capture_output=True, text=True, timeout=10)
if result.returncode or not result.stdout.strip():
raise RuntimeError('Source provider credential could not be resolved')
return result.stdout.strip()
def native_encryptor(environment: Path):
env = dict(line.split("=", 1) for line in environment.read_text().splitlines()
if line and not line.startswith("#") and "=" in line)
key = env.get("SYSTEM_AES_KEY", "")
if env.get("DB_NAME") != TARGET_DATABASE or len(key.encode()) != 32:
raise ValueError("Acceptance encryption configuration is invalid")
def encrypt(value):
# Node's standard crypto API matches internal/utils/crypto.go. Secrets
# enter through stdin, never argv, an environment override, or a file.
script = """const fs = require('node:fs'), crypto = require('node:crypto');
const input = JSON.parse(fs.readFileSync(0, 'utf8'));
const nonce = crypto.randomBytes(12);
const cipher = crypto.createCipheriv('aes-256-gcm', Buffer.from(input.key), nonce);
const data = Buffer.concat([nonce, cipher.update(input.value, 'utf8'), cipher.final(), cipher.getAuthTag()]);
process.stdout.write('enc:v1:' + data.toString('base64url'));
"""
result = subprocess.run(["node", "-e", script], input=json.dumps({"key": key, "value": value}),
capture_output=True, text=True, timeout=10)
if result.returncode:
raise RuntimeError("Native credential encryption failed")
return result.stdout
return encrypt
def target_snapshot():
return json.loads(sql_run("""SELECT json_build_object(
'database',current_database(),
'models',(SELECT coalesce(json_agg(json_build_object('id',id,'tenant_id',tenant_id,
'name',name,'type',type,'fingerprint',md5(to_jsonb(m)::text)) ORDER BY id),'[]') FROM models m),
'knowledge_count',(SELECT count(*) FROM knowledges),
'chunk_count',(SELECT count(*) FROM chunks));"""))
def transaction(query, commit=False):
return f"""BEGIN ISOLATION LEVEL SERIALIZABLE;
SET LOCAL statement_timeout='30s';
SET LOCAL lock_timeout='5s';
CREATE TEMP TABLE _model_before ON COMMIT DROP AS SELECT * FROM models;
{query}
DO $preservation$
BEGIN
IF EXISTS (SELECT 1 FROM _model_before b LEFT JOIN models m ON m.id=b.id
WHERE m.id IS NULL OR to_jsonb(m) IS DISTINCT FROM to_jsonb(b))
THEN RAISE EXCEPTION 'Model import changed an existing row'; END IF;
END $preservation$;
SELECT json_build_object('inserted_models',
(SELECT count(*) FROM models m WHERE NOT EXISTS (SELECT 1 FROM _model_before b WHERE b.id=m.id)),
'existing_models_unchanged',true);
{model_verification_sql()}
{'COMMIT' if commit else 'ROLLBACK'};
"""
def verify(plan, before, after, summaries):
old = {m['id']: m for m in before['models']}
new = {m['id']: m for m in after['models']}
expected = {m['target_id'] for m in plan['models']}
checks = {
'acceptance_scope': before['database'] == after['database'] == TARGET_DATABASE,
'existing_models_unchanged': all(new.get(k) == v for k, v in old.items()),
'only_planned_models_added': set(new) - set(old) == expected - set(old),
'all_models_present': expected <= set(new),
'knowledge_and_chunks_unchanged': all(before[k] == after[k] for k in ('knowledge_count', 'chunk_count')),
'ledger_and_encrypted_credentials': any(s.get('mapped') == len(expected)
and len(s.get('models', [])) == len(expected)
and all(m['encrypted_credential'] for m in s['models']) for s in summaries),
}
return {'passed': all(checks.values()), 'checks': checks}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('command', choices=('plan', 'apply'))
parser.add_argument('--sqlite', type=Path, required=True)
parser.add_argument('--environment', type=Path, required=True)
parser.add_argument('--out', type=Path, required=True)
args = parser.parse_args()
suffixes = ('.before.json', '.manifest.json', '.attempt.json', '.committed.json', '.after.json')
if any(p.exists() for p in (args.out, *(args.out.with_suffix(s) for s in suffixes))):
parser.error('Checkpoint exists; use a new output name')
with consistent_snapshot(args.sqlite) as (db, info):
plan = model_plan(db, TARGET_TENANT)
if info['source_mutated']:
raise RuntimeError('Source changed while taking the snapshot')
plan['source_snapshot_sha256'] = info['sha256']
query = model_sql(plan, read_credentials(db, plan, resolve_source_credential), native_encryptor(args.environment))
before = target_snapshot()
save_json(args.out.with_suffix('.before.json'), before)
save_json(args.out.with_suffix('.manifest.json'), plan)
rehearsal = [json.loads(line) for line in sql_run(transaction(query)).splitlines() if line.startswith('{')]
result = {'checked_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
'database': TARGET_DATABASE, 'tenant_id': TARGET_TENANT,
'source_snapshot_sha256': info['sha256'], 'source_mutated': info['source_mutated'],
'planned_models': len(plan['models']), 'skipped_models': len(plan['skipped']),
'rehearsal': rehearsal, 'committed': False}
if args.command == 'apply':
save_json(args.out.with_suffix('.attempt.json'), {**result, 'commit_requested': True,
'recovery': 'Reconcile model ledger and committed receipt; never delete models automatically'})
summaries = [json.loads(line) for line in sql_run(transaction(query, True)).splitlines() if line.startswith('{')]
result.update(committed=True, changes=summaries)
save_json(args.out.with_suffix('.committed.json'), result)
after = target_snapshot()
save_json(args.out.with_suffix('.after.json'), after)
result['verification'] = verify(plan, before, after, summaries)
save_json(args.out, result)
print(json.dumps(result))
return 0 if result.get('verification', {}).get('passed', True) else 1
if __name__ == '__main__':
raise SystemExit(main())
+120
View File
@@ -0,0 +1,120 @@
#!/usr/bin/env bash
# Build the anydoc static archive that the `anydoc` build tag links.
#
# The archive is a Rust build artifact (~30 MB), so it is neither committed nor
# downloaded: this script builds it from third_party/anydoc-go, which pins the
# published anydoc crate, and drops it where cgo looks for it.
#
# Usage:
# scripts/build-anydoc-lib.sh # host platform
# TARGET=aarch64-unknown-linux-musl scripts/build-anydoc-lib.sh
set -euo pipefail
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
crate_dir="$repo_root/third_party/anydoc-go"
if ! command -v cargo >/dev/null 2>&1; then
echo "error: cargo not found. Install Rust (https://rustup.rs) to build the anydoc archive." >&2
exit 1
fi
# The Rust target triple decides the archive; the directory name mirrors what
# the cgo LDFLAGS in third_party/anydoc-go expect for that platform.
target=${TARGET:-$(rustc -vV | sed -n 's/^host: //p')}
case "$target" in
x86_64-apple-darwin) lib_dir=darwin_amd64 ;;
aarch64-apple-darwin) lib_dir=darwin_arm64 ;;
x86_64-pc-windows-msvc) lib_dir=windows_amd64 ;;
x86_64-unknown-linux-gnu) lib_dir=linux_amd64_gnu ;;
aarch64-unknown-linux-gnu) lib_dir=linux_arm64_gnu ;;
x86_64-unknown-linux-musl) lib_dir=linux_amd64_musl ;;
aarch64-unknown-linux-musl) lib_dir=linux_arm64_musl ;;
*)
echo "error: unsupported target '$target'." >&2
echo "Supported: {x86_64,aarch64}-{apple-darwin,unknown-linux-{gnu,musl}}, x86_64-pc-windows-msvc" >&2
exit 1
;;
esac
case "$target" in
*windows-msvc) lib_name=anydoc_go.lib ;;
*) lib_name=libanydoc_go.a ;;
esac
# The crate version to patch is read from Cargo.toml rather than repeated here,
# so an upgrade is a one-line change to the `anydoc = "=X.Y.Z"` pin.
anydoc_version=$(sed -n 's/^anydoc = "=\([0-9][^"]*\)".*/\1/p' "$crate_dir/Cargo.toml" | head -1)
if [ -z "$anydoc_version" ]; then
echo "error: no pinned 'anydoc = \"=X.Y.Z\"' dependency in $crate_dir/Cargo.toml" >&2
exit 1
fi
# document_to_markdown is private in published anydoc. Copy the pinned version
# and re-export that one function so anydoc-go can keep the official serializer.
prepare_patched_anydoc() {
local dest="$crate_dir/patched-anydoc"
# The marker records which version was patched: after a version bump the
# copy left by the previous build is the wrong crate, and reusing it would
# fail deep inside cargo's patch resolution instead of here. Cargo.toml is
# required too: a truncated leftover with only the marker would otherwise
# skip the copy and fail with "failed to read .../Cargo.toml".
if [ -f "$dest/.weknora-patched" ] && [ "$(cat "$dest/.weknora-patched")" = "$anydoc_version" ] && [ -f "$dest/Cargo.toml" ]; then
return
fi
local src=""
local cargo_home="${CARGO_HOME:-$HOME/.cargo}"
src=$(find "$cargo_home/registry/src" -maxdepth 2 -type d -name "anydoc-$anydoc_version" 2>/dev/null | head -1 || true)
if [ -z "$src" ]; then
local tarball="$crate_dir/.anydoc-$anydoc_version.crate"
echo "Fetching anydoc $anydoc_version to patch document_to_markdown into the public API"
if ! curl -fsSL "https://rsproxy.cn/api/v1/crates/anydoc/$anydoc_version/download" -o "$tarball"; then
curl -fsSL "https://static.crates.io/crates/anydoc/anydoc-$anydoc_version.crate" -o "$tarball"
fi
local unpack="$crate_dir/.anydoc-unpack"
rm -rf "$unpack"
mkdir -p "$unpack"
tar -xzf "$tarball" -C "$unpack"
src=$(find "$unpack" -maxdepth 1 -type d -name "anydoc-$anydoc_version" | head -1)
if [ -z "$src" ]; then
echo "error: unpacked anydoc-$anydoc_version crate is missing" >&2
exit 1
fi
fi
rm -rf "$dest"
cp -R "$src" "$dest"
if ! grep -q '^use render::markdown::document_to_markdown;$' "$dest/src/lib.rs"; then
echo "error: anydoc $anydoc_version lib.rs no longer has the expected document_to_markdown import" >&2
exit 1
fi
sed -i.bak 's/^use render::markdown::document_to_markdown;/pub use render::markdown::document_to_markdown;/' "$dest/src/lib.rs"
rm -f "$dest/src/lib.rs.bak"
printf '%s\n' "$anydoc_version" > "$dest/.weknora-patched"
rm -rf "$crate_dir/.anydoc-unpack" "$crate_dir/.anydoc-$anydoc_version.crate"
}
# version.go is what the parser reports as `anydoc_version` on every parsed
# document, so a bump that misses it would mislabel the whole knowledge base.
go_version=$(sed -n 's/^const Version = "\([^"]*\)".*/\1/p' "$crate_dir/version.go" | head -1)
if [ "$go_version" != "$anydoc_version" ]; then
echo "error: version.go says '$go_version' but Cargo.toml pins anydoc '$anydoc_version'." >&2
echo "Bump both: the Go constant is the version WeKnora records for parsed documents." >&2
exit 1
fi
echo "Building anydoc $anydoc_version archive for $target"
prepare_patched_anydoc
# --locked: build exactly the dependency versions in the committed Cargo.lock.
# That lockfile is what pins the patched pdf-inspector/lopdf, so a silent
# resolver drift must fail the build rather than ship an unaudited tree.
cargo build --release --locked --manifest-path "$crate_dir/Cargo.toml" --target "$target"
dest="$crate_dir/lib/$lib_dir"
mkdir -p "$dest"
cp "$crate_dir/target/$target/release/$lib_name" "$dest/$lib_name"
echo "Wrote $dest/$lib_name"
echo "Build WeKnora with the archive: go build -tags anydoc ./cmd/server"
+18
View File
@@ -0,0 +1,18 @@
#!/usr/bin/env bash
# Build frontend static assets for Docker / release packaging.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
if [ -z "${VITE_FRONTEND_COMMIT:-}" ]; then
# shellcheck source=/dev/null
eval "$("$PROJECT_ROOT/scripts/get_version.sh" env)"
export VITE_FRONTEND_COMMIT="${COMMIT_ID:-unknown}"
fi
export VITE_IS_DOCKER="${VITE_IS_DOCKER:-true}"
cd "$PROJECT_ROOT/frontend"
npm ci
npm run build
+428
View File
@@ -0,0 +1,428 @@
#!/bin/bash
# 该脚本用于从源码构建WeKnora的所有Docker镜像
# 设置颜色
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m' # 无颜色
# 获取项目根目录(脚本所在目录的上一级)
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
# 启用 BuildKit
export DOCKER_BUILDKIT=1
# 版本信息
VERSION="1.0.0"
SCRIPT_NAME=$(basename "$0")
# 显示帮助信息
show_help() {
echo -e "${GREEN}WeKnora 镜像构建脚本 v${VERSION}${NC}"
echo -e "${GREEN}用法:${NC} $0 [选项]"
echo "选项:"
echo " -h, --help 显示帮助信息"
echo " -a, --all 构建所有镜像(默认)"
echo " -p, --app 仅构建应用镜像"
echo " -d, --docreader 仅构建文档读取器镜像"
echo " -f, --frontend 仅构建前端镜像"
echo " -s, --sandbox 仅构建沙箱镜像"
echo " -c, --clean 清理所有本地镜像"
echo " -v, --version 显示版本信息"
exit 0
}
# 显示版本信息
show_version() {
echo -e "${GREEN}WeKnora 镜像构建脚本 v${VERSION}${NC}"
exit 0
}
# 日志函数
log_info() {
echo -e "${BLUE}[INFO]${NC} $1"
}
log_warning() {
echo -e "${YELLOW}[WARNING]${NC} $1"
}
log_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
log_success() {
echo -e "${GREEN}[SUCCESS]${NC} $1"
}
# 检查Docker是否已安装
check_docker() {
log_info "检查Docker环境..."
if ! command -v docker &> /dev/null; then
log_error "未安装Docker,请先安装Docker"
return 1
fi
# 检查Docker服务运行状态
if ! docker info &> /dev/null; then
log_error "Docker服务未运行,请启动Docker服务"
return 1
fi
log_success "Docker环境检查通过"
return 0
}
# 检测平台
check_platform() {
log_info "检测系统平台信息..."
if [ "$(uname -m)" = "x86_64" ]; then
export PLATFORM="linux/amd64"
export TARGETARCH="amd64"
elif [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then
export PLATFORM="linux/arm64"
export TARGETARCH="arm64"
else
log_warning "未识别的平台类型:$(uname -m),将使用默认平台 linux/amd64"
export PLATFORM="linux/amd64"
export TARGETARCH="amd64"
fi
log_info "当前平台:$PLATFORM"
log_info "当前架构:$TARGETARCH"
}
# 获取版本信息
get_version_info() {
# 从VERSION文件获取版本号
if [ -f "VERSION" ]; then
VERSION=$(cat VERSION | tr -d '\n\r')
else
VERSION="unknown"
fi
# 获取commit ID
if command -v git >/dev/null 2>&1; then
COMMIT_ID=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
else
COMMIT_ID="unknown"
fi
# 获取构建时间
BUILD_TIME=$(date -u '+%Y-%m-%d %H:%M:%S UTC')
# 获取Go版本
if command -v go >/dev/null 2>&1; then
GO_VERSION=$(go version 2>/dev/null || echo "unknown")
else
GO_VERSION="unknown"
fi
log_info "版本信息: $VERSION"
log_info "Commit ID: $COMMIT_ID"
log_info "构建时间: $BUILD_TIME"
log_info "Go版本: $GO_VERSION"
}
# 构建应用镜像
build_app_image() {
log_info "构建应用镜像 (weknora-app)..."
cd "$PROJECT_ROOT"
# 获取版本信息
get_version_info
docker build \
--platform $PLATFORM \
--build-arg GOPRIVATE_ARG=${GOPRIVATE:-""} \
--build-arg GOPROXY_ARG=${GOPROXY:-"https://goproxy.cn,direct"} \
--build-arg GOSUMDB_ARG=${GOSUMDB:-"off"} \
--build-arg VERSION_ARG="$VERSION" \
--build-arg COMMIT_ID_ARG="$COMMIT_ID" \
--build-arg BUILD_TIME_ARG="$BUILD_TIME" \
--build-arg GO_VERSION_ARG="$GO_VERSION" \
--build-arg WITH_ANYDOC=${WITH_ANYDOC:-1} \
-f docker/Dockerfile.app \
-t wechatopenai/weknora-app:latest \
.
if [ $? -eq 0 ]; then
log_success "应用镜像构建成功"
return 0
else
log_error "应用镜像构建失败"
return 1
fi
}
# 构建文档读取器镜像
build_docreader_image() {
log_info "构建文档读取器镜像 (weknora-docreader)..."
cd "$PROJECT_ROOT"
docker build \
--platform $PLATFORM \
--build-arg PLATFORM=$PLATFORM \
--build-arg TARGETARCH=$TARGETARCH \
--build-arg APT_MIRROR=${APT_MIRROR:-} \
-f docker/Dockerfile.docreader \
-t wechatopenai/weknora-docreader:latest \
.
if [ $? -eq 0 ]; then
log_success "文档读取器镜像构建成功"
return 0
else
log_error "文档读取器镜像构建失败"
return 1
fi
}
# 构建前端镜像
build_frontend_image() {
log_info "构建前端镜像 (weknora-ui)..."
cd "$PROJECT_ROOT"
# 获取版本信息(用于注入前端 commit hash)
get_version_info
log_info "构建前端静态资源..."
VITE_IS_DOCKER=true VITE_FRONTEND_COMMIT="$COMMIT_ID" "$SCRIPT_DIR/build_frontend_dist.sh"
docker build \
--platform $PLATFORM \
-f frontend/Dockerfile \
-t wechatopenai/weknora-ui:latest \
frontend/
if [ $? -eq 0 ]; then
log_success "前端镜像构建成功"
return 0
else
log_error "前端镜像构建失败"
return 1
fi
}
# 构建沙箱镜像
build_sandbox_image() {
log_info "构建沙箱镜像 (weknora-sandbox)..."
cd "$PROJECT_ROOT"
# 同时打上 main:默认镜像跟的是 main(见 sandbox.go 的 DefaultDockerImage),
# 而 Docker 后端只在本地缺失时才去拉,本地不打这个标签就等于白构建。
docker build \
--platform $PLATFORM \
-f docker/Dockerfile.sandbox \
--target sandbox \
-t wechatopenai/weknora-sandbox:latest \
-t wechatopenai/weknora-sandbox:main \
.
if [ $? -ne 0 ]; then
log_error "沙箱镜像构建失败"
return 1
fi
# Cube 从镜像直接构建模板,并以 :49983/health 探活,缺 envd 必然失败,
# 因此 Cube 用的是注入了 envd 的变体镜像。详见 docs/sandbox-cluster.md。
# 固定 linux/amd64:envd 的来源镜像 cubesandbox-base 不发布 arm64。
log_info "构建沙箱镜像 Cube 变体 (weknora-sandbox:main-cube)..."
docker build \
--platform linux/amd64 \
-f docker/Dockerfile.sandbox \
--target cube \
-t wechatopenai/weknora-sandbox:latest-cube \
-t wechatopenai/weknora-sandbox:main-cube \
.
if [ $? -eq 0 ]; then
log_success "沙箱镜像构建成功"
return 0
else
log_error "沙箱镜像 Cube 变体构建失败"
return 1
fi
}
# 构建所有镜像
build_all_images() {
log_info "开始构建所有镜像..."
local app_result=0
local docreader_result=0
local frontend_result=0
local sandbox_result=0
# 构建应用镜像
build_app_image
app_result=$?
# 构建文档读取器镜像
build_docreader_image
docreader_result=$?
# 构建前端镜像
build_frontend_image
frontend_result=$?
# 构建沙箱镜像
build_sandbox_image
sandbox_result=$?
# 显示构建结果
echo ""
log_info "=== 构建结果 ==="
if [ $app_result -eq 0 ]; then
log_success "✓ 应用镜像构建成功"
else
log_error "✗ 应用镜像构建失败"
fi
if [ $docreader_result -eq 0 ]; then
log_success "✓ 文档读取器镜像构建成功"
else
log_error "✗ 文档读取器镜像构建失败"
fi
if [ $frontend_result -eq 0 ]; then
log_success "✓ 前端镜像构建成功"
else
log_error "✗ 前端镜像构建失败"
fi
if [ $sandbox_result -eq 0 ]; then
log_success "✓ 沙箱镜像构建成功"
else
log_error "✗ 沙箱镜像构建失败"
fi
if [ $app_result -eq 0 ] && [ $docreader_result -eq 0 ] && [ $frontend_result -eq 0 ] && [ $sandbox_result -eq 0 ]; then
log_success "所有镜像构建完成!"
return 0
else
log_error "部分镜像构建失败"
return 1
fi
}
# 清理本地镜像
clean_images() {
log_info "清理本地WeKnora镜像..."
# 停止相关容器
log_info "停止相关容器..."
docker stop $(docker ps -q --filter "ancestor=wechatopenai/weknora-app:latest" 2>/dev/null) 2>/dev/null || true
docker stop $(docker ps -q --filter "ancestor=wechatopenai/weknora-docreader:latest" 2>/dev/null) 2>/dev/null || true
docker stop $(docker ps -q --filter "ancestor=wechatopenai/weknora-ui:latest" 2>/dev/null) 2>/dev/null || true
# 删除相关容器
log_info "删除相关容器..."
docker rm $(docker ps -aq --filter "ancestor=wechatopenai/weknora-app:latest" 2>/dev/null) 2>/dev/null || true
docker rm $(docker ps -aq --filter "ancestor=wechatopenai/weknora-docreader:latest" 2>/dev/null) 2>/dev/null || true
docker rm $(docker ps -aq --filter "ancestor=wechatopenai/weknora-ui:latest" 2>/dev/null) 2>/dev/null || true
# 删除镜像
log_info "删除本地镜像..."
docker rmi wechatopenai/weknora-app:latest 2>/dev/null || true
docker rmi wechatopenai/weknora-docreader:latest 2>/dev/null || true
docker rmi wechatopenai/weknora-ui:latest 2>/dev/null || true
docker rmi wechatopenai/weknora-sandbox:latest 2>/dev/null || true
docker rmi wechatopenai/weknora-sandbox:latest-cube 2>/dev/null || true
docker rmi wechatopenai/weknora-sandbox:main 2>/dev/null || true
docker rmi wechatopenai/weknora-sandbox:main-cube 2>/dev/null || true
docker image prune -f
log_success "镜像清理完成"
return 0
}
# 解析命令行参数
BUILD_ALL=false
BUILD_APP=false
BUILD_DOCREADER=false
BUILD_FRONTEND=false
BUILD_SANDBOX=false
CLEAN_IMAGES=false
# 没有参数时默认构建所有镜像
if [ $# -eq 0 ]; then
BUILD_ALL=true
fi
while [ "$1" != "" ]; do
case $1 in
-h | --help ) show_help
;;
-a | --all ) BUILD_ALL=true
;;
-p | --app ) BUILD_APP=true
;;
-d | --docreader ) BUILD_DOCREADER=true
;;
-f | --frontend ) BUILD_FRONTEND=true
;;
-s | --sandbox ) BUILD_SANDBOX=true
;;
-c | --clean ) CLEAN_IMAGES=true
;;
-v | --version ) show_version
;;
* ) log_error "未知选项: $1"
show_help
;;
esac
shift
done
# 检查Docker环境
check_docker
if [ $? -ne 0 ]; then
exit 1
fi
# 检测平台
check_platform
# 执行清理操作
if [ "$CLEAN_IMAGES" = true ]; then
clean_images
exit $?
fi
# 执行构建操作
if [ "$BUILD_ALL" = true ]; then
build_all_images
exit $?
fi
if [ "$BUILD_APP" = true ]; then
build_app_image
exit $?
fi
if [ "$BUILD_DOCREADER" = true ]; then
build_docreader_image
exit $?
fi
if [ "$BUILD_FRONTEND" = true ]; then
build_frontend_image
exit $?
fi
if [ "$BUILD_SANDBOX" = true ]; then
build_sandbox_image
exit $?
fi
exit 0
+208
View File
@@ -0,0 +1,208 @@
#!/bin/bash
# 检查开发环境配置
# 设置颜色
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m' # 无颜色
# 获取项目根目录
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
log_info() {
printf "%b\n" "${BLUE}[INFO]${NC} $1"
}
log_success() {
printf "%b\n" "${GREEN}[✓]${NC} $1"
}
log_error() {
printf "%b\n" "${RED}[✗]${NC} $1"
}
log_warning() {
printf "%b\n" "${YELLOW}[!]${NC} $1"
}
echo ""
printf "%b\n" "${GREEN}========================================${NC}"
printf "%b\n" "${GREEN} WeKnora 开发环境配置检查${NC}"
printf "%b\n" "${GREEN}========================================${NC}"
echo ""
cd "$PROJECT_ROOT"
# 检查 .env 文件
log_info "检查 .env 文件..."
if [ -f ".env" ]; then
log_success ".env 文件存在"
else
log_error ".env 文件不存在"
echo ""
log_info "解决方法:"
echo " 1. 复制示例文件: cp .env.example .env"
echo " 2. 编辑 .env 文件并配置必要的环境变量"
exit 1
fi
echo ""
log_info "检查必要的环境变量..."
# 加载 .env 文件
set -a
source .env
set +a
# 检查必要的环境变量
errors=0
check_var() {
local var_name=$1
local var_value="${!var_name}"
if [ -z "$var_value" ]; then
log_error "$var_name 未设置"
errors=$((errors + 1))
else
log_success "$var_name = $var_value"
fi
}
# 数据库配置
log_info "数据库配置:"
check_var "DB_DRIVER"
check_var "DB_HOST"
check_var "DB_PORT"
check_var "DB_USER"
check_var "DB_PASSWORD"
check_var "DB_NAME"
echo ""
log_info "存储配置:"
check_var "STORAGE_TYPE"
if [ "$STORAGE_TYPE" = "minio" ]; then
check_var "MINIO_BUCKET_NAME"
fi
if [ "$STORAGE_TYPE" = "tos" ]; then
check_var "TOS_ENDPOINT"
check_var "TOS_REGION"
check_var "TOS_ACCESS_KEY"
check_var "TOS_SECRET_KEY"
check_var "TOS_BUCKET_NAME"
fi
if [ "$STORAGE_TYPE" = "s3" ]; then
check_var "S3_REGION"
check_var "S3_BUCKET_NAME"
fi
echo ""
log_info "Redis 配置:"
check_var "REDIS_ADDR"
echo ""
log_info "Ollama 配置:"
check_var "OLLAMA_BASE_URL"
echo ""
log_info "模型配置:"
if [ -n "$INIT_LLM_MODEL_NAME" ]; then
log_success "INIT_LLM_MODEL_NAME = $INIT_LLM_MODEL_NAME"
else
log_warning "INIT_LLM_MODEL_NAME 未设置(可选)"
fi
if [ -n "$INIT_EMBEDDING_MODEL_NAME" ]; then
log_success "INIT_EMBEDDING_MODEL_NAME = $INIT_EMBEDDING_MODEL_NAME"
else
log_warning "INIT_EMBEDDING_MODEL_NAME 未设置(可选)"
fi
# 检查 Go 环境
echo ""
log_info "检查 Go 环境..."
if command -v go &> /dev/null; then
go_version=$(go version)
log_success "Go 已安装: $go_version"
else
log_error "Go 未安装"
errors=$((errors + 1))
fi
# 检查 Air
if command -v air &> /dev/null; then
log_success "Air 已安装(支持热重载)"
else
log_warning "Air 未安装(可选,用于热重载)"
log_info "安装命令: go install github.com/air-verse/air@latest"
fi
# 检查 npm
echo ""
log_info "检查 Node.js 环境..."
if command -v npm &> /dev/null; then
npm_version=$(npm --version)
log_success "npm 已安装: $npm_version"
else
log_error "npm 未安装"
errors=$((errors + 1))
fi
# 检查 Docker
echo ""
log_info "检查 Docker 环境..."
if command -v docker &> /dev/null; then
docker_version=$(docker --version)
log_success "Docker 已安装: $docker_version"
if docker info &> /dev/null; then
log_success "Docker 服务正在运行"
else
log_error "Docker 服务未运行"
errors=$((errors + 1))
fi
else
log_error "Docker 未安装"
errors=$((errors + 1))
fi
# 检查 Docker Compose
if docker compose version &> /dev/null; then
compose_version=$(docker compose version)
log_success "Docker Compose 已安装: $compose_version"
elif command -v docker-compose &> /dev/null; then
compose_version=$(docker-compose --version)
log_success "docker-compose 已安装: $compose_version"
else
log_error "Docker Compose 未安装"
errors=$((errors + 1))
fi
# 总结
echo ""
printf "%b\n" "${GREEN}========================================${NC}"
if [ $errors -eq 0 ]; then
log_success "所有检查通过!环境配置正常"
echo ""
log_info "下一步:"
echo " 1. 启动开发环境: make dev-start"
echo " 2. 启动后端: make dev-app"
echo " 3. 启动前端: make dev-frontend"
else
log_error "发现 $errors 个问题,请修复后再启动开发环境"
echo ""
log_info "常见问题:"
echo " - 如果 .env 文件不存在,请复制 .env.example"
echo " - 确保 DB_DRIVER 设置为 'postgres' 或 'mysql'"
echo " - 确保 Docker 服务正在运行"
fi
printf "%b\n" "${GREEN}========================================${NC}"
echo ""
exit $errors
@@ -0,0 +1,34 @@
const fs = require('node:fs')
const assert = require('node:assert/strict')
const root = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
const base = 'http://127.0.0.1:18080/api/v1'
const fixtures = [
['training-maps','c9fb3c98-a9f3-40f0-8ac0-2756c83288de'],
['training-maps','013a398f-fe0e-4287-8f3f-6fe7d6f468f9'],
['flows','1011c8bc-a0f4-4baf-abab-cd1c05a85889'],
['flows','15fc92d5-d0f8-416f-97bc-1927ffe39a71'],
['agents','6065fcd5-d827-40a1-8fe4-902906576b85'],
['agents','e69f8f18-010d-43c7-be2c-4cd1409df2aa'],
]
async function main() {
const c=JSON.parse(fs.readFileSync(`${root}/super-admin-login.json`,'utf8'))
const login=await fetch(`${base}/auth/login`,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({email:c.email,password:c.password})})
assert.equal(login.status,200)
const auth=await login.json()
const headers={Authorization:`Bearer ${auth.token}`,'X-Tenant-ID':'10059'}
const archive=[]
for(const [kind,id] of fixtures) {
const path=kind==='agents'?`/agents/${id}`:`/workbench/resources/${kind}/${id}`
const r=await fetch(base+path,{headers})
if(r.status===404) continue
assert.equal(r.status,200)
const row=(await r.json()).data
assert.ok(/^real-generation-178958(7501724|7865185)/.test(row.name),'Only known failed verification fixtures may be removed')
archive.push({kind,row})
fs.writeFileSync(`${root}/failed-generation-fixtures-archived.json`,JSON.stringify(archive,null,2),{mode:0o600})
const deletion=await fetch(base+path,{method:'DELETE',headers})
assert.ok([200,204].includes(deletion.status),`Fixture deletion ${deletion.status}`)
}
console.log(JSON.stringify({removed:archive.length,preserved:'Successful real-generation sample, usage ledger and test evidence'}))
}
main().catch(e=>{console.error(e.message);process.exitCode=1})
+276
View File
@@ -0,0 +1,276 @@
# WeKnora 云镜像打包脚本(Cloud-Agnostic)
> **本文档面向「想把 WeKnora 打包成云镜像(AMI / 自定义镜像 / Snapshot)分发给其他人」的用户。**
> **如果你只是想自己用 WeKnora,请直接看主仓 [README](../../README.md),`docker compose up -d` 即可。**
## 这套脚本能做什么
帮你把任意一台「能跑 Docker 的 Linux 实例」变成一份**可分发的云镜像模板**:
- 别人基于这份镜像创建新实例后,**首次开机会自动**:
- 生成全新的随机密钥(DB / Redis / JWT / AES)
- 启动 WeKnora 全部默认容器
- 把生成的凭证写到 `/root/weknora-credentials.txt`
- 自删除一次性初始化脚本
- 实现「**开机即用、零私密泄漏、每实例独立密钥**」
适用平台(任意 systemd + Docker 的 Linux 都行):
- 腾讯云轻量应用服务器(Lighthouse) / 云服务器 CVM
- AWS EC2 AMI
- 阿里云 ECS 自定义镜像
- 火山引擎 / 华为云 / Vultr Snapshot
- 本地 KVM / Proxmox 模板
各平台具体的「制作镜像 / 共享 / 上架」操作步骤,请参考 [`docs/cloud-image/`](../../docs/cloud-image/) 下对应文档。
---
## 目录结构
```
scripts/cloud-image/
├── README.md # 本文档
├── prepare.sh # 步骤一: 装 Docker + 拉运行时 + 装 firstboot
├── cleanup.sh # 步骤二: 制作镜像前清理(执行后将锁定 SSH)
├── firstboot.sh # 新实例首次开机自动执行(用户无感)
└── systemd/
├── weknora.service # 开机自启 docker compose
└── weknora-firstboot.service # 首次启动 init(执行后自删)
```
## 不需要 clone 整个 WeKnora 仓库
WeKnora 所有容器都从 Docker Hub 拉镜像(`wechatopenai/weknora-*`),Go / Python / 前端源码都不需要带到宿主机。
`docker-compose.yml` 实际从宿主机挂载到容器的只有:
```
- ./config/config.yaml (单文件)
- ./skills/preloaded/ (目录)
```
所以镜像里需要的运行时文件**总共不到 100KB**:
| 文件 | 大小 | 用途 |
|---|---|---|
| `docker-compose.yml` | 12K | 容器编排 |
| `.env` | 12K | 环境变量 |
| `config/config.yaml` | 8K | 后端业务配置 |
| `skills/preloaded/` | 56K | Agent 预置技能 |
`prepare.sh` 用 `curl + tar` 只下载这 4 项,不 `git clone`。
## 镜像里启动哪些容器
WeKnora `docker-compose.yml` 大量服务是 **profile 限定**,本镜像只默认启动核心 5 个。
**默认启动(5 个常驻容器,开机自启):**
| 容器 | 角色 |
|---|---|
| `frontend` | Vue UI / NGINX 反代 |
| `app` | WeKnora Go 后端 |
| `docreader` | Python 文档解析 (gRPC) |
| `postgres` (ParadeDB) | 主库 + pgvector 向量检索 + BM25 |
| `redis` | 流式输出 / 缓存 / 异步队列 |
> ParadeDB 自带 pgvector,默认场景下不需额外起向量库。
**额外预拉但不常驻:**
- `sandbox` 镜像:Agent Skills 由 app 按需 `docker run`。提前 `pull` 避免新实例首次执行 Skill 卡在下载。
**Profile 限定,不预装(用户需要时自己 `pull`):**
| profile | 用途 |
|---|---|
| `minio` | 对象存储替代本地文件 |
| `qdrant` / `milvus` / `weaviate` / `doris` | 替代 pgvector |
| `neo4j` | GraphRAG 知识图谱 |
| `langfuse` | 自建 Langfuse 可观测平台 |
| `dex` | OIDC 登录 |
| `odl-hybrid` | OpenDataLoader Docling hybrid(体积大,无预发布镜像,需 `--build`) |
启用方式:
```bash
cd /opt/WeKnora
docker compose --profile neo4j up -d # 启用 GraphRAG
docker compose --profile langfuse up -d # 启用自建 Langfuse
docker compose --profile qdrant up -d # 切换到 Qdrant
docker compose --profile odl-hybrid up -d --build odl-hybrid # Docling hybrid(按需)
```
---
## 完整流程(云无关)
```
1) 在目标云上买/装一台干净 Linux 实例(建议 4C8G+,Ubuntu 22.04)
2) SSH 进去, 拷入本目录, 执行 prepare.sh
3) 浏览器验证功能
4) 执行 cleanup.sh (清掉私密 + SSH key, 自动关机)
5) 在云控制台「制作镜像 / 创建快照 / 创建 AMI」
6) 用新镜像创建测试实例, 验证 firstboot 工作正常
7) 共享 / 公开镜像(参考各平台文档)
```
### 步骤一:在干净实例上部署
要求:systemd + 联网 + sudo 权限。推荐 Ubuntu 22.04 / Debian 12 / CentOS Stream 9。
**1. 拷入脚本(任选一种,都不用 clone 整个 WeKnora 仓库)。**
> 命令需要写入 `/opt/`,最省心的做法是先 `sudo -i` 切到 root 再粘贴。
> 如果坚持每行加 `sudo`,注意 `>>` 重定向是在你当前 shell 执行的,必须改用 `sudo tee -a`。
> **中国大陆云主机请直接看方式 C (scp)**。实测腾讯云 / 阿里云轻量服务器经常连 `github.com`、`raw.githubusercontent.com`、`gh-proxy.com` 这类境外 / 公益代理都连不上 (TLS RST 或超时),方式 A/B 会全军覆没。本机已经有这份仓库,scp 上去最稳。
```bash
sudo -i # 切到 root, 后续命令直接执行
# === 方式 A: sparse checkout (~60KB) ===
# 不通时设 GH_PROXY=https://gh-proxy.com/ 或 https://ghfast.top/, 注意末尾斜杠。
GH_PROXY="${GH_PROXY:-}"
mkdir -p /opt/weknora-tools && cd /opt/weknora-tools
git init -q && git remote add origin "${GH_PROXY}https://github.com/Tencent/WeKnora.git"
git config core.sparseCheckout true
echo "scripts/cloud-image/" >> .git/info/sparse-checkout
git pull -q --depth=1 origin main
# === 方式 B: 直接 curl (无 git 时用这个) ===
# 不通时设 GH_PROXY=https://gh-proxy.com/ 或 https://ghfast.top/, 注意末尾斜杠。
GH_PROXY="${GH_PROXY:-}"
mkdir -p /opt/weknora-tools/scripts/cloud-image/systemd && cd /opt/weknora-tools
base="${GH_PROXY}https://raw.githubusercontent.com/Tencent/WeKnora/main/scripts/cloud-image"
for f in prepare.sh cleanup.sh firstboot.sh README.md; do
curl -fsSL "$base/$f" -o "scripts/cloud-image/$f"
done
for f in weknora.service weknora-firstboot.service; do
curl -fsSL "$base/systemd/$f" -o "scripts/cloud-image/systemd/$f"
done
chmod +x scripts/cloud-image/*.sh
# === 方式 C: 从本地 scp 上来 (推荐: 中国大陆云主机直接走这条) ===
# 在本机 (能正常访问 GitHub 的机器) 执行:
# scp -r scripts/cloud-image root@<实例IP>:/opt/weknora-tools/scripts/
```
> 不确定 VM 能不能访问代理时,先探一下:
> `for h in gh-proxy.com ghfast.top mirror.ghproxy.com github.moeyy.xyz kkgithub.com; do printf '%-25s' "$h"; curl -sS -o /dev/null -m 5 -w 'http=%{http_code} t=%{time_total}s\n' "https://$h/" 2>&1 || echo FAIL; done`
> 哪个返回 `http=200/301/302` 就把 `GH_PROXY` 设成它 (后面加 `/`)。一个都不通的话,认命走方式 C。
**2. 执行部署:**
```bash
sudo bash /opt/weknora-tools/scripts/cloud-image/prepare.sh
# 想 pin 特定版本(推荐, 保证镜像可复现)
sudo WEKNORA_REF=v0.5.0 bash /opt/weknora-tools/scripts/cloud-image/prepare.sh
# 中国大陆机器三件套: 同时绕开 GitHub / get.docker.com / Docker Hub 的境外 CDN
# (以腾讯云为例, 阿里云 / 华为云换对应镜像即可)
sudo \
WEKNORA_REF=v0.5.0 \
WEKNORA_GH_PROXY=https://gh-proxy.com/ \
DOCKER_INSTALL_MIRROR=https://mirrors.tencent.com/docker-ce/linux/ubuntu \
DOCKER_REGISTRY_MIRROR=https://mirror.ccs.tencentyun.com \
bash /opt/weknora-tools/scripts/cloud-image/prepare.sh
```
> 三个变量分别解决三个不同的境外 CDN 不可达问题:
> - `WEKNORA_GH_PROXY`:加速 **GitHub tarball** 下载(`prepare.sh` 步骤 2,运行时文件)
> - `DOCKER_INSTALL_MIRROR`:绕开 **`get.docker.com`**,改用 apt + docker-ce 镜像源装 Docker(步骤 1)
> - `DOCKER_REGISTRY_MIRROR`:加速 **Docker Hub** 镜像拉取(步骤 4,`wechatopenai/weknora-*`)
>
> 不同云厂商对应地址(按需替换 ubuntu/debian 部分以匹配实际发行版):
> | 厂商 | `DOCKER_INSTALL_MIRROR` | `DOCKER_REGISTRY_MIRROR` |
> |---|---|---|
> | 腾讯云 | `https://mirrors.tencent.com/docker-ce/linux/ubuntu` | `https://mirror.ccs.tencentyun.com` |
> | 阿里云 | `https://mirrors.aliyun.com/docker-ce/linux/ubuntu` | `https://<your-id>.mirror.aliyuncs.com` |
> | 华为云 | `https://mirrors.huaweicloud.com/docker-ce/linux/ubuntu` | `https://<id>.mirror.swr.myhuaweicloud.com` |
>
> `DOCKER_INSTALL_MIRROR` 目前仅支持 apt 系(Ubuntu / Debian / TencentOS-apt)。
> CentOS / Rocky 等 yum 系发行版 `get.docker.com` 一般能直连,没碰到再说。
`prepare.sh` 会:
1. 安装 Docker / Docker Compose plugin(已装则跳过)
2. 用 `curl + tar` 下载 4 个运行时文件到 `/opt/WeKnora`
3. 拉取并启动默认 5 个容器 + 预拉 sandbox 镜像
4. 安装 `weknora.service`(开机自启)+ `weknora-firstboot.service`(首启 init)
完成后访问 `http://<公网IP>`。
### 步骤二:验证
至少验证:
- 能注册管理员、能登录
- 能创建一个知识库
- 能上传一个文档并完成解析
- 能进行一次问答
```bash
sudo docker compose -f /opt/WeKnora/docker-compose.yml ps
curl -f http://localhost:8080/health
```
### 步骤三:清理并制作镜像
> **重要**:`cleanup.sh` 会删除所有 SSH 公钥、清空日志、清空数据库与 docker volume。执行后**不要再 SSH 进来**,直接去云控制台关机制作镜像。
```bash
sudo bash /opt/weknora-tools/scripts/cloud-image/cleanup.sh
```
执行完会自动 `poweroff`。然后到对应云控制台按其文档制作镜像。
### 新实例首次开机行为
用户用你的镜像创建实例后,第一次开机时 `weknora-firstboot.service` 会:
1. 生成随机的 `DB_PASSWORD` / `REDIS_PASSWORD` / `JWT_SECRET` / `SYSTEM_AES_KEY`
2. 写回 `/opt/WeKnora/.env`
3. `docker compose up -d` 启动全部服务
4. 把生成的凭证写到 `/root/weknora-credentials.txt`(仅 root 可读)
5. 把自己 disable + 删除自己(确保只跑一次)
之后每次开机都由 `weknora.service` 接管。
> **注意**:`firstboot.sh` 默认**不**禁用注册(`DISABLE_REGISTRATION=false`),第一个注册的人会成为管理员。
> 凭证文件里有「请尽快注册以防被抢注」的醒目提示。需要更严格控制可在 `firstboot.sh` 的 `replace` 调用列表中追加一行 `replace DISABLE_REGISTRATION true` 后再重制镜像。
---
## 升级镜像版本
镜像里没有 git 仓库,升级直接重跑 `prepare.sh`(会覆盖 4 个运行时文件,**不动 `.env` 和 docker volume 数据**):
```bash
sudo WEKNORA_REF=v0.6.0 bash /opt/weknora-tools/scripts/cloud-image/prepare.sh
sudo bash /opt/weknora-tools/scripts/cloud-image/cleanup.sh # 制作新镜像前
```
> **打新镜像时想顺便清掉旧版本镜像层**(每个旧 weknora-* tag 几百 MB,4 个镜像 ~2-4GB):
> ```bash
> sudo PRUNE_OLD_IMAGES=true WEKNORA_REF=v0.6.0 \
> bash /opt/weknora-tools/scripts/cloud-image/prepare.sh
> ```
> 默认 `false` 是为了保留回滚路径。打镜像前确认新版本稳定后再开。
## 安全注意事项
- 镜像里**不要**预置任何 LLM API Key、Langfuse Key、个人 SSH key
- 数据库 / Redis / MinIO 端口默认仅对 docker 网络可见,不要在云防火墙里对外开放
- `/root/weknora-credentials.txt` 用 `umask 077` 创建,仅 root 可读
- 每次重制镜像前必须执行 `cleanup.sh`,避免泄漏上一份测试数据 / SSH key / machine-id
## 各云平台具体操作
| 平台 | 文档 |
|---|---|
| 腾讯云轻量应用服务器 / CVM | [`docs/cloud-image/tencent-lighthouse.md`](../../docs/cloud-image/tencent-lighthouse.md) |
| AWS EC2 AMI | (欢迎贡献) |
| 阿里云 ECS | (欢迎贡献) |
+94
View File
@@ -0,0 +1,94 @@
#!/usr/bin/env bash
# cleanup.sh - 在制作云镜像前清理私密数据。
# 警告: 本脚本会删除 SSH 公钥、清空数据库与日志,最后会自动关机。
# 执行后请直接在云控制台「制作镜像 / 创建快照 / 创建 AMI」,不要再 SSH 进来。
set -euo pipefail
WEKNORA_DIR="${WEKNORA_DIR:-/opt/WeKnora}"
if [[ "${EUID}" -ne 0 ]]; then
echo "[cleanup] 请使用 sudo 或 root 运行" >&2
exit 1
fi
read -r -p "[cleanup] 该操作不可逆,确认继续? 输入 YES 继续: " ans
if [[ "${ans}" != "YES" ]]; then
echo "[cleanup] 已取消"
exit 0
fi
echo "[cleanup] 1/8 停止 WeKnora 容器"
COMPOSE_PROJECT=""
if [[ -d "${WEKNORA_DIR}" ]]; then
cd "${WEKNORA_DIR}"
# 优先用 compose ls 拿到真实 project 名 (默认是目录名小写, 如 weknora)
COMPOSE_PROJECT="$(docker compose ls --format json 2>/dev/null \
| grep -oE '"Name":"[^"]+"' | head -1 | cut -d'"' -f4 || true)"
docker compose down -v --remove-orphans || true
fi
echo "[cleanup] 2/8 清空 WeKnora 业务数据 + 首启 marker / 日志"
if [[ -d "${WEKNORA_DIR}" ]]; then
rm -rf "${WEKNORA_DIR}/data"/* "${WEKNORA_DIR}/logs"/* 2>/dev/null || true
# 故意不在这里重建 .env: 让镜像里 .env 缺失, 任何在 firstboot 之前
# 起来的 docker compose 都会因找不到 .env 而失败, 避免明文默认密码
# (postgres123!@# 之类) 把 postgres 数据卷初始化坏。
# firstboot.sh 自己会从 .env.example 拷贝并替换密钥。
rm -f "${WEKNORA_DIR}/.env" "${WEKNORA_DIR}/.firstboot.done"
fi
rm -f /root/weknora-credentials.txt /var/log/weknora-firstboot.log
echo "[cleanup] 3/8 清理残留 docker 卷与构建缓存"
# 严格按 compose project 名前缀匹配, 避免误伤同宿主上其它 postgres/redis 卷。
if [[ -n "${COMPOSE_PROJECT}" ]]; then
docker volume ls -q --filter "label=com.docker.compose.project=${COMPOSE_PROJECT}" \
| xargs -r docker volume rm -f || true
fi
# 注意: 这里只清"卷 / 已停容器 / 构建缓存", 绝对不能清镜像。
# 此前用过 `docker system prune -af --volumes`, 会把 prepare.sh 预拉的
# wechatopenai/weknora-* 等镜像一并清掉, 导致基于镜像建出来的新实例
# 在 firstboot 时还要重新从 Docker Hub 拉数 GB 镜像, 完全违背预装初衷。
docker container prune -f || true
docker builder prune -af || true
# 只清未挂载的 dangling 卷 (此时 compose down -v 已清掉业务卷)
docker volume prune -f || true
echo "[cleanup] 4/8 清空系统日志"
journalctl --rotate || true
journalctl --vacuum-time=1s || true
find /var/log -type f \( -name '*.log' -o -name '*.gz' -o -name '*.[0-9]' \) -print0 \
| xargs -0 -r truncate -s 0 || true
find /var/log -type f \( -name '*.gz' -o -name '*.[0-9]' \) -print0 \
| xargs -0 -r rm -f || true
echo "[cleanup] 5/8 清理 SSH 历史与授权 key(执行后将无法 SSH 进来)"
rm -f /root/.ssh/authorized_keys /root/.ssh/known_hosts /root/.bash_history
for d in /home/*; do
[[ -d "$d" ]] || continue
rm -f "$d/.ssh/authorized_keys" "$d/.ssh/known_hosts" "$d/.bash_history"
done
find / -xdev -type f \( -name 'id_rsa*' -o -name '*.pem' -o -name '*.key' \) \
-not -path '/etc/ssl/*' -not -path '/usr/*' -not -path '/var/lib/docker/*' 2>/dev/null \
| tee /tmp/cleanup-secrets-found.txt || true
echo "[cleanup] ↑ 上面是疑似遗留的密钥文件,必要时人工再核对"
echo "[cleanup] 6/8 重置 cloud-init / machine-id(让新实例拿到新 ID)"
cloud-init clean --logs --seed 2>/dev/null || true
truncate -s 0 /etc/machine-id || true
rm -f /var/lib/dbus/machine-id || true
echo "[cleanup] 7/8 清理 apt / tmp"
if command -v apt-get >/dev/null 2>&1; then
apt-get clean
rm -rf /var/lib/apt/lists/*
fi
rm -rf /tmp/* /var/tmp/* /root/.cache /home/*/.cache 2>/dev/null || true
echo "[cleanup] 8/8 同步磁盘并关机"
history -c || true
sync
echo
echo " 即将关机。关机完成后请到云控制台执行「制作镜像 / 创建快照 / 创建 AMI」。"
echo
sleep 3
poweroff
+139
View File
@@ -0,0 +1,139 @@
#!/usr/bin/env bash
# firstboot.sh - 由 weknora-firstboot.service 在新实例首次开机时自动执行。
# 任务: 生成随机密钥写入 .env -> 启动容器 -> 输出凭证 -> 标记完成 + 自禁用。
#
# 幂等性策略:
# 1) 一上来先写 ${MARKER} 标记 "已生成过密钥",
# 之后即便 docker compose 失败导致脚本中断, 重启后由 unit 的
# ConditionPathExists=!${MARKER} 拦住, 不会再生成新密钥覆盖 .env。
# (旧密钥已经写进 postgres 数据卷, 二次生成会导致数据库永久无法登录)
# 2) 失败时 unit 标记 failed, 用户可手动 docker compose up -d 恢复;
# 凭证仍可在 ${ENV_FILE} 中查到。
set -euo pipefail
WEKNORA_DIR="${WEKNORA_DIR:-/opt/WeKnora}"
ENV_FILE="${WEKNORA_DIR}/.env"
ENV_TEMPLATE="${WEKNORA_DIR}/.env.example"
CRED_FILE="/root/weknora-credentials.txt"
LOG_FILE="/var/log/weknora-firstboot.log"
MARKER="${WEKNORA_DIR}/.firstboot.done"
# 提前打开 LOG_FILE, 同时把 stderr 也复制一份到 systemd journal 方便调试
# (单纯 exec >> LOG_FILE 时, 脚本一开始的失败会因为 stderr 已被吞掉而看不到)
mkdir -p "$(dirname "${LOG_FILE}")"
exec > >(tee -a "${LOG_FILE}") 2>&1
echo "==== firstboot started at $(date -Iseconds) ===="
if [[ -f "${MARKER}" ]]; then
echo "marker ${MARKER} exists, skip (already initialized)"
exit 0
fi
# cleanup.sh 不再保留 .env, 这里从 .env.example 拷贝模板再做替换。
# 这样保证 firstboot 之前不会有任何含明文默认密码的 .env 让 weknora.service
# 抢先把 postgres 数据卷用错的密码初始化掉。
if [[ ! -f "${ENV_FILE}" ]]; then
if [[ -f "${ENV_TEMPLATE}" ]]; then
echo "creating ${ENV_FILE} from ${ENV_TEMPLATE}"
cp "${ENV_TEMPLATE}" "${ENV_FILE}"
else
echo "ERROR: neither ${ENV_FILE} nor ${ENV_TEMPLATE} found"
exit 1
fi
fi
DOCKER_BIN="$(command -v docker || true)"
if [[ -z "${DOCKER_BIN}" ]]; then
echo "ERROR: docker binary not found in PATH"
exit 1
fi
# 生成 32 字节强随机字符串(用于 AES-256 key, 必须刚好 32 字节)
# 用 `() ... ()` 启子 shell 并关闭 pipefail: head 只读 N 字节后关闭 stdin,
# tr 会收到 SIGPIPE (退出码 141), 在 `set -o pipefail` 下整条管道被判失败,
# 触发顶层 `set -e` 把 firstboot.sh 8ms exit 1 干掉。
gen32() ( set +o pipefail; LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c 32 )
# 通用密码: 24 字符, 不含 / + = (避免出现在 URL / sed 替换时出问题)
genpw() ( set +o pipefail; LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c 24 )
DB_PWD=$(genpw)
REDIS_PWD=$(genpw)
JWT=$(genpw)$(genpw)
SYS_AES=$(gen32)
# 用 | 作 sed 分隔符避免冲突; 仅替换以 KEY= 开头的行
replace() {
local key="$1" val="$2"
if grep -qE "^${key}=" "${ENV_FILE}"; then
sed -i "s|^${key}=.*|${key}=${val}|" "${ENV_FILE}"
else
echo "${key}=${val}" >>"${ENV_FILE}"
fi
}
replace DB_PASSWORD "${DB_PWD}"
replace REDIS_PASSWORD "${REDIS_PWD}"
replace JWT_SECRET "${JWT}"
replace SYSTEM_AES_KEY "${SYS_AES}"
replace GIN_MODE "release"
# 把 prepare.sh 阶段记录在 .cloud-image-meta 里的 WEKNORA_REF 还原为 .env 的
# WEKNORA_VERSION, 否则 docker compose 会落回 :latest 默认值, 导致镜像版本
# 与 prepare 时拉取的版本不一致。
META_FILE="${WEKNORA_DIR}/.cloud-image-meta"
if [[ -f "${META_FILE}" ]]; then
META_REF=$(grep -E '^WEKNORA_REF=' "${META_FILE}" | tail -1 | cut -d= -f2- || true)
if [[ -n "${META_REF}" ]]; then
replace WEKNORA_VERSION "${META_REF}"
echo "restored WEKNORA_VERSION=${META_REF} from ${META_FILE}"
fi
fi
# 关键: .env 改完立刻写 marker。
# 这之后即便 docker compose up 失败, 重启也不会再次重写 .env,
# 防止与 postgres 已经持久化的密码不一致。
umask 077
touch "${MARKER}"
chmod 0600 "${MARKER}"
echo "env updated, marker written, starting docker compose..."
cd "${WEKNORA_DIR}"
"${DOCKER_BIN}" compose up -d
# 尝试拿公网 IP, 失败就用内网
PUB_IP=$(curl -fsS --max-time 5 https://ifconfig.me 2>/dev/null \
|| curl -fsS --max-time 5 https://api.ipify.org 2>/dev/null \
|| hostname -I | awk '{print $1}')
cat >"${CRED_FILE}" <<INFO
========================================
WeKnora 实例初始化完成
生成时间: $(date -Iseconds)
========================================
访问地址 : http://${PUB_IP}
注册后如需关闭后续注册, 编辑 ${ENV_FILE}:
DISABLE_REGISTRATION=true
然后执行: cd ${WEKNORA_DIR} && docker compose up -d
下列随机凭证已写入 ${ENV_FILE}, 请妥善保存(仅 root 可读):
DB_PASSWORD = ${DB_PWD}
REDIS_PASSWORD = ${REDIS_PWD}
JWT_SECRET = ${JWT}
SYSTEM_AES_KEY = ${SYS_AES}
注意:
- 本文件只反映首启时刻的密钥, 后续以 ${ENV_FILE} 为准。
- 切勿直接对外暴露 5432 / 6379 / 9000 等基础设施端口。
- 仅用 80 / 443 对外服务, 必要时配置反向代理 + HTTPS。
INFO
echo "credentials written to ${CRED_FILE}"
# 仅停掉 unit, 不删 unit 文件 (否则当前正在运行的 oneshot 可能被 systemd 标记 failed)。
# 下次重启时, weknora-firstboot.service 通过 ConditionPathExists=!${MARKER} 自动跳过。
systemctl disable weknora-firstboot.service || true
echo "==== firstboot finished at $(date -Iseconds) ===="
+237
View File
@@ -0,0 +1,237 @@
#!/usr/bin/env bash
# prepare.sh - 在干净的 Linux 实例上部署 WeKnora 运行时, 用于制作云镜像模板。
# 不需要 clone 整个 WeKnora 仓库, 只下载 4 个运行时文件 (~100KB)。
# 兼容: Ubuntu / Debian / CentOS / Rocky / TencentOS 等带 systemd + Docker 的发行版。
# 使用方式: sudo bash prepare.sh
# 可调环境变量:
# WEKNORA_REF 要拉取的 git ref (tag / branch / commit), 默认 main
# WEKNORA_DIR 部署目录, 默认 /opt/WeKnora
# WEKNORA_REPO 仓库地址, 默认 https://github.com/Tencent/WeKnora
# WEKNORA_GH_PROXY GitHub 加速前缀, 默认空。中国大陆机器可设
# https://gh-proxy.com/ 或 https://ghfast.top/
# (实际下载地址变成 ${WEKNORA_GH_PROXY}${WEKNORA_REPO}/archive/...)
# DOCKER_INSTALL_MIRROR Docker 安装包镜像源, 默认空 (走 get.docker.com)。
# 中国大陆机器境外 CDN 不通时设为, 例如:
# https://mirrors.tencent.com/docker-ce/linux/ubuntu
# https://mirrors.aliyun.com/docker-ce/linux/ubuntu
# 会改用 apt + docker-ce 官方仓库镜像安装,
# 含 docker-ce / containerd.io / docker-compose-plugin,
# 完全不访问 get.docker.com。仅支持 apt 系发行版。
# DOCKER_REGISTRY_MIRROR Docker Hub 加速器, 默认空。腾讯云内网可设
# https://mirror.ccs.tencentyun.com
# (会写入 /etc/docker/daemon.json 并重启 docker)
# PRUNE_OLD_IMAGES 升级场景下是否清理 dangling / 旧版本 tag 镜像,
# 默认 false。设为 true 时在拉新镜像之后执行
# `docker image prune -af`, 把没有容器引用的镜像
# (含旧 WEKNORA_VERSION 的 wechatopenai/weknora-*)
# 一次性删掉, 减少要打进云镜像的体积。
set -euo pipefail
WEKNORA_REF="${WEKNORA_REF:-main}"
WEKNORA_DIR="${WEKNORA_DIR:-/opt/WeKnora}"
WEKNORA_REPO="${WEKNORA_REPO:-https://github.com/Tencent/WeKnora}"
WEKNORA_GH_PROXY="${WEKNORA_GH_PROXY:-}"
DOCKER_INSTALL_MIRROR="${DOCKER_INSTALL_MIRROR:-}"
DOCKER_REGISTRY_MIRROR="${DOCKER_REGISTRY_MIRROR:-}"
PRUNE_OLD_IMAGES="${PRUNE_OLD_IMAGES:-false}"
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
if [[ "${EUID}" -ne 0 ]]; then
echo "[prepare] 请使用 sudo 或 root 运行" >&2
exit 1
fi
# 通过镜像源 apt 安装 docker-ce 全家桶 (含 compose-plugin)。
# 用于中国大陆云主机直连 get.docker.com 被 RST 的场景。
install_docker_via_apt_mirror() {
local mirror="$1"
if ! command -v apt-get >/dev/null 2>&1; then
echo "[prepare] DOCKER_INSTALL_MIRROR 目前仅支持 apt 系发行版 (Ubuntu/Debian)" >&2
return 1
fi
apt-get update -y
apt-get install -y ca-certificates curl gnupg lsb-release
install -m 0755 -d /etc/apt/keyrings
curl -fsSL "${mirror%/}/gpg" | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg
chmod a+r /etc/apt/keyrings/docker.gpg
local arch codename
arch="$(dpkg --print-architecture)"
codename="$(lsb_release -cs)"
echo "deb [arch=${arch} signed-by=/etc/apt/keyrings/docker.gpg] ${mirror%/} ${codename} stable" \
> /etc/apt/sources.list.d/docker.list
apt-get update -y
apt-get install -y docker-ce docker-ce-cli containerd.io \
docker-buildx-plugin docker-compose-plugin curl tar
}
echo "[prepare] 1/6 安装 Docker 与依赖"
if ! command -v docker >/dev/null 2>&1; then
if [[ -n "${DOCKER_INSTALL_MIRROR}" ]]; then
echo "[prepare] 通过 ${DOCKER_INSTALL_MIRROR} 走 apt 安装 docker-ce (跳过 get.docker.com)"
install_docker_via_apt_mirror "${DOCKER_INSTALL_MIRROR}"
else
curl -fsSL https://get.docker.com | bash
fi
fi
systemctl enable --now docker
if ! docker compose version >/dev/null 2>&1; then
if command -v apt-get >/dev/null 2>&1; then
apt-get update -y
apt-get install -y docker-compose-plugin curl tar
elif command -v yum >/dev/null 2>&1; then
yum install -y docker-compose-plugin curl tar
fi
fi
# 可选: 配置 Docker Hub 加速器, 解决直连 registry-1.docker.io 超时的场景
# (典型: 中国大陆云主机 / 内网受限环境)。仅在用户显式传入时才动 daemon.json。
if [[ -n "${DOCKER_REGISTRY_MIRROR}" ]]; then
echo "[prepare] 1.5/6 配置 Docker Hub 加速器: ${DOCKER_REGISTRY_MIRROR}"
mkdir -p /etc/docker
# 已存在的 daemon.json 走 python 合并, 避免覆盖用户其它配置
if [[ -s /etc/docker/daemon.json ]] && command -v python3 >/dev/null 2>&1; then
python3 - "$DOCKER_REGISTRY_MIRROR" <<'PY'
import json, sys, pathlib
p = pathlib.Path("/etc/docker/daemon.json")
mirror = sys.argv[1]
try:
cfg = json.loads(p.read_text())
except Exception:
cfg = {}
mirrors = cfg.get("registry-mirrors") or []
if mirror not in mirrors:
mirrors.insert(0, mirror)
cfg["registry-mirrors"] = mirrors
p.write_text(json.dumps(cfg, indent=2) + "\n")
PY
else
cat >/etc/docker/daemon.json <<EOF
{
"registry-mirrors": ["${DOCKER_REGISTRY_MIRROR}"]
}
EOF
fi
systemctl restart docker
# 给 docker daemon 一点重启时间, 避免下面 docker compose pull 立刻拿到 EOF
for _ in 1 2 3 4 5; do
docker info >/dev/null 2>&1 && break
sleep 1
done
fi
echo "[prepare] 2/6 拉取 WeKnora 运行时文件 (ref=${WEKNORA_REF})"
# 只下载实际需要的 4 个文件, 不 clone 整个仓库 (~MB 级 -> ~KB 级)
mkdir -p "${WEKNORA_DIR}/config" "${WEKNORA_DIR}/skills"
tmp=$(mktemp -d)
trap 'rm -rf "${tmp}"' EXIT
tarball_url="${WEKNORA_GH_PROXY}${WEKNORA_REPO}/archive/${WEKNORA_REF}.tar.gz"
echo "[prepare] tarball: ${tarball_url}"
curl -fsSL "${tarball_url}" -o "${tmp}/repo.tar.gz"
# 仅解压需要的路径, 显著加速且省空间
tar -xzf "${tmp}/repo.tar.gz" -C "${tmp}" \
--wildcards \
'*/docker-compose.yml' \
'*/.env.example' \
'*/config/config.yaml' \
'*/skills/preloaded'
src=$(find "${tmp}" -maxdepth 1 -mindepth 1 -type d -name 'WeKnora-*' | head -1)
if [[ -z "${src}" ]]; then
echo "[prepare] 解压失败, 未找到 WeKnora-* 目录" >&2
exit 1
fi
cp "${src}/docker-compose.yml" "${WEKNORA_DIR}/"
cp "${src}/.env.example" "${WEKNORA_DIR}/"
cp "${src}/config/config.yaml" "${WEKNORA_DIR}/config/"
rm -rf "${WEKNORA_DIR}/skills/preloaded"
cp -r "${src}/skills/preloaded" "${WEKNORA_DIR}/skills/"
# 记录元信息, 供 firstboot / 升级时参考
cat >"${WEKNORA_DIR}/.cloud-image-meta" <<EOF
WEKNORA_REF=${WEKNORA_REF}
WEKNORA_REPO=${WEKNORA_REPO}
PREPARED_AT=$(date -Iseconds)
EOF
echo "[prepare] 3/6 准备 .env (默认值, firstboot 会替换为随机密钥)"
cd "${WEKNORA_DIR}"
[[ -f .env ]] || cp .env.example .env
sed -i 's/^GIN_MODE=.*/GIN_MODE=release/' .env || true
# 把 WEKNORA_VERSION 与 WEKNORA_REF 对齐, 让 docker compose 拉取与 ref 一致的
# 镜像 tag。无条件覆盖, 避免 .env 残留上一次 prepare 留下的旧版本号。
# Docker Hub 上 wechatopenai/weknora-* 的 tag 命名约定:
# - 浮动 tag:main(持续指向最新构建)
# - 固定 release tag:v 前缀 + semver(如 v0.7.2、v0.5.2)
# 因此这里不剥 v、也不映射到 latest。
WEKNORA_VERSION_VAL="${WEKNORA_REF}"
if grep -qE '^WEKNORA_VERSION=' .env; then
sed -i "s|^WEKNORA_VERSION=.*|WEKNORA_VERSION=${WEKNORA_VERSION_VAL}|" .env
else
echo "WEKNORA_VERSION=${WEKNORA_VERSION_VAL}" >>.env
fi
echo "[prepare] -> WEKNORA_VERSION=${WEKNORA_VERSION_VAL}"
echo "[prepare] 4/6 拉取并启动默认 5 个常驻容器 (frontend/app/docreader/postgres/redis)"
docker compose pull
docker compose up -d
# 提前 pull sandbox 镜像 (Agent Skills 运行时由 app 按需 docker run, 非常驻)
# 不预拉的话, 用户首次跑 Skill 会卡在下载
echo "[prepare] 4.5/6 预拉 sandbox 镜像 (Agent Skills 用, 非常驻)"
docker compose --profile full pull sandbox || true
# 其他向量库 / 可观测组件 (qdrant, milvus, weaviate, doris, neo4j, langfuse-*, minio, dex)
# 不预拉, 体积可省 5-15GB. 用户如需启用:
# cd /opt/WeKnora && docker compose --profile <name> up -d
# 升级场景: 清理旧版本 tag 的 wechatopenai/weknora-* 镜像。
# 默认关闭, 保留回滚路径; 制作镜像前显式打开以减小体积。
#
# 注意: 不用 `docker image prune -af`!
# sandbox 镜像在 compose 里只 pull 不 up (Agent Skills 由 app 按需 docker run),
# 没有任何容器引用它, 一旦 `prune -a` 会把当前版本的 sandbox 一起删掉,
# 反而违背 prepare.sh 4.5 步预拉 sandbox 的目的。
# 这里精确按 tag 比对, 只删 wechatopenai/weknora-* 仓库下、tag 不等于当前
# WEKNORA_VERSION 的镜像, 基础设施镜像 (paradedb / redis) 不动。
if [[ "${PRUNE_OLD_IMAGES,,}" == "true" || "${PRUNE_OLD_IMAGES}" == "1" ]]; then
echo "[prepare] 4.6/6 清理 wechatopenai/weknora-* 仓库下旧版本镜像 (PRUNE_OLD_IMAGES=true, keep=${WEKNORA_VERSION_VAL})"
docker image ls --format '{{.Repository}}:{{.Tag}}' \
| grep -E '^wechatopenai/weknora-' \
| grep -vE ":${WEKNORA_VERSION_VAL}\$" \
| xargs -r docker rmi -f 2>/dev/null || true
fi
echo "[prepare] 5/6 安装 systemd 单元"
# 探测 docker 二进制路径, 不同发行版可能在 /usr/bin 或 /usr/local/bin
DOCKER_BIN="$(command -v docker)"
if [[ -z "${DOCKER_BIN}" ]]; then
echo "[prepare] 未找到 docker 二进制" >&2
exit 1
fi
echo "[prepare] docker binary: ${DOCKER_BIN}"
install -m 0644 "${SCRIPT_DIR}/systemd/weknora.service" /etc/systemd/system/weknora.service
install -m 0644 "${SCRIPT_DIR}/systemd/weknora-firstboot.service" /etc/systemd/system/weknora-firstboot.service
install -m 0755 "${SCRIPT_DIR}/firstboot.sh" /usr/local/sbin/weknora-firstboot.sh
# 把 systemd 单元里的 docker 路径模板替换为实际路径
sed -i "s|@DOCKER_BIN@|${DOCKER_BIN}|g" /etc/systemd/system/weknora.service
systemctl daemon-reload
systemctl enable weknora.service
systemctl enable weknora-firstboot.service
echo "[prepare] 6/6 完成"
echo
echo " WeKnora 运行时已部署到 ${WEKNORA_DIR}"
echo " docker-compose.yml / config/config.yaml / skills/preloaded / .env"
echo " 版本: ${WEKNORA_REF} (见 ${WEKNORA_DIR}/.cloud-image-meta)"
echo
echo " 打开浏览器访问 http://<本机公网IP> 验证功能"
echo
echo " 验证通过后执行清理并制作镜像:"
echo " sudo bash ${SCRIPT_DIR}/cleanup.sh"
@@ -0,0 +1,21 @@
[Unit]
Description=WeKnora (docker compose)
Requires=docker.service weknora-firstboot.service
After=docker.service network-online.target weknora-firstboot.service
Wants=network-online.target
# 关键: Requires=weknora-firstboot.service 让首启 init 失败时本服务也不启动,
# 避免 firstboot 没成功生成随机密钥就用 .env.example 默认明文密码把 postgres
# 数据卷初始化掉。weknora-firstboot.service 在已初始化的实例上由
# ConditionPathExists=!/opt/WeKnora/.firstboot.done 跳过, 跳过 = 成功,
# 不会阻塞本服务后续重启。
[Service]
Type=oneshot
RemainAfterExit=yes
WorkingDirectory=/opt/WeKnora
ExecStart=@DOCKER_BIN@ compose up -d
ExecStop=@DOCKER_BIN@ compose down
TimeoutStartSec=600
[Install]
WantedBy=multi-user.target
+52
View File
@@ -0,0 +1,52 @@
const fs = require('node:fs')
const path = require('node:path')
const assert = require('node:assert/strict')
const root = path.resolve(__dirname, '../..')
const model = '404e005b-83ae-5e2e-a3b3-999f9d5cd47e'
const tenant = '10059'
let token
async function api(route, method = 'GET', body) {
const response = await fetch(`http://127.0.0.1:18080/api/v1${route}`, { method,
headers: { 'Content-Type': 'application/json', ...(token ? { Authorization: `Bearer ${token}`, 'X-Tenant-ID': tenant } : {}) },
body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(30000) })
const result = await response.json()
assert.ok(response.ok && result.success !== false, `${method} ${route}: HTTP ${response.status}`)
return result
}
async function main() {
const credentials = JSON.parse(fs.readFileSync(path.join(root, 'acceptance/super-admin-login.json'), 'utf8'))
const login = await api('/auth/login', 'POST', { username: credentials.username, password: credentials.password })
token = (login.data || login).token
const before = (await api('/knowledge-bases')).data
const targets = before.filter(kb => !kb.summary_model_id && kb.embedding_model_id)
if (!process.argv.includes('--apply')) { console.log(JSON.stringify({ missing_models: targets.map(kb => kb.name) })); return }
fs.writeFileSync(path.join(root, 'acceptance', `kb-model-config-before-${Date.now()}.json`), JSON.stringify(targets, null, 2), { mode: 0o600, flag: 'wx' })
for (const kb of targets) {
const c = kb.chunking_config || {}, e = kb.extract_config || {}, q = kb.question_generation_config || {}
const request = {
llmModelId: model, embeddingModelId: kb.embedding_model_id,
vlm_config: kb.vlm_config, asr_config: kb.asr_config,
multimodal: { enabled: !!kb.vlm_config?.enabled },
storageBackendId: kb.storage_backend_id, storageProvider: kb.storage_config?.provider || 'local',
documentSplitting: {
chunkSize: c.chunk_size || 512, chunkOverlap: c.chunk_size ? c.chunk_overlap : 80,
separators: c.separators?.length ? c.separators : ['\n\n', '\n', '。', '!', '?', ';', ';'],
parserEngineRules: c.parser_engine_rules, enableParentChild: !!c.enable_parent_child,
parentChunkSize: c.parent_chunk_size || 4096, childChunkSize: c.child_chunk_size || 384,
strategy: c.strategy, tokenLimit: c.token_limit, languages: c.languages, tableMetadataInstructions: c.table_metadata_instructions,
},
nodeExtract: { ...e, customInstructions: e.custom_instructions },
questionGeneration: { enabled: !!q.enabled, questionCount: q.question_count, customInstructions: q.custom_instructions },
}
await api(`/initialization/config/${kb.id}`, 'PUT', request)
const after = (await api(`/knowledge-bases/${kb.id}`)).data
assert.equal(after.summary_model_id, model)
assert.equal(after.embedding_model_id, kb.embedding_model_id)
assert.deepEqual(after.indexing_strategy, kb.indexing_strategy)
assert.deepEqual(after.wiki_config, kb.wiki_config)
assert.equal(after.storage_backend_id, kb.storage_backend_id)
}
const after = (await api('/knowledge-bases')).data
console.log(JSON.stringify({ updated: targets.map(kb => kb.name), initialized: after.filter(kb => kb.summary_model_id && kb.embedding_model_id).length, total: after.length, existing_indexes_preserved: true }))
}
main().catch(error => { console.error(error.message); process.exitCode = 1 })
Executable
+601
View File
@@ -0,0 +1,601 @@
#!/bin/bash
# 开发环境启动脚本 - 只启动基础设施,app 和 frontend 需要手动在本地运行
# 设置颜色
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m' # 无颜色
# 获取项目根目录
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
# 日志函数
log_info() {
printf "%b\n" "${BLUE}[INFO]${NC} $1"
}
log_success() {
printf "%b\n" "${GREEN}[SUCCESS]${NC} $1"
}
log_error() {
printf "%b\n" "${RED}[ERROR]${NC} $1"
}
log_warning() {
printf "%b\n" "${YELLOW}[WARNING]${NC} $1"
}
# 选择可用的 Docker Compose 命令
DOCKER_COMPOSE_BIN=""
DOCKER_COMPOSE_SUBCMD=""
detect_compose_cmd() {
if docker compose version &> /dev/null; then
DOCKER_COMPOSE_BIN="docker"
DOCKER_COMPOSE_SUBCMD="compose"
return 0
fi
if command -v docker-compose &> /dev/null; then
if docker-compose version &> /dev/null; then
DOCKER_COMPOSE_BIN="docker-compose"
DOCKER_COMPOSE_SUBCMD=""
return 0
fi
fi
return 1
}
# 显示帮助信息
show_help() {
printf "%b\n" "${GREEN}WeKnora 开发环境脚本${NC}"
echo "用法: $0 [命令] [选项]"
echo ""
echo "命令:"
echo " start 启动基础设施服务(postgres, redis, docreader, langfuse)"
echo " stop 停止所有服务"
echo " restart 重启所有服务"
echo " logs 查看服务日志"
echo " status 查看服务状态"
echo " app 启动后端应用(本地运行)"
echo " frontend 启动前端开发服务器(本地运行)"
echo " help 显示此帮助信息"
echo ""
echo "可选 Profile(用于 start 命令):"
echo " --minio 启动 MinIO 对象存储"
echo " --qdrant 启动 Qdrant 向量数据库"
echo " --neo4j 启动 Neo4j 图数据库"
echo " --dex 启动 Dex(OIDC 身份认证)"
echo " --langfuse 启动 Langfuse(默认已开启)"
echo " --no-langfuse 不启动 Langfuse"
echo " --odl-hybrid 启动 OpenDataLoader hybrid(Docling,镜像较大,按需启用)"
echo " --full 启动所有可选服务(不含 odl-hybrid,需另加 --odl-hybrid)"
echo ""
echo "示例:"
echo " $0 start # 启动基础服务"
echo " $0 start --qdrant # 启动基础服务 + Qdrant"
echo " $0 start --dex # 启动基础服务 + Dex"
echo " $0 start --odl-hybrid # 启动基础服务 + OpenDataLoader hybrid"
echo " $0 start --full # 启动所有服务"
echo " make dev-start DEV_ARGS=--odl-hybrid # 同上(Makefile 传参)"
echo " $0 app # 在另一个终端启动后端"
echo " $0 frontend # 在另一个终端启动前端"
}
# 加载 .env 与可选的 .env.local(后者覆盖前者)
# 读取时去掉行尾 \r,兼容 Windows 风格(CRLF)换行符,
# 否则 bash source 会把残留的 \r 当成命令导致 "...: $'\r': command not found"。
# 注意:不能用 source <(sed ...)——macOS 自带 Bash 3.2 对 process substitution
# 的 source 不会把变量导入当前 shell;必须落到可 seek 的临时文件再 source。
_source_env_file() {
local src="$1"
local tmp
tmp="$(mktemp)" || return 1
sed -e 's/\r$//' "$src" > "$tmp"
set -a
# shellcheck source=/dev/null
source "$tmp"
set +a
rm -f "$tmp"
}
load_env_files() {
if [ -f ".env" ]; then
_source_env_file .env || return 1
else
return 1
fi
if [ -f ".env.local" ]; then
log_info "加载 .env.local 覆盖配置..."
_source_env_file .env.local || return 1
fi
return 0
}
# 检查 Docker
check_docker() {
if ! command -v docker &> /dev/null; then
log_error "未安装Docker,请先安装Docker"
return 1
fi
if ! detect_compose_cmd; then
log_error "未检测到 Docker Compose"
return 1
fi
if ! docker info &> /dev/null; then
log_error "Docker服务未运行"
return 1
fi
return 0
}
# 检查 .env 是否启用了 hybrid 模式(用于 --odl-hybrid 启动后重建 docreader)
_should_enable_odl_hybrid_from_env() {
local hybrid="${DOCREADER_ODL_HYBRID:-off}"
hybrid=$(printf '%s' "$hybrid" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')
case "$hybrid" in
off|"") return 1 ;;
*) return 0 ;;
esac
}
_enable_odl_hybrid_profile() {
PROFILES="$PROFILES --profile odl-hybrid"
ENABLED_SERVICES="$ENABLED_SERVICES odl-hybrid"
}
# 等待 odl-hybrid HTTP 健康检查通过(compose 启动后服务可能仍在拉依赖)
_wait_odl_hybrid_ready() {
local port="${ODL_HYBRID_PORT:-5002}"
local max_wait="${ODL_HYBRID_STARTUP_WAIT_SEC:-180}"
local waited=0
local interval=5
if ! command -v curl &> /dev/null; then
log_warning "未安装 curl,跳过 odl-hybrid 就绪等待;请手动检查 http://localhost:${port}/health"
return 0
fi
log_info "等待 odl-hybrid 就绪(最多 ${max_wait}s,首次需构建镜像: docker compose ... build odl-hybrid)..."
while [ "$waited" -lt "$max_wait" ]; do
if curl -sf "http://127.0.0.1:${port}/health" >/dev/null 2>&1; then
log_success "odl-hybrid 已就绪 (http://localhost:${port}/health)"
return 0
fi
sleep "$interval"
waited=$((waited + interval))
done
log_warning "odl-hybrid 在 ${max_wait}s 内未就绪,请查看: docker logs WeKnora-odl-hybrid"
return 1
}
# 启动基础设施服务
start_services() {
log_info "启动开发环境基础设施服务..."
check_docker
if [ $? -ne 0 ]; then
return 1
fi
cd "$PROJECT_ROOT"
# 检查 .env 文件
if [ ! -f ".env" ]; then
log_error ".env 文件不存在,请先创建"
return 1
fi
load_env_files
if [ $? -ne 0 ]; then
log_error ".env 文件不存在,请先创建"
return 1
fi
if [ -n "${DEV_REMOTE_HOST:-}" ]; then
log_warning "已配置 DEV_REMOTE_HOST=${DEV_REMOTE_HOST},跳过本地 Docker 基础设施启动"
log_info "远程服务: PostgreSQL/Redis/DocReader/Langfuse → ${DEV_REMOTE_HOST}"
log_info "接下来: make dev-app(本地后端)或 make dev-frontend(前端)"
return 0
fi
# 解析 profile 参数
shift # 移除 "start" 命令本身
# 默认启动基础设施(postgres / redis / docreader)+ langfuse,
# 其余可选服务通过 --minio / --qdrant / --neo4j / --dex / --full 按需开启。
PROFILES="--profile langfuse"
ENABLED_SERVICES="langfuse"
while [ $# -gt 0 ]; do
case "$1" in
--minio)
PROFILES="$PROFILES --profile minio"
ENABLED_SERVICES="$ENABLED_SERVICES minio"
;;
--qdrant)
PROFILES="$PROFILES --profile qdrant"
ENABLED_SERVICES="$ENABLED_SERVICES qdrant"
;;
--neo4j)
PROFILES="$PROFILES --profile neo4j"
ENABLED_SERVICES="$ENABLED_SERVICES neo4j"
;;
--dex)
PROFILES="$PROFILES --profile dex"
ENABLED_SERVICES="$ENABLED_SERVICES dex"
;;
--langfuse)
PROFILES="$PROFILES --profile langfuse"
ENABLED_SERVICES="$ENABLED_SERVICES langfuse"
;;
--no-langfuse)
PROFILES="${PROFILES//--profile langfuse/}"
ENABLED_SERVICES="${ENABLED_SERVICES//langfuse/}"
;;
--odl-hybrid)
if [[ "$ENABLED_SERVICES" != *"odl-hybrid"* ]]; then
_enable_odl_hybrid_profile
fi
;;
--full)
PROFILES="--profile full"
ENABLED_SERVICES="minio qdrant neo4j dex"
break
;;
*)
log_warning "未知参数: $1"
;;
esac
shift
done
# 启动服务(odl-hybrid 单独 --build,避免每次重建 docreader)
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml $PROFILES up -d
local compose_rc=$?
if [ "$compose_rc" -eq 0 ] && [[ "$ENABLED_SERVICES" == *"odl-hybrid"* ]]; then
log_info "构建/更新 odl-hybrid 镜像..."
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml $PROFILES up -d --build odl-hybrid
_wait_odl_hybrid_ready || true
# docreader 需读取 DOCREADER_ODL_HYBRID;若刚改 .env,强制重建以注入环境变量
if _should_enable_odl_hybrid_from_env; then
log_info "重建 docreader 以应用 DOCREADER_ODL_HYBRID=${DOCREADER_ODL_HYBRID} ..."
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml up -d --force-recreate docreader
fi
fi
if [ "$compose_rc" -eq 0 ]; then
log_success "基础设施服务已启动"
echo ""
log_info "服务访问地址:"
echo " - PostgreSQL: localhost:5432"
echo " - Redis: localhost:6379"
echo " - DocReader: localhost:50051"
# 根据启用的 profile 显示额外服务
if [[ "$ENABLED_SERVICES" == *"minio"* ]]; then
echo " - MinIO: localhost:9000 (Console: localhost:9001)"
fi
if [[ "$ENABLED_SERVICES" == *"qdrant"* ]]; then
echo " - Qdrant: localhost:6333 (gRPC: localhost:6334)"
fi
if [[ "$ENABLED_SERVICES" == *"neo4j"* ]]; then
echo " - Neo4j: localhost:7474 (Bolt: localhost:7687)"
fi
if [[ "$ENABLED_SERVICES" == *"dex"* ]]; then
echo " - Dex: localhost:5556"
fi
if [[ "$ENABLED_SERVICES" == *"langfuse"* ]]; then
echo " - Langfuse: http://localhost:${LANGFUSE_WEB_PORT:-3000}"
fi
if [[ "$ENABLED_SERVICES" == *"odl-hybrid"* ]]; then
echo " - ODL Hybrid: http://localhost:${ODL_HYBRID_PORT:-5002} (health: /health)"
echo " docreader 需 DOCREADER_ODL_HYBRID=docling-fast"
fi
echo ""
log_info "接下来的步骤:"
printf "%b\n" "${YELLOW}1. 在新终端运行后端:${NC} make dev-app"
printf "%b\n" "${YELLOW}2. 在新终端运行前端:${NC} make dev-frontend"
return 0
else
log_error "服务启动失败"
return 1
fi
}
# 停止服务
stop_services() {
log_info "停止开发环境服务..."
check_docker
if [ $? -ne 0 ]; then
return 1
fi
cd "$PROJECT_ROOT"
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml down
if [ $? -eq 0 ]; then
log_success "所有服务已停止"
return 0
else
log_error "服务停止失败"
return 1
fi
}
# 重启服务
restart_services() {
stop_services
sleep 2
start_services
}
# 查看日志
show_logs() {
check_docker
if [ $? -ne 0 ]; then
return 1
fi
cd "$PROJECT_ROOT"
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml logs -f
}
# 查看状态
show_status() {
check_docker
if [ $? -ne 0 ]; then
return 1
fi
cd "$PROJECT_ROOT"
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml ps
}
# 远程开发模式下检查基础设施端口是否可达
check_remote_dev_connectivity() {
local host="${DEV_REMOTE_HOST:-}"
if [ -z "$host" ]; then
return 0
fi
local db_port="${DB_PORT:-5432}"
local redis_port
redis_port="${REDIS_ADDR#*:}"
if [ "$redis_port" = "$REDIS_ADDR" ]; then
redis_port=6379
fi
local docreader_port="${DOCREADER_PORT:-50051}"
log_info "检查远程基础设施连通性 (${host})..."
local failed=0
for spec in "PostgreSQL:${host}:${db_port}" "Redis:${host}:${redis_port}" "DocReader:${host}:${docreader_port}"; do
local name="${spec%%:*}"
local rest="${spec#*:}"
local h="${rest%%:*}"
local p="${rest##*:}"
if command -v nc &> /dev/null; then
if nc -z -w 3 "$h" "$p" 2>/dev/null; then
log_success "${name} ${h}:${p} 可达"
else
log_error "${name} ${h}:${p} 不可达 (no route / connection refused)"
failed=1
fi
else
log_warning "未安装 nc,跳过 ${name} 连通性检查"
fi
done
if [ "$failed" -ne 0 ]; then
echo ""
log_error "无法连接远程开发环境 ${host}"
log_info "排查建议:"
echo " 1. 确认远程机器 Docker 容器在运行 (postgres/redis/docreader)"
echo " 2. 确认本机与 ${host} 在同一局域网 (本机: $(ipconfig getifaddr en0 2>/dev/null || echo '未知'))"
echo " 3. 在远程检查端口映射: docker ps --format 'table {{.Names}}\t{{.Ports}}'"
echo " 4. 检查远程防火墙是否放行 5432/6379/50051"
return 1
fi
return 0
}
# Host-platform path of the anydoc static archive (built by `make anydoc-lib`).
anydoc_host_archive() {
case "$(uname -s)-$(uname -m)" in
Darwin-arm64) echo "$PROJECT_ROOT/third_party/anydoc-go/lib/darwin_arm64/libanydoc_go.a" ;;
Darwin-x86_64) echo "$PROJECT_ROOT/third_party/anydoc-go/lib/darwin_amd64/libanydoc_go.a" ;;
Linux-x86_64)
if [ -f "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_amd64_gnu/libanydoc_go.a" ]; then
echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_amd64_gnu/libanydoc_go.a"
else
echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_amd64_musl/libanydoc_go.a"
fi
;;
Linux-aarch64)
if [ -f "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_arm64_gnu/libanydoc_go.a" ]; then
echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_arm64_gnu/libanydoc_go.a"
else
echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_arm64_musl/libanydoc_go.a"
fi
;;
*) echo "" ;;
esac
}
# Enable the in-process anydoc engine when the archive is present, unless the
# caller already set GO_BUILD_TAGS (including empty, which opts out).
enable_anydoc_build_tag() {
if [ -n "${GO_BUILD_TAGS+x}" ]; then
export GO_BUILD_TAGS
return
fi
local archive
archive="$(anydoc_host_archive)"
if [ -n "$archive" ] && [ -f "$archive" ]; then
export GO_BUILD_TAGS=anydoc
log_info "检测到 anydoc 静态库,已启用 -tags anydoc"
else
log_info "未检测到 anydoc 静态库,解析引擎不可用。需要时先运行: make anydoc-lib"
fi
}
# 启动后端应用(本地)
start_app() {
log_info "启动后端应用(本地开发模式)..."
cd "$PROJECT_ROOT"
# 检查 Go 是否安装
if ! command -v go &> /dev/null; then
log_error "Go 未安装"
return 1
fi
log_info "加载环境配置..."
if ! load_env_files; then
log_error ".env 文件不存在,请先创建配置文件"
return 1
fi
# 本地 docker-compose.dev 模式:把容器服务名映射到宿主机回环地址
# 远程开发模式(DEV_REMOTE_HOST 或 .env.local 已设地址)则保留 .env/.env.local 中的值
if [ -n "${DEV_REMOTE_HOST:-}" ]; then
log_info "远程开发模式: 基础设施 → ${DEV_REMOTE_HOST}"
export DB_HOST="${DB_HOST:-$DEV_REMOTE_HOST}"
export REDIS_ADDR="${REDIS_ADDR:-$DEV_REMOTE_HOST:6379}"
export DOCREADER_ADDR="${DOCREADER_ADDR:-$DEV_REMOTE_HOST:50051}"
export MINIO_ENDPOINT="${MINIO_ENDPOINT:-$DEV_REMOTE_HOST:9000}"
export MILVUS_ADDRESS="${MILVUS_ADDRESS:-$DEV_REMOTE_HOST:19530}"
export NEO4J_URI="${NEO4J_URI:-bolt://$DEV_REMOTE_HOST:7687}"
export QDRANT_HOST="${QDRANT_HOST:-$DEV_REMOTE_HOST}"
if [ -z "${LANGFUSE_HOST:-}" ] || [ "$LANGFUSE_HOST" = "http://langfuse-web:3000" ]; then
export LANGFUSE_HOST="http://${DEV_REMOTE_HOST}:3000"
fi
else
export DB_HOST=127.0.0.1
export DOCREADER_ADDR=127.0.0.1:50051
export MINIO_ENDPOINT=127.0.0.1:9000
export REDIS_ADDR=127.0.0.1:6379
export MILVUS_ADDRESS=127.0.0.1:19530
export NEO4J_URI=bolt://127.0.0.1:7687
export QDRANT_HOST=127.0.0.1
fi
export DOCREADER_TRANSPORT="${DOCREADER_TRANSPORT:-grpc}"
if ! check_remote_dev_connectivity; then
return 1
fi
# .env.example uses /data/files for the Docker app container, where a
# volume is mounted at that path. When the backend runs directly on the
# host via dev-app, /data is often read-only or missing, so use a repo-local
# writable directory unless the developer explicitly configured another
# local storage path.
if [ -z "${LOCAL_STORAGE_BASE_DIR:-}" ] || [ "$LOCAL_STORAGE_BASE_DIR" = "/data/files" ]; then
export LOCAL_STORAGE_BASE_DIR="$PROJECT_ROOT/.local-data/files"
fi
mkdir -p "$LOCAL_STORAGE_BASE_DIR"
# 确保必要的环境变量已设置
if [ -z "$DB_DRIVER" ]; then
log_error "DB_DRIVER 环境变量未设置,请检查 .env 文件"
return 1
fi
log_info "环境变量已设置,启动应用..."
log_info "数据库地址: $DB_HOST:${DB_PORT:-5432}"
export CGO_CFLAGS="-Wno-deprecated-declarations -Wno-gnu-folding-constant"
if [[ "$(uname)" == "Darwin" ]]; then
export CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries"
fi
enable_anydoc_build_tag
# 检查是否安装了 Air(热重载工具)
if command -v air &> /dev/null; then
log_success "检测到 Air,使用热重载模式启动..."
log_info "修改 Go 代码后将自动重新编译和重启"
air
else
log_info "未检测到 Air,使用普通模式启动"
log_warning "提示: 安装 Air 可以实现代码修改后自动重启"
log_info "安装命令: go install github.com/air-verse/air@latest"
LDFLAGS="$(./scripts/get_version.sh ldflags) -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"
go run -tags "${GO_BUILD_TAGS:-}" -ldflags="$LDFLAGS" ./cmd/server
fi
}
# 启动前端(本地)
start_frontend() {
log_info "启动前端开发服务器..."
cd "$PROJECT_ROOT"
if [ -f ".env" ] || [ -f ".env.local" ]; then
load_env_files >/dev/null 2>&1 || true
fi
cd "$PROJECT_ROOT/frontend"
# 检查 npm 是否安装
if ! command -v npm &> /dev/null; then
log_error "npm 未安装"
return 1
fi
# 检查依赖是否已安装
if [ ! -d "node_modules" ]; then
log_warning "node_modules 不存在,正在安装依赖..."
npm install
fi
log_info "启动 Vite 开发服务器..."
log_info "前端将运行在 http://localhost:5173"
log_info "前端 API 代理目标: ${VITE_DEV_PROXY_TARGET:-${FRONTEND_BACKEND_URL:-http://localhost:8080}}"
# 运行开发服务器
npm run dev
}
# 解析命令
CMD="${1:-help}"
case "$CMD" in
start)
start_services "$@"
;;
stop)
stop_services
;;
restart)
restart_services
;;
logs)
show_logs
;;
status)
show_status
;;
app)
start_app
;;
frontend)
start_frontend
;;
help|--help|-h)
show_help
;;
*)
log_error "未知命令: $CMD"
show_help
exit 1
;;
esac
exit 0
+101
View File
@@ -0,0 +1,101 @@
const fs = require('node:fs')
const path = require('node:path')
const { execFile } = require('node:child_process')
const { promisify } = require('node:util')
const execute = promisify(execFile)
const directory = path.resolve(__dirname, '../../acceptance')
const startedAt = new Date().toISOString()
const reportPath = path.join(directory, `public-latency-${startedAt.replace(/[:.]/g, '-')}.json`)
const report = { started_at: startedAt, samples: [], summaries: [] }
const headerNames = ['cache-control', 'cf-cache-status', 'cf-ray', 'age', 'etag', 'last-modified', 'content-type', 'content-encoding', 'content-length', 'vary', 'server']
const measurementFormat = '\n__BEGIN__%{json}__HEADERS__%{header_json}__END__\n'
function decode(output, label, url) {
return [...output.matchAll(/__BEGIN__([\s\S]*?)__HEADERS__([\s\S]*?)__END__/g)].map((match, index) => {
const measurement = JSON.parse(match[1])
const headers = JSON.parse(match[2])
const ms = (value) => Math.round(value * 1000000) / 1000
return {
label, url, connection_sequence: index + 1,
status: measurement.http_code, exit_code: measurement.exitcode,
http_version: measurement.http_version, new_connections: measurement.num_connects,
remote_ip: measurement.remote_ip, downloaded_bytes: measurement.size_download,
dns_ms: ms(measurement.time_namelookup),
connect_ms: ms(measurement.time_connect), tls_ms: ms(measurement.time_appconnect),
ttfb_ms: ms(measurement.time_starttransfer), total_ms: ms(measurement.time_total),
wait_after_request_ready_ms: ms(measurement.time_starttransfer - measurement.time_pretransfer),
headers: Object.fromEntries(headerNames.filter(name => headers[name]).map(name => [name, headers[name]])),
}
})
}
async function measure(label, url, family, repetitions = 1) {
const args = ['--silent', '--show-error', '--compressed', '--connect-timeout', '8', '--max-time', '15', '--write-out', measurementFormat]
if (family) args.push(family)
for (let index = 0; index < repetitions; index++) args.push('--output', '/dev/null', url)
let result
try { result = await execute('curl', args, { maxBuffer: 1024 * 1024 }) }
catch (error) { result = error }
const samples = decode(result.stdout || '', label, url)
if (!samples.length) samples.push({ label, url, failed: true, exit_code: result.code || 1 })
report.samples.push(...samples)
console.log(JSON.stringify({ label, samples: samples.map(sample => ({ status: sample.status, ttfb_ms: sample.ttfb_ms, total_ms: sample.total_ms, exit_code: sample.exit_code })) }))
}
async function tunnelMetrics() {
try {
const response = await fetch('http://127.0.0.1:20241/metrics', { signal: AbortSignal.timeout(3000) })
const lines = (await response.text()).split('\n')
return lines.filter(line => /^cloudflared_tunnel_(?:ha_connections|request_errors|total_requests|server_locations)/.test(line))
} catch { return ['metrics_unavailable'] }
}
function summarize() {
for (const label of new Set(report.samples.map(sample => sample.label))) {
const samples = report.samples.filter(sample => sample.label === label && sample.exit_code === 0)
const median = key => {
const values = samples.map(sample => sample[key]).sort((a, b) => a - b)
const middle = Math.floor(values.length / 2)
return values.length ? (values.length % 2 ? values[middle] : (values[middle - 1] + values[middle]) / 2) : null
}
report.summaries.push({ label, successful_samples: samples.length,
dns_median_ms: median('dns_ms'), connect_median_ms: median('connect_ms'), tls_median_ms: median('tls_ms'),
ttfb_median_ms: median('ttfb_ms'), total_median_ms: median('total_ms'), wait_median_ms: median('wait_after_request_ready_ms'),
cache_states: [...new Set(samples.flatMap(sample => sample.headers?.['cf-cache-status'] || []))],
edge_locations: [...new Set(samples.flatMap(sample => sample.headers?.['cf-ray']?.map(ray => ray.split('-').at(-1)) || []))],
})
}
}
async function main() {
report.tunnel_before = await tunnelMetrics()
const targets = [
['html', '/login'],
['small_static', '/assets/favicon-bEsoBimf.ico'],
['api_guard', '/api/v1/auth/me'],
]
for (let round = 0; round < 3; round++) {
for (const [name, resource] of targets) {
await measure(`origin_${name}`, `http://127.0.0.1:18088${resource}`)
await measure(`public_ipv4_${name}`, `https://agent.pducv.cn${resource}`, '--ipv4')
await measure(`public_ipv6_${name}`, `https://agent.pducv.cn${resource}`, '--ipv6')
}
await measure('backend_health', 'http://127.0.0.1:18080/health')
}
for (const [label, family] of [['ipv4', '--ipv4'], ['ipv6', '--ipv6']]) {
await measure(`public_${label}_warm_html`, 'https://agent.pducv.cn/login', family, 3)
await measure(`public_${label}_config`, 'https://agent.pducv.cn/config.js', family)
await measure(`public_${label}_icons`, 'https://agent.pducv.cn/tdesign-icons/0.4.1/fonts/index.js', family)
}
await measure('origin_config', 'http://127.0.0.1:18088/config.js')
await measure('origin_icons', 'http://127.0.0.1:18088/tdesign-icons/0.4.1/fonts/index.js')
report.tunnel_after = await tunnelMetrics()
}
main().catch(error => { report.error = error.name; process.exitCode = 1 }).finally(() => {
report.completed_at = new Date().toISOString()
summarize()
fs.writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n', { flag: 'wx' })
console.log(JSON.stringify({ report: reportPath, summaries: report.summaries, error: report.error }))
})
+88
View File
@@ -0,0 +1,88 @@
#!/bin/bash
set -e
# ─── Fix ownership of bind-mounted directories ───
# When users bind-mount host directories (e.g. ./skills/preloaded),
# the mount inherits the host UID/GID which may differ from the
# container's appuser. This entrypoint runs as root, fixes ownership,
# then drops privileges to appuser via gosu — the same pattern used
# by official postgres/redis images.
# Directories that may be bind-mounted and need appuser access
MOUNT_DIRS=(
/app/skills/preloaded
/data/files
)
for dir in "${MOUNT_DIRS[@]}"; do
if [ -d "$dir" ]; then
chown -R appuser:appuser "$dir" 2>/dev/null || true
fi
done
# ─── Merge built-in skills into preloaded ───
# Built-in skills are backed up at /app/skills/_builtin during image build.
# After a bind-mount replaces /app/skills/preloaded, copy back any
# missing built-in skills (without overwriting user-provided ones).
BUILTIN_DIR="/app/skills/_builtin"
PRELOADED_DIR="/app/skills/preloaded"
if [ -d "$BUILTIN_DIR" ]; then
mkdir -p "$PRELOADED_DIR"
for skill_dir in "$BUILTIN_DIR"/*/; do
[ -d "$skill_dir" ] || continue
skill_name="$(basename "$skill_dir")"
if [ ! -d "$PRELOADED_DIR/$skill_name" ]; then
cp -r "$skill_dir" "$PRELOADED_DIR/$skill_name"
fi
done
chown -R appuser:appuser "$PRELOADED_DIR"
fi
# ─── Docker socket access for the sandbox backend ───
# The Engine API socket is typically root:docker 0660. This process then
# drops to appuser via gosu, which calls initgroups and therefore drops
# compose group_add. Match the socket's GID in /etc/group before gosu.
# Never chmod the host socket: that would weaken daemon access on the host.
grant_docker_sock_to_appuser() {
local sock="$1"
local gid grp
[ -S "$sock" ] || return 0
if gosu appuser sh -c "test -r \"$sock\" && test -w \"$sock\"" 2>/dev/null; then
return 0
fi
gid="$(stat -c '%g' "$sock" 2>/dev/null || true)"
if [ -z "$gid" ]; then
echo "weknora: cannot stat $sock; Docker sandbox may be unable to reach the daemon" >&2
return 0
fi
if [ "$gid" = "0" ]; then
echo "weknora: $sock is not writable by appuser and owned by GID 0; Docker sandbox needs a group-writable socket with a non-root GID" >&2
return 0
fi
if ! getent group "$gid" >/dev/null 2>&1; then
if ! groupadd -g "$gid" dockersock >/dev/null 2>&1; then
echo "weknora: failed to create group for $sock GID $gid; Docker sandbox may be unable to reach the daemon" >&2
return 0
fi
fi
grp="$(getent group "$gid" | cut -d: -f1)"
if [ -z "$grp" ]; then
echo "weknora: no group name for GID $gid on $sock" >&2
return 0
fi
if ! usermod -aG "$grp" appuser >/dev/null 2>&1; then
echo "weknora: failed to add appuser to $grp for $sock; Docker sandbox may be unable to reach the daemon" >&2
return 0
fi
}
grant_docker_sock_to_appuser /var/run/docker.sock
case "${DOCKER_HOST:-}" in
unix://*)
grant_docker_sock_to_appuser "${DOCKER_HOST#unix://}"
;;
esac
# ─── Drop privileges and exec the main process ───
exec gosu appuser "$@"
+57
View File
@@ -0,0 +1,57 @@
const fs = require('node:fs')
const path = require('node:path')
const assert = require('node:assert/strict')
const root = path.resolve(__dirname, '../..')
const ids = ['16c3e59c-4bb6-41cf-ae0a-6d0941b3d032', '20b9280e-ed73-4700-8fa5-944d6ae6dd5f', 'bb86b44a-576b-4bf8-a6bf-8f1baca3a845', 'badd084b-f229-4005-8fb3-196fbe139fe0', '5d28a7e5-e00e-44a7-a0e1-c72342cdcf68', '3ae9257d-928a-4886-8101-7cfd0ed8c27c']
let token
async function api(route, method = 'GET', body) {
const response = await fetch(`http://127.0.0.1:18080/api/v1${route}`, { method,
headers: { 'Content-Type': 'application/json', ...(token ? { Authorization: `Bearer ${token}`, 'X-Tenant-ID': '10059' } : {}) },
body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(30000) })
const result = await response.json()
assert.ok(response.ok && result.success !== false, `${method} ${route}: HTTP ${response.status}`)
return result
}
async function main() {
const credentials = JSON.parse(fs.readFileSync(path.join(root, 'acceptance/super-admin-login.json'), 'utf8'))
const login = await api('/auth/login', 'POST', { username: credentials.username, password: credentials.password })
token = (login.data || login).token
const selected = process.argv.includes('--product-only') ? ids.slice(0, 1) : ids
const before = []
for (const id of selected) before.push((await api(`/knowledge-bases/${id}`)).data)
if (!process.argv.includes('--apply')) {
console.log(JSON.stringify(before.map(kb => ({ name: kb.name, enabled: !!kb.extract_config?.enabled, initialized: !!(kb.embedding_model_id && kb.summary_model_id) }))))
return
}
fs.writeFileSync(path.join(root, 'acceptance', `kb-graph-config-before-${Date.now()}.json`), JSON.stringify(before, null, 2), { mode: 0o600, flag: 'wx' })
for (const kb of before) {
assert.ok(kb.embedding_model_id && kb.summary_model_id, 'Models must be configured first')
const config = (await api(`/initialization/config/${kb.id}`)).data
const request = { ...config,
llmModelId: kb.summary_model_id, embeddingModelId: kb.embedding_model_id,
nodeExtract: {
enabled: true,
text: '基础概念是实践练习的先修条件。',
tags: ['先修条件', '包含', '适用于', '条件适用于', '禁用于', '慎用于', '不适用于', '不能用于', '关联', '导致', '不导致', '约束', '对比', '支持', '不支持', '抑制'],
nodes: [{ name: '基础概念', attributes: ['概念'] }, { name: '实践练习', attributes: ['方法'] }],
relations: [{ node1: '基础概念', node2: '实践练习', type: '先修条件', evidence: '基础概念是实践练习的先修条件。' }],
customInstructions: '按当前原文的实际主题提取有依据的实体和关系,不根据知识库名称预设领域。每条关系的 evidence 必须逐字引用连续原文,保留主体、时间、否定、条件、观点归属和不确定性。叙事和个人观点不能改写成通用因果;先修条件按前提指向后续目标;约束须保留完整动作。产品使用标签只适用于明确的产品使用说明,不同产品分节不得串用关系。不要把格式示例写入实际图谱,不要补造原文没有的关系;确实没有关系时如实返回,交由覆盖复核。',
},
}
await api(`/initialization/config/${kb.id}`, 'PUT', request)
await api(`/knowledge-bases/${kb.id}`, 'PUT', { name: kb.name, description: kb.description,
config: { chunking_config: kb.chunking_config, image_processing_config: kb.image_processing_config,
faq_config: kb.faq_config, auto_tag_config: kb.auto_tag_config, wiki_config: kb.wiki_config,
indexing_strategy: { ...kb.indexing_strategy, graph_enabled: true } } })
const after = (await api(`/knowledge-bases/${kb.id}`)).data
assert.equal(after.extract_config.enabled, true)
assert.equal(after.indexing_strategy.graph_enabled, true)
assert.equal(after.embedding_model_id, kb.embedding_model_id)
assert.equal(after.summary_model_id, kb.summary_model_id)
assert.equal(after.storage_backend_id, kb.storage_backend_id)
assert.deepEqual(after.wiki_config, kb.wiki_config)
assert.deepEqual(after.chunking_config, kb.chunking_config)
}
console.log(JSON.stringify({ configured: before.map(kb => kb.name), existing_indexes_preserved: true, graph_build_complete: false }))
}
main().catch(error => { console.error(error.message); process.exitCode = 1 })
+96
View File
@@ -0,0 +1,96 @@
const fs = require('node:fs')
const path = require('node:path')
const assert = require('node:assert/strict')
const { execFileSync } = require('node:child_process')
const root = path.resolve(__dirname, '../..')
const directory = path.join(root, 'acceptance')
const tenant = 10059
const businessIDs = ['16c3e59c-4bb6-41cf-ae0a-6d0941b3d032', '20b9280e-ed73-4700-8fa5-944d6ae6dd5f', 'bb86b44a-576b-4bf8-a6bf-8f1baca3a845', 'badd084b-f229-4005-8fb3-196fbe139fe0', '5d28a7e5-e00e-44a7-a0e1-c72342cdcf68', '3ae9257d-928a-4886-8101-7cfd0ed8c27c']
const model = '404e005b-83ae-5e2e-a3b3-999f9d5cd47e'
const origin = 'http://127.0.0.1:18080/api/v1'
const statePath = path.join(directory, 'business-wiki-backfill.json')
const literal = s => `'${String(s).replaceAll("'", "''")}'`
const docker = (args, input) => execFileSync('docker', ['--context', 'colima-yawan', ...args], {input, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], maxBuffer: 8 * 1024 * 1024})
const sql = statement => docker(['exec', '-i', 'WeKnora-postgres', 'psql', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-At', '-v', 'ON_ERROR_STOP=1'], statement).trim()
let auth
async function api(route, method = 'GET', body) {
const response = await fetch(origin + route, {method, headers: {'Content-Type': 'application/json', ...(auth ? {Authorization: `Bearer ${auth.token}`, 'X-Tenant-ID': String(tenant)} : {})}, body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(30000)})
const value = await response.json()
assert.ok(response.ok && value.success !== false, `${method} ${route}: HTTP ${response.status}`)
return value
}
function inventory() {
return JSON.parse(sql(`SELECT coalesce(json_agg(row_to_json(t)),'[]'::json) FROM (
SELECT k.id,k.name,k.type,k.indexing_strategy,k.wiki_config,k.embedding_model_id,
(SELECT count(*) FROM knowledges n WHERE n.knowledge_base_id=k.id AND n.deleted_at IS NULL) AS documents,
(SELECT count(*) FROM chunks c WHERE c.knowledge_base_id=k.id AND c.deleted_at IS NULL) AS chunks,
(SELECT count(*) FROM wiki_pages p WHERE p.knowledge_base_id=k.id AND p.deleted_at IS NULL) AS wiki_pages,
(SELECT count(*) FROM task_pending_ops o WHERE o.scope_id=k.id AND o.task_type='wiki:ingest') AS pending_documents,
(SELECT count(*) FROM task_pending_ops o WHERE o.scope_id=k.id AND o.task_type='wiki:finalize') AS pending_finalization,
(SELECT count(*) FROM task_dead_letters d WHERE d.scope_id=k.id AND d.task_type='wiki:ingest') AS failed_documents
FROM knowledge_bases k WHERE k.tenant_id=${tenant} AND k.id IN (${businessIDs.map(literal).join(',')}) AND k.deleted_at IS NULL AND k.is_temporary=false ORDER BY k.name
) t;`))
}
async function main() {
const credentials = JSON.parse(fs.readFileSync(path.join(directory, 'super-admin-login.json'), 'utf8'))
auth = await api('/auth/login', 'POST', {username: credentials.username, password: credentials.password})
assert.equal(auth.user.is_system_admin, true)
const before = inventory()
assert.equal(before.length, businessIDs.length, 'All six business knowledge bases must exist')
let state = fs.existsSync(statePath) ? JSON.parse(fs.readFileSync(statePath, 'utf8')) : {started_at: new Date().toISOString(), tenant_id: tenant, before}
if (process.argv.includes('--apply')) {
const modelRow = sql(`SELECT count(*) FROM models WHERE id=${literal(model)} AND tenant_id=${tenant} AND deleted_at IS NULL AND type='KnowledgeQA';`)
assert.equal(modelRow, '1')
fs.writeFileSync(path.join(directory, `business-wiki-config-before-${Date.now()}.json`), JSON.stringify(before, null, 2), {mode: 0o600, flag: 'wx'})
for (const row of before) {
assert.equal(row.type, 'document', 'Only document knowledge bases are supported')
assert.equal(row.indexing_strategy.vector_enabled, true, 'Existing vector index is required')
assert.equal(row.indexing_strategy.keyword_enabled, true, 'Existing keyword index is required')
const kb = (await api(`/knowledge-bases/${row.id}`)).data
const updated = await api(`/knowledge-bases/${row.id}`, 'PUT', {name: kb.name, description: kb.description, config: {
chunking_config: kb.chunking_config, image_processing_config: kb.image_processing_config,
faq_config: kb.faq_config, auto_tag_config: kb.auto_tag_config,
indexing_strategy: {...kb.indexing_strategy, wiki_enabled: true},
wiki_config: {...kb.wiki_config, synthesis_model_id: model, extraction_granularity: 'standard', max_pages_per_ingest: 0,
ingest_batch_size: 1, ingest_map_parallel: 1, ingest_reduce_parallel: 1, ingest_max_inflight: 1},
}})
assert.equal(updated.data.indexing_strategy.wiki_enabled, true)
assert.equal(updated.data.embedding_model_id, row.embedding_model_id)
}
const ids = before.map(k => literal(k.id)).join(',')
// Backfill the Wiki queue without resetting document status or rebuilding RAG indexes.
state.enqueued = Number(sql(`WITH added AS (
INSERT INTO task_pending_ops (tenant_id,task_type,scope,scope_id,op,dedup_key,payload,fail_count,enqueued_at)
SELECT n.tenant_id,'wiki:ingest','knowledge_base',n.knowledge_base_id,'ingest',n.id,
jsonb_build_object('op','ingest','knowledge_id',n.id,'language','zh-CN'),0,now()
FROM knowledges n JOIN knowledge_bases k ON k.id=n.knowledge_base_id AND k.tenant_id=n.tenant_id
WHERE n.tenant_id=${tenant} AND n.knowledge_base_id IN (${ids}) AND n.deleted_at IS NULL
AND k.deleted_at IS NULL AND k.indexing_strategy->>'wiki_enabled'='true'
AND n.parse_status='completed' AND n.pending_subtasks_count=0
AND EXISTS (SELECT 1 FROM chunks c WHERE c.knowledge_id=n.id AND c.deleted_at IS NULL)
AND NOT EXISTS (SELECT 1 FROM task_pending_ops o WHERE o.tenant_id=n.tenant_id AND o.scope_id=n.knowledge_base_id AND o.task_type='wiki:ingest' AND o.dedup_key=n.id)
AND NOT EXISTS (SELECT 1 FROM wiki_pages p WHERE p.knowledge_base_id=n.knowledge_base_id AND p.deleted_at IS NULL AND p.source_refs @> jsonb_build_array(n.id))
RETURNING id
) SELECT count(*) FROM added;`))
state.configured_at = new Date().toISOString()
fs.writeFileSync(statePath, JSON.stringify(state, null, 2), {mode: 0o600})
const scopes = before.map(k => ({tenant_id: tenant, knowledge_base_id: k.id, language: 'zh-CN'}))
docker(['cp', path.join(directory, 'wiki-backfill-trigger'), 'yawan-native-acceptance:/tmp/wiki-backfill-trigger'])
state.trigger_result = JSON.parse(docker(['exec', '-i', 'yawan-native-acceptance', '/tmp/wiki-backfill-trigger'], JSON.stringify(scopes)))
}
state.checked_at = new Date().toISOString()
state.current = inventory()
for (const prior of state.before) {
const current = state.current.find(k => k.id === prior.id)
assert.equal(current.documents, prior.documents)
assert.equal(current.chunks, prior.chunks)
assert.equal(current.embedding_model_id, prior.embedding_model_id)
}
state.rag_preserved = true
state.all_wiki_enabled = state.current.every(k => k.indexing_strategy.wiki_enabled && k.indexing_strategy.vector_enabled && k.indexing_strategy.keyword_enabled)
if (process.argv.includes('--apply')) assert.equal(state.all_wiki_enabled, true)
fs.writeFileSync(statePath, JSON.stringify(state, null, 2), {mode: 0o600})
console.log(JSON.stringify({checked_at: state.checked_at, all_wiki_enabled: state.all_wiki_enabled, rag_preserved: true, enqueued: state.enqueued, knowledge_bases: state.current.map(k => ({name: k.name,documents: k.documents,chunks: k.chunks,wiki_pages: k.wiki_pages,pending_documents:k.pending_documents,pending_finalization:k.pending_finalization,failed_documents:k.failed_documents}))}))
}
main().catch(error => {console.error(error.message); process.exitCode = 1})
+91
View File
@@ -0,0 +1,91 @@
const assert = require('node:assert/strict')
const fs = require('node:fs')
const path = require('node:path')
const { execFileSync } = require('node:child_process')
const { idle } = require('./publish_public_runtime.cjs')
const root = path.resolve(__dirname, '../..')
const modelID = '404e005b-83ae-5e2e-a3b3-999f9d5cd47e'
const tenantID = 10059
const report = { checked_at: new Date().toISOString(), model_id: modelID, tenant_id: tenantID, model_calls: 0, changed: false }
let token, stage = 'snapshot'
function snapshot() {
const sql = `BEGIN READ ONLY;
SELECT json_build_object('model_id',id,'provider',parameters->>'provider',
'thinking_control',parameters->'extra_config'->>'thinking_control',
'unchanged_fields_sha256',encode(sha256(convert_to((to_jsonb(models) - 'updated_at' - 'parameters' ||
jsonb_build_object('parameters',parameters #- '{extra_config,thinking_control}'))::text,'UTF8')),'hex'))
FROM models WHERE id='${modelID}' AND tenant_id=${tenantID} AND deleted_at IS NULL;
COMMIT;`
const result = execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres',
'psql', '-X', '-qAt', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915'],
{ input: sql, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 15000 })
return JSON.parse(result)
}
async function api(route, method = 'GET', body) {
const response = await fetch(`http://127.0.0.1:18080/api/v1${route}`, {
method, headers: { 'Content-Type': 'application/json', ...(token ? { Authorization: `Bearer ${token}`, 'X-Tenant-ID': String(tenantID) } : {}) },
body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(30000),
})
assert.equal(response.status, 200)
const result = await response.json()
assert.notEqual(result.success, false)
return result.data || result
}
async function main() {
const before = snapshot()
report.before = before
assert.equal(before.provider, 'deepseek')
assert.equal(before.thinking_control, 'none')
if (!process.argv.includes('--apply')) {
console.log(JSON.stringify({ ...report, proposed: 'thinking_type', dry_run: true }))
return
}
stage = 'idle_check'
idle()
stage = 'login'
const credentials = JSON.parse(fs.readFileSync(path.join(root, 'acceptance/super-admin-login.json'), 'utf8'))
token = (await api('/auth/login', 'POST', credentials)).token
assert.ok(token)
stage = 'model_read'
const model = await api(`/models/${modelID}`)
assert.equal(model.tenant_id, tenantID)
assert.equal(model.name, 'deepseek-flash')
assert.equal(model.type, 'KnowledgeQA')
assert.equal(model.source, 'remote')
assert.equal(model.parameters.provider, 'deepseek')
assert.equal(new URL(model.parameters.base_url).hostname, 'api.deepseek.com')
assert.equal(model.parameters.extra_config?.thinking_control, 'none')
const receiptPath = path.join(root, 'acceptance', `graph-model-thinking-fix-${Date.now()}.json`)
fs.writeFileSync(receiptPath, JSON.stringify(report, null, 2) + '\n', { flag: 'wx', mode: 0o600 })
stage = 'compare_before_update'
assert.deepEqual(snapshot(), before)
stage = 'model_update'
await api(`/models/${modelID}`, 'PUT', {
name: model.name, display_name: model.display_name, description: model.description,
type: model.type, source: model.source,
parameters: { ...model.parameters, extra_config: { ...model.parameters.extra_config, thinking_control: 'thinking_type' } },
})
report.changed = true
stage = 'verify'
report.after = snapshot()
fs.writeFileSync(receiptPath, JSON.stringify(report, null, 2) + '\n', { mode: 0o600 })
assert.equal(report.after.thinking_control, 'thinking_type')
assert.equal(report.after.unchanged_fields_sha256, before.unchanged_fields_sha256)
const after = await api(`/models/${modelID}`)
assert.equal(after.parameters.extra_config.thinking_control, 'thinking_type')
report.passed = true
report.verified_at = new Date().toISOString()
fs.writeFileSync(receiptPath, JSON.stringify(report, null, 2) + '\n', { mode: 0o600 })
console.log(JSON.stringify({ passed: true, changed: true, before: 'none', after: 'thinking_type',
unrelated_fields_and_credentials_preserved: true, model_calls: 0, receipt_path: receiptPath }))
}
main().catch(() => {
console.error(JSON.stringify({ passed: false, stage, changed: report.changed, model_calls: 0,
error: 'Guarded model configuration update stopped; inspect the private receipt without printing credentials.' }))
process.exitCode = 1
})
+121
View File
@@ -0,0 +1,121 @@
const assert = require('node:assert/strict')
const { execFileSync } = require('node:child_process')
const fs = require('node:fs')
const os = require('node:os')
const path = require('node:path')
const test = require('node:test')
const { setTimeout: delay } = require('node:timers/promises')
const frontend = path.resolve(__dirname, '../frontend')
const dockerContext = process.env.NGINX_TEST_DOCKER_CONTEXT
const docker = (...args) => execFileSync('docker', [
...(dockerContext ? ['--context', dockerContext] : []), ...args,
], { encoding: 'utf8', timeout: 30000, stdio: ['ignore', 'pipe', 'pipe'] }).trim()
// Runs only a disposable nginx with static fixtures, never the application stack.
test('nginx serves runtime config and versioned assets with distinct cache policies', { timeout: 90000 }, async t => {
const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'weknora-cache-test-'))
const container = `weknora-cache-test-${process.pid}-${Date.now()}`
const image = process.env.NGINX_TEST_IMAGE || fs.readFileSync(path.join(frontend, 'Dockerfile'), 'utf8').match(/^FROM (\S+)/m)[1]
let created = false
try {
const site = path.join(temp, 'html')
const fixtures = {
'index.html': '<html>main cache fixture</html>',
'embed.html': '<html>embed cache fixture</html>',
'config.js': 'window.__RUNTIME_CONFIG__ = { MAX_FILE_SIZE_MB: 50 };',
'tdesign-icons/0.4.1/fonts/index.js': 'window.cacheTestIcon = true;',
'assets/cache-test-012345.js': 'window.cacheTestAsset = true;',
}
for (const [name, body] of Object.entries(fixtures)) {
const filename = path.join(site, name)
fs.mkdirSync(path.dirname(filename), { recursive: true })
fs.writeFileSync(filename, body)
}
const replacements = { APP_SCHEME: 'http', APP_HOST: '127.0.0.1', APP_PORT: '9', MAX_FILE_SIZE: '50M', MAX_SKILL_BUNDLE_SIZE: '256M' }
const config = fs.readFileSync(path.join(frontend, 'nginx.conf'), 'utf8')
.replace(/\$\{(APP_SCHEME|APP_HOST|APP_PORT|MAX_FILE_SIZE|MAX_SKILL_BUNDLE_SIZE)\}/g, (_, key) => replacements[key])
fs.writeFileSync(path.join(temp, 'default.conf'), config)
docker('create', '--rm', '--pull', 'never', '--name', container,
'--publish', '127.0.0.1::80',
'--entrypoint', 'nginx', image, '-g', 'daemon off;')
created = true
// Copying also supports remote Docker contexts without host-directory mounts.
docker('cp', `${site}/.`, `${container}:/usr/share/nginx/html`)
docker('cp', path.join(temp, 'default.conf'), `${container}:/etc/nginx/conf.d/default.conf`)
docker('cp', path.join(frontend, 'nginx-api-proxy.conf'), `${container}:/etc/nginx/api-proxy.conf`)
docker('start', container)
docker('exec', container, 'nginx', '-t')
const address = docker('port', container, '80/tcp').split('\n')[0]
assert.match(address, /^127\.0\.0\.1:\d+$/)
const base = `http://${address}`
const request = async (url, options) => {
const response = await fetch(`${base}${url}`, { ...options, signal: AbortSignal.timeout(3000) })
return { status: response.status, headers: response.headers, body: await response.text() }
}
for (let attempt = 0; ; attempt += 1) {
try { await request('/login'); break } catch (error) {
if (attempt >= 19) throw error
await delay(250)
}
}
await t.test('runtime config is never stored, including conditional responses', async () => {
const response = await request('/config.js?deployment=next')
assert.equal(response.status, 200)
assert.equal(response.body, fixtures['config.js'])
assert.equal(response.headers.get('cache-control'), 'no-store')
assert.equal(response.headers.get('x-content-type-options'), 'nosniff')
assert.ok(response.headers.get('etag'))
const conditional = await request('/config.js', { headers: { 'If-None-Match': response.headers.get('etag') } })
assert.equal(conditional.status, 304)
assert.equal(conditional.headers.get('cache-control'), 'no-store')
})
await t.test('versioned icons and hashed build assets retain immutable caching', async () => {
for (const name of ['tdesign-icons/0.4.1/fonts/index.js', 'assets/cache-test-012345.js']) {
const response = await request(`/${name}`)
assert.equal(response.status, 200)
assert.equal(response.body, fixtures[name])
assert.equal(response.headers.get('cache-control'), 'public, max-age=31536000, immutable')
assert.equal(response.headers.get('x-content-type-options'), 'nosniff')
assert.ok(response.headers.get('etag'))
const conditional = await request(`/${name}`, { headers: { 'If-None-Match': response.headers.get('etag') } })
assert.equal(conditional.status, 304)
assert.equal(conditional.headers.get('cache-control'), 'public, max-age=31536000, immutable')
}
})
await t.test('missing versioned assets return an uncacheable 404, never the SPA', async () => {
for (const url of ['/tdesign-icons/0.4.1/fonts/missing.js', '/tdesign-icons/99.0.0/fonts/index.js', '/assets/missing.js']) {
const response = await request(url)
assert.equal(response.status, 404)
assert.equal(response.headers.get('cache-control'), 'no-store')
assert.ok(!response.body.includes('main cache fixture'))
}
})
await t.test('SPA and embed entry points still require revalidation', async () => {
for (const [url, fixture] of [['/login', 'index.html'], ['/embed/cache-test', 'embed.html']]) {
const response = await request(url)
assert.equal(response.status, 200)
assert.equal(response.body, fixtures[fixture])
assert.equal(response.headers.get('cache-control'), 'no-cache, must-revalidate')
}
})
await t.test('missing runtime configuration is an uncacheable 404', async () => {
docker('exec', container, 'rm', '/usr/share/nginx/html/config.js')
const response = await request('/config.js')
assert.equal(response.status, 404)
assert.equal(response.headers.get('cache-control'), 'no-store')
assert.ok(!response.body.includes('main cache fixture'))
})
} finally {
try {
if (created) docker('rm', '--force', container)
} finally {
fs.rmSync(temp, { recursive: true, force: true })
}
}
})
+146
View File
@@ -0,0 +1,146 @@
const assert = require('node:assert/strict')
const { execFileSync } = require('node:child_process')
const fs = require('node:fs')
const os = require('node:os')
const path = require('node:path')
const test = require('node:test')
const { setTimeout: delay } = require('node:timers/promises')
const frontend = path.resolve(__dirname, '../frontend')
const source = fs.readFileSync(path.join(frontend, 'nginx.conf'), 'utf8')
test('ingress access format exposes only approved fields and preserves nginx variables', () => {
const format = source.match(/log_format ingress_timing escape=json\s+([\s\S]*?);/)
assert.ok(format, 'escaped timing format exists')
const variables = [...new Set(format[1].match(/\$[a-z][a-z0-9_]*/g))].sort()
assert.deepEqual(variables, [
'$ingress_cf_ray', '$ingress_route_class', '$ingress_upstream_connect_time',
'$ingress_upstream_header_time', '$ingress_upstream_response_time', '$request_time', '$status',
'$time_iso8601',
].sort())
const active = source.split('\n').filter(line => !line.trim().startsWith('#')).join('\n')
assert.deepEqual([...active.matchAll(/access_log\s+([^;]+);/g)].map(match => match[1]), [
'/var/log/nginx/access.log ingress_timing',
])
const entrypoint = fs.readFileSync(path.join(frontend, 'docker-entrypoint.sh'), 'utf8')
const substituted = entrypoint.match(/envsubst '([^']+)'/)
assert.ok(substituted)
for (const variable of variables) assert.ok(!substituted[1].includes(variable))
})
const dockerContext = process.env.NGINX_TEST_DOCKER_CONTEXT
const docker = (...args) => execFileSync('docker', [
...(dockerContext ? ['--context', dockerContext] : []), ...args,
], { encoding: 'utf8', timeout: 30000, stdio: ['ignore', 'pipe', 'pipe'] }).trim()
// Enable only after the coordinated runtime freeze. No production network is used.
test('isolated nginx classifies requests and records timings without request secrets', {
skip: process.env.NGINX_TEST_RUN_INGRESS !== '1', timeout: 90000,
}, async () => {
const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'weknora-ingress-test-'))
const container = `weknora-ingress-test-${process.pid}-${Date.now()}`
const seedContainer = `${container}-seed`
const volume = `${container}-fixtures`
const image = process.env.NGINX_TEST_IMAGE || fs.readFileSync(path.join(frontend, 'Dockerfile'), 'utf8').match(/^FROM (\S+)/m)[1]
let created = false
let seedCreated = false
let volumeCreated = false
try {
const site = path.join(temp, 'html')
fs.mkdirSync(site)
for (const [filename, body] of Object.entries({
'index.html': '<html>timing fixture</html>',
'embed.html': '<html>embed fixture</html>',
'config.js': 'window.fixture = true;',
'release.json': '{"release":"fixture"}',
})) fs.writeFileSync(path.join(site, filename), body)
const replacements = { APP_SCHEME: 'http', APP_HOST: '127.0.0.1', APP_PORT: '81', MAX_FILE_SIZE: '50M', MAX_SKILL_BUNDLE_SIZE: '256M' }
const config = source.replace(/\$\{(APP_SCHEME|APP_HOST|APP_PORT|MAX_FILE_SIZE|MAX_SKILL_BUNDLE_SIZE)\}/g, (_, key) => replacements[key])
.replaceAll('/usr/share/nginx/html', '/fixtures/html')
.replaceAll('/etc/nginx/api-proxy.conf', '/fixtures/api-proxy.conf')
.replaceAll('/var/log/nginx/access.log ingress_timing', '/proc/1/fd/1 ingress_timing')
.replaceAll('/var/log/nginx/error.log warn', '/proc/1/fd/2 warn')
fs.writeFileSync(path.join(temp, 'nginx.conf'),
'worker_processes 1; error_log /dev/stderr warn; pid /var/run/nginx.pid;\n'
+ 'events { worker_connections 128; }\nhttp { include /etc/nginx/mime.types;\n'
+ config + '\nserver { listen 127.0.0.1:81; access_log off; location / { return 204; } }\n}\n')
fs.copyFileSync(path.join(frontend, 'nginx-api-proxy.conf'), path.join(temp, 'api-proxy.conf'))
// Stage through Docker copy so a remote context needs no host-path sharing.
docker('volume', 'create', volume)
volumeCreated = true
docker('create', '--pull', 'never', '--read-only', '--network', 'none', '--name', seedContainer,
'--mount', `type=volume,src=${volume},dst=/fixtures`, '--entrypoint', 'true', image)
seedCreated = true
docker('cp', `${temp}/.`, `${seedContainer}:/fixtures`)
docker('rm', seedContainer)
seedCreated = false
docker('create', '--rm', '--pull', 'never', '--read-only', '--network', 'none', '--name', container,
'--tmpfs', '/var/cache/nginx:rw,noexec,nosuid,size=16m',
'--tmpfs', '/var/run:rw,noexec,nosuid,size=1m',
'--mount', `type=volume,src=${volume},dst=/fixtures,readonly`,
'--entrypoint', 'nginx', image, '-c', '/fixtures/nginx.conf', '-g', 'daemon off;')
created = true
docker('start', container)
docker('exec', container, 'nginx', '-t', '-c', '/fixtures/nginx.conf')
const fetchFixture = (url, extra = []) => docker('exec', container, 'wget', '-q', '-O', '/dev/null', ...extra, `http://127.0.0.1${url}`)
for (let attempt = 0; ; attempt += 1) {
try { fetchFixture('/release.json'); break } catch (error) {
if (attempt >= 19) throw error
await delay(250)
}
}
const secret = 'ingress-secret-sentinel'
const cases = [
['/admin', 'shell', 200], ['/platform/workbench/content', 'shell', 200],
['/api/v1/auth/me', 'auth_me', 204], ['/api/v1/access/me', 'access_me', 204],
['/api/v1/system/capabilities', 'capabilities', 204],
['/api/v1/workbench/resources/training-maps', 'resource_list', 204],
[`/api/v1/workbench/resources/training-maps/${secret}`, 'resource_detail', 204],
[`/api/v1/workbench/resources/training-maps/${secret}/jobs`, 'api', 204],
[`/r/${secret}`, 'resource_link', 204], [`/files/${secret}`, 'file', 200],
[`/embed/${secret}`, 'embed', 200], ['/config.js', 'config', 200],
['/release.json', 'release', 200], [`/unknown/${secret}`, 'other', 200],
]
const rays = cases.map((_, index) => `${(index + 1).toString(16).padStart(16, '0')}-AMS`)
for (const [index, [url]] of cases.entries()) fetchFixture(`${url}?token=${secret}`, [
`--header=CF-Ray: ${rays[index]}`, `--header=Authorization: Bearer ${secret}`,
`--header=Cookie: session=${secret}`, `--header=Referer: https://example.test/${secret}`,
`--header=User-Agent: ${secret}`,
])
fetchFixture('/api/body', [`--header=CF-Ray: ${secret}`, `--post-data=${secret}`])
const lines = docker('logs', container).split('\n').filter(line => line.startsWith('{"time":'))
assert.ok(lines.length >= cases.length + 2)
assert.ok(!lines.join('\n').includes(secret), 'no path, query, header, or body secret appears in access records')
const records = lines.map(line => JSON.parse(line))
const fields = ['time', 'route', 'status', 'request_time', 'upstream_connect_time', 'upstream_header_time', 'upstream_response_time', 'cf_ray'].sort()
for (const [index, [, route, status]] of cases.entries()) {
const matches = records.filter(record => record.cf_ray === rays[index])
assert.equal(matches.length, 1)
const record = matches[0]
assert.deepEqual(Object.keys(record).sort(), fields)
assert.equal(record.route, route)
assert.equal(record.status, status, `${route} ${record.cf_ray}`)
assert.match(record.request_time, /^\d+\.\d{3}$/)
for (const field of ['upstream_connect_time', 'upstream_header_time', 'upstream_response_time']) {
assert.match(record[field], status === 204 ? /^\d+\.\d{3}$/ : /^-$/)
}
}
assert.ok(records.some(record => record.route === 'api' && record.cf_ray === '-' && record.status === 204))
} finally {
try {
if (created) docker('rm', '--force', container)
} finally {
try {
if (seedCreated) docker('rm', '--force', seedContainer)
} finally {
try {
if (volumeCreated) docker('volume', 'rm', volume)
} finally {
fs.rmSync(temp, { recursive: true, force: true })
}
}
}
}
})
+62
View File
@@ -0,0 +1,62 @@
const assert = require('node:assert/strict')
const { execFileSync } = require('node:child_process')
const fs = require('node:fs')
const os = require('node:os')
const path = require('node:path')
const vm = require('node:vm')
const test = require('node:test')
const frontend = path.resolve(__dirname, '../frontend')
const docker = (...args) => execFileSync('docker', [
...(process.env.NGINX_TEST_DOCKER_CONTEXT ? ['--context', process.env.NGINX_TEST_DOCKER_CONTEXT] : []), ...args,
], { encoding: 'utf8', timeout: 30000, stdio: ['ignore', 'pipe', 'pipe'] }).trim()
test('startup embeds current validated config in both entry points on every run', { timeout: 90000 }, async () => {
const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'weknora-runtime-config-'))
const name = `weknora-runtime-config-${process.pid}-${Date.now()}`
const image = process.env.NGINX_TEST_IMAGE || fs.readFileSync(path.join(frontend, 'Dockerfile'), 'utf8').match(/^FROM (\S+)/m)[1]
let created = false
try {
// The stand-in nginx exits, so the real startup script can be run repeatedly.
fs.writeFileSync(path.join(temp, 'nginx'), '#!/bin/sh\nexit 0\n', { mode: 0o755 })
docker('create', '--rm', '--pull', 'never', '--network', 'none', '--name', name,
'--entrypoint', 'sh', image, '-c', 'while :; do sleep 30; done')
created = true
docker('start', name)
docker('exec', name, 'mkdir', '-p', '/usr/share/nginx/runtime-templates', '/etc/nginx/templates', '/fixture-bin')
for (const page of ['index.html', 'embed.html']) docker('cp', path.join(frontend, page), `${name}:/usr/share/nginx/runtime-templates/${page}`)
docker('cp', path.join(frontend, 'nginx.conf'), `${name}:/etc/nginx/templates/default.conf.template`)
docker('cp', path.join(frontend, 'docker-entrypoint.sh'), `${name}:/fixture-entrypoint.sh`)
docker('cp', path.join(temp, 'nginx'), `${name}:/fixture-bin/nginx`)
const run = env => docker('exec', '-e', 'PATH=/fixture-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin',
...Object.entries(env).flatMap(([key, value]) => ['-e', `${key}=${value}`]), name, 'sh', '/fixture-entrypoint.sh')
const readConfig = (source) => {
const matches = [...source.matchAll(/<script data-runtime-config>([\s\S]*?)<\/script>/g)]
assert.equal(matches.length, 1)
assert.ok(!source.includes('src="/config.js"'))
const context = { window: {} }
vm.runInNewContext(matches[0][1], context, { timeout: 500 })
return JSON.parse(JSON.stringify(context.window.__RUNTIME_CONFIG__))
}
for (const [env, expected] of [
[{ DEFAULT_LOCALE: 'zh-CN', MAX_FILE_SIZE_MB: '50', MAX_SKILL_BUNDLE_SIZE_MB: '300' },
{ DEFAULT_LOCALE: 'zh-CN', MAX_FILE_SIZE_MB: 50, MAX_SKILL_BUNDLE_SIZE_MB: 300 }],
[{ DEFAULT_LOCALE: 'en-US', MAX_FILE_SIZE_MB: '99999', MAX_SKILL_BUNDLE_SIZE_MB: '600' },
{ DEFAULT_LOCALE: 'en-US', MAX_FILE_SIZE_MB: 20480, MAX_SKILL_BUNDLE_SIZE_MB: 512 }],
[{ DEFAULT_LOCALE: '</script><script>alert(1)</script>', MAX_FILE_SIZE_MB: 'invalid', MAX_SKILL_BUNDLE_SIZE_MB: '1;alert(1)' },
{ DEFAULT_LOCALE: '', MAX_FILE_SIZE_MB: 50, MAX_SKILL_BUNDLE_SIZE_MB: 256 }],
]) {
run(env)
for (const page of ['index.html', 'embed.html']) {
const body = docker('exec', name, 'cat', `/usr/share/nginx/html/${page}`)
assert.deepEqual(readConfig(body), expected)
assert.ok(body.includes('type="module"'))
}
const compatibility = { window: {} }
vm.runInNewContext(docker('exec', name, 'cat', '/usr/share/nginx/html/config.js'), compatibility, { timeout: 500 })
assert.deepEqual(JSON.parse(JSON.stringify(compatibility.window.__RUNTIME_CONFIG__)), expected)
}
} finally {
try { if (created) docker('rm', '--force', name) } finally { fs.rmSync(temp, { recursive: true, force: true }) }
}
})
+90
View File
@@ -0,0 +1,90 @@
#!/bin/bash
# 统一的版本信息获取脚本
# 支持本地构建和CI构建环境
# 设置默认值
VERSION="unknown"
EDITION="${EDITION:-standard}"
COMMIT_ID="unknown"
BUILD_TIME="unknown"
GO_VERSION="unknown"
# 获取版本号
if [ -f "VERSION" ]; then
VERSION=$(cat VERSION | tr -d '\n\r')
fi
# 获取commit ID
if [ -n "$GITHUB_SHA" ]; then
# GitHub Actions环境
COMMIT_ID="${GITHUB_SHA:0:7}"
elif command -v git >/dev/null 2>&1; then
# 本地环境
COMMIT_ID=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
fi
# 获取构建时间
if [ -n "$GITHUB_ACTIONS" ]; then
# GitHub Actions环境,使用标准时间格式
BUILD_TIME=$(date -u '+%Y-%m-%d %H:%M:%S UTC')
else
# 本地环境
BUILD_TIME=$(date -u '+%Y-%m-%d %H:%M:%S UTC')
fi
# 获取Go版本
if command -v go >/dev/null 2>&1; then
GO_VERSION=$(go version 2>/dev/null || echo "unknown")
fi
# 根据参数输出不同格式
case "${1:-env}" in
"env")
# 输出环境变量格式,对包含空格的值进行转义
echo "VERSION=$VERSION"
echo "EDITION=$EDITION"
echo "COMMIT_ID=$COMMIT_ID"
echo "BUILD_TIME=\"$BUILD_TIME\""
echo "GO_VERSION=\"$GO_VERSION\""
;;
"json")
# 输出JSON格式
cat << EOF
{
"version": "$VERSION",
"edition": "$EDITION",
"commit_id": "$COMMIT_ID",
"build_time": "$BUILD_TIME",
"go_version": "$GO_VERSION"
}
EOF
;;
"docker-args")
# 输出Docker构建参数格式
echo "--build-arg VERSION_ARG=$VERSION"
echo "--build-arg COMMIT_ID_ARG=$COMMIT_ID"
echo "--build-arg BUILD_TIME_ARG=$BUILD_TIME"
echo "--build-arg GO_VERSION_ARG=$GO_VERSION"
;;
"ldflags")
# 输出Go ldflags格式
echo "-X 'github.com/Tencent/WeKnora/internal/handler.Version=$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.Edition=$EDITION' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$GO_VERSION'"
;;
"info")
# 输出信息格式
echo "版本信息: $VERSION"
echo "版本类型: $EDITION"
echo "Commit ID: $COMMIT_ID"
echo "构建时间: $BUILD_TIME"
echo "Go版本: $GO_VERSION"
;;
*)
echo "用法: $0 [env|json|docker-args|ldflags|info]"
echo " env - 输出环境变量格式 (默认)"
echo " json - 输出JSON格式"
echo " docker-args - 输出Docker构建参数格式"
echo " ldflags - 输出Go ldflags格式"
echo " info - 输出信息格式"
exit 1
;;
esac
+234
View File
@@ -0,0 +1,234 @@
#!/usr/bin/env bash
# Shared helpers for WeKnora git hooks (mirrors .github/workflows and PR checklist).
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
hook_skip() {
[[ "${SKIP_HOOKS:-}" == "1" ]]
}
log_step() {
printf '→ %s\n' "$*"
}
log_ok() {
printf '✓ %s\n' "$*"
}
log_fail() {
printf '✗ %s\n' "$*" >&2
}
# File types we expect hand-edited in PRs (skip sample-data / generated blobs).
WHITESPACE_PATHSPECS=(
'*.go'
'*.ts' '*.tsx' '*.vue' '*.js' '*.mjs'
'*.yaml' '*.yml' '*.json'
'*.sh' '*.sql'
'Makefile'
'frontend/package.json'
'frontend/package-lock.json'
)
# Best-effort fetch so merge-base matches CI (origin/main).
ensure_origin_main() {
git -C "$ROOT" fetch origin main --quiet 2>/dev/null || true
}
merge_base() {
ensure_origin_main
local base
base="$(git -C "$ROOT" merge-base HEAD origin/main 2>/dev/null || true)"
if [[ -n "$base" ]]; then
printf '%s\n' "$base"
return
fi
base="$(git -C "$ROOT" merge-base HEAD main 2>/dev/null || true)"
if [[ -n "$base" ]]; then
printf '%s\n' "$base"
return
fi
git -C "$ROOT" rev-parse HEAD~1
}
# Prefer the open PR's base SHA (matches GitHub CI diff) when gh is available.
pr_diff_base_ref() {
ensure_origin_main
if command -v gh >/dev/null 2>&1; then
local base_sha
base_sha="$(gh pr view --json baseRefOid --jq .baseRefOid 2>/dev/null || true)"
if [[ -n "$base_sha" && "$base_sha" != "null" ]]; then
printf '%s\n' "$base_sha"
return
fi
fi
if git -C "$ROOT" rev-parse --verify origin/main >/dev/null 2>&1; then
printf '%s\n' "origin/main"
return
fi
printf '%s\n' "main"
}
check_whitespace() {
log_step "git diff --check (source files)"
if [[ "${1:-}" == "--cached" ]]; then
if git -C "$ROOT" diff --cached --quiet -- "${WHITESPACE_PATHSPECS[@]}"; then
log_ok "no staged source changes to check"
return 0
fi
git -C "$ROOT" diff --check --cached -- "${WHITESPACE_PATHSPECS[@]}"
else
if git -C "$ROOT" diff --quiet "$@" -- "${WHITESPACE_PATHSPECS[@]}"; then
log_ok "no changed source files to check"
return 0
fi
git -C "$ROOT" diff --check "$@" -- "${WHITESPACE_PATHSPECS[@]}"
fi
log_ok "no whitespace errors"
}
check_gofmt_files() {
local mode="$1" # check | write
shift
local -a files=("$@")
if [[ ${#files[@]} -gt 0 ]]; then
local -a existing=()
local f rel
for f in "${files[@]}"; do
if [[ -f "$f" ]]; then
existing+=("$f")
continue
fi
rel="${f#"$ROOT"/}"
if [[ -f "$ROOT/$rel" ]]; then
existing+=("$ROOT/$rel")
fi
done
files=("${existing[@]}")
fi
[[ ${#files[@]} -gt 0 ]] || return 0
local unformatted
if [[ "$mode" == "write" ]]; then
log_step "gofmt (auto-format staged Go files)"
gofmt -w "${files[@]}"
# Re-stage formatted files.
git -C "$ROOT" add -- "${files[@]}"
log_ok "gofmt applied"
return 0
fi
log_step "gofmt (check)"
unformatted="$(gofmt -l "${files[@]}")"
if [[ -n "$unformatted" ]]; then
log_fail "the following Go files are not gofmt-formatted:"
printf '%s\n' "$unformatted" >&2
log_fail "run: gofmt -w <files> (or commit again after pre-commit auto-format)"
return 1
fi
log_ok "gofmt"
}
collect_go_packages_from_files() {
local -a files=("$@")
local f dir pkg module root_module
local -a pkgs=()
# Nested modules (docs/poc, cli, client) are not testable from the root
# module, so only keep packages that belong to it.
root_module="$(cd "$ROOT" && go list -m 2>/dev/null || true)"
for f in "${files[@]}"; do
[[ "$f" == *.go ]] || continue
dir="$(dirname "$f")"
module="$(cd "$ROOT/$dir" && go list -m 2>/dev/null)" || continue
[[ "$module" == "$root_module" ]] || continue
pkg="$(cd "$ROOT/$dir" && go list . 2>/dev/null)" || continue
pkgs+=("$pkg")
done
if [[ ${#pkgs[@]} -eq 0 ]]; then
return 0
fi
printf '%s\n' "${pkgs[@]}" | sort -u
}
run_golangci_if_available() {
local from_rev="$1"
if ! command -v golangci-lint >/dev/null 2>&1; then
log_step "golangci-lint not installed; skip (install: https://golangci-lint.run)"
return 0
fi
log_step "golangci-lint run --new-from-rev=$from_rev ./..."
(
cd "$ROOT"
golangci-lint run --new-from-rev="$from_rev" ./...
)
log_ok "golangci-lint"
}
run_full_app_vet() {
# Mirrors .github/workflows/app.yml: vet every app package (excl. docreader).
log_step "go vet (all app packages, like CI)"
(
cd "$ROOT"
local -a pkgs=()
while IFS= read -r line; do
[[ -n "$line" ]] && pkgs+=("$line")
done < <(go list ./... | grep -v '/docreader/' || true)
go vet "${pkgs[@]}"
)
log_ok "go vet"
}
run_go_test_packages() {
local -a pkgs=("$@")
if [[ ${#pkgs[@]} -eq 0 ]]; then
log_step "no changed Go packages; skip go test"
return 0
fi
if [[ "${HOOK_SKIP_TEST:-}" == "1" ]]; then
log_step "HOOK_SKIP_TEST=1; skip go test"
return 0
fi
log_step "go test (${#pkgs[@]} changed package(s))"
(
cd "$ROOT"
go test -count=1 "${pkgs[@]}"
)
log_ok "go test"
}
run_app_vet_test() {
run_full_app_vet
run_go_test_packages "$@"
}
run_cli_checks() {
log_step "cli: go vet && go test"
(
cd "$ROOT/cli"
go vet ./...
if [[ "${HOOK_SKIP_TEST:-}" == "1" ]]; then
log_step "HOOK_SKIP_TEST=1; skip cli go test"
else
go test -count=1 ./...
fi
)
log_ok "cli checks"
}
run_frontend_checks() {
log_step "frontend: verify (test, type-check, build)"
"$ROOT/scripts/verify_frontend_pr.sh"
log_ok "frontend checks"
}
paths_match_prefix() {
local prefix="$1"
shift
local p
for p in "$@"; do
[[ "$p" == "$prefix"* ]] && return 0
done
return 1
}
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# Fast checks on staged files (PR checklist + gofmt). Mirrors app.yml formatting gate.
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
# shellcheck source=scripts/git-hooks/common.sh
source "$ROOT/scripts/git-hooks/common.sh"
if hook_skip; then
exit 0
fi
cd "$ROOT"
log_step "pre-commit"
check_whitespace --cached
staged_go=()
while IFS= read -r line; do
[[ -n "$line" ]] && staged_go+=("$line")
done < <(git diff --cached --name-only --diff-filter=ACMR -- '*.go' ':!cli/**' || true)
if [[ ${#staged_go[@]} -gt 0 ]]; then
go_files=()
for f in "${staged_go[@]}"; do
go_files+=("$ROOT/$f")
done
check_gofmt_files write "${go_files[@]}"
if command -v golangci-lint >/dev/null 2>&1; then
# Lint only what differs from HEAD (staged + unstaged in those packages).
run_golangci_if_available HEAD
fi
fi
log_ok "pre-commit passed"
+98
View File
@@ -0,0 +1,98 @@
#!/usr/bin/env bash
# Pre-push checks aligned with CI (.github/workflows/app.yml, frontend.yml, cli.yml).
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
# shellcheck source=scripts/git-hooks/common.sh
source "$ROOT/scripts/git-hooks/common.sh"
if hook_skip; then
exit 0
fi
cd "$ROOT"
remote="${1:-origin}"
while read -r _local_ref local_sha remote_ref remote_sha; do
if [[ "$local_sha" =~ ^0+$ ]]; then
continue
fi
log_step "pre-push ($remote ${remote_ref#refs/heads/})"
ensure_origin_main
pr_ref="$(pr_diff_base_ref)"
pr_tip="$local_sha"
pr_diff="${pr_ref}...${pr_tip}"
lint_from_rev="$pr_ref"
if [[ "$remote_sha" =~ ^0+$ ]]; then
push_start="$(git -C "$ROOT" merge-base "$pr_ref" "$local_sha")"
push_range="${push_start}..${local_sha}"
log_step "new branch; PR diff ${pr_ref}...${local_sha:0:12}"
else
if [[ "$remote_sha" == "$local_sha" ]]; then
log_ok "pre-push: nothing new to push"
continue
fi
push_range="${remote_sha}..${local_sha}"
log_step "pushing ${remote_sha:0:12}..${local_sha:0:12}; PR diff ${pr_ref}...${local_sha:0:12}"
fi
check_whitespace "$push_range"
pr_changed=()
while IFS= read -r line; do
[[ -n "$line" ]] && pr_changed+=("$line")
done < <(git diff --name-only --diff-filter=ACMR "$pr_diff" || true)
if [[ ${#pr_changed[@]} -eq 0 ]]; then
log_ok "pre-push: no PR diff changes"
continue
fi
pr_changed_go=()
while IFS= read -r line; do
[[ -n "$line" ]] && pr_changed_go+=("$line")
done < <(git diff --name-only --diff-filter=ACMR "$pr_diff" -- '*.go' ':!cli/**' || true)
if [[ ${#pr_changed_go[@]} -gt 0 ]]; then
go_files=()
for f in "${pr_changed_go[@]}"; do
go_files+=("$ROOT/$f")
done
check_gofmt_files check "${go_files[@]}"
run_golangci_if_available "$lint_from_rev"
pkgs=()
while IFS= read -r line; do
[[ -n "$line" ]] && pkgs+=("$line")
done < <(collect_go_packages_from_files "${pr_changed_go[@]}")
filtered=()
for pkg in "${pkgs[@]}"; do
[[ "$pkg" == *"/docreader/"* ]] && continue
filtered+=("$pkg")
done
run_full_app_vet
run_go_test_packages "${filtered[@]}"
log_step "go build ./cmd/server"
go build ./cmd/server
log_ok "go build ./cmd/server"
fi
if paths_match_prefix "cli/" "${pr_changed[@]}"; then
run_cli_checks
fi
if paths_match_prefix "frontend/" "${pr_changed[@]}"; then
run_frontend_checks
fi
log_ok "pre-push passed"
done
exit 0
+73
View File
@@ -0,0 +1,73 @@
package main
import (
"context"
"fmt"
"strconv"
"sync"
"github.com/Tencent/WeKnora/internal/models/requestbudget"
)
func probeSendLimit(requested *int, environment string, envSet bool) (int, error) {
limit := requestbudget.GraphMaxSends
if requested != nil {
limit = *requested
}
if limit < 1 || limit > requestbudget.GraphMaxSends {
return 0, fmt.Errorf("invalid probe HTTP send cap")
}
if envSet && environment != strconv.Itoa(limit) {
return 0, fmt.Errorf("probe HTTP send cap differs between request and environment")
}
return limit, nil
}
// This tighter probe allowance delegates model, phase and deadline enforcement
// to the existing graph guard; worker construction and defaults are untouched.
type probeExecutionBudget struct {
*requestbudget.GraphExecution
mu sync.Mutex
limit int
stopped error
cancel context.CancelCauseFunc
}
func bindProbeExecution(ctx context.Context, key, model string, limit int) (context.Context, *probeExecutionBudget, context.CancelFunc, error) {
if _, err := probeSendLimit(&limit, "", false); err != nil {
return ctx, nil, func() {}, err
}
ctx, base, baseCancel, err := requestbudget.BindGraphExecution(ctx, key, model)
if err != nil {
return ctx, nil, baseCancel, err
}
ctx, cancel := context.WithCancelCause(ctx)
guard := &probeExecutionBudget{GraphExecution: base, limit: limit, cancel: cancel}
return requestbudget.WithGuard(ctx, guard), guard, func() { cancel(nil); baseCancel() }, nil
}
func (g *probeExecutionBudget) Check(ctx context.Context) error {
g.mu.Lock()
defer g.mu.Unlock()
if g.stopped != nil {
return g.stopped
}
return g.GraphExecution.Check(ctx)
}
func (g *probeExecutionBudget) Reserve(ctx context.Context, model, phase string) error {
g.mu.Lock()
defer g.mu.Unlock()
if g.stopped != nil {
return g.stopped
}
if err := g.GraphExecution.Check(ctx); err != nil {
return err
}
if g.Reservations() >= g.limit {
g.stopped = fmt.Errorf("%w: probe HTTP send cap reached", requestbudget.ErrStopped)
g.cancel(g.stopped)
return g.stopped
}
return g.GraphExecution.Reserve(ctx, model, phase)
}
@@ -0,0 +1,196 @@
package main
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/models/requestbudget"
)
func TestProbeSendLimit(t *testing.T) {
one, two, three, zero, four := 1, 2, 3, 0, 4
for _, tc := range []struct {
name string
requested *int
environment string
envSet bool
want int
}{
{name: "default", want: 3},
{name: "minimum", requested: &one, want: 1},
{name: "two", requested: &two, environment: "2", envSet: true, want: 2},
{name: "explicit default", requested: &three, environment: "3", envSet: true, want: 3},
{name: "default environment", environment: "3", envSet: true, want: 3},
{name: "zero", requested: &zero},
{name: "excess", requested: &four},
{name: "environment mismatch", requested: &two, environment: "3", envSet: true},
{name: "environment without request", environment: "2", envSet: true},
{name: "empty environment", requested: &two, envSet: true},
{name: "padded environment", requested: &two, environment: "02", envSet: true},
{name: "decimal environment", requested: &two, environment: "2.0", envSet: true},
{name: "whitespace environment", requested: &two, environment: " 2", envSet: true},
} {
t.Run(tc.name, func(t *testing.T) {
got, err := probeSendLimit(tc.requested, tc.environment, tc.envSet)
if got != tc.want || (err != nil) != (tc.want == 0) {
t.Fatalf("limit = %d, error = %v; want %d", got, err, tc.want)
}
})
}
}
type probeRoundTripper func(*http.Request) (*http.Response, error)
func (f probeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
func TestProbeTransportHonorsActualSendCap(t *testing.T) {
for _, limit := range []int{1, 2, requestbudget.GraphMaxSends} {
t.Run(fmt.Sprint(limit), func(t *testing.T) {
ctx, guard, cancel, err := bindProbeExecution(context.Background(), "offline", "model", limit)
if err != nil {
t.Fatal(err)
}
defer cancel()
ctx = requestbudget.WithModel(ctx, "model")
sends := 0
transport := requestbudget.Transport{Base: probeRoundTripper(func(req *http.Request) (*http.Response, error) {
sends++
if req.Context().Err() != nil {
t.Fatal("allowed response was canceled")
}
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("usable response"))}, nil
})}
for i := 0; i < limit; i++ {
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, "https://offline.invalid", nil)
response, err := transport.RoundTrip(req)
if err != nil {
t.Fatalf("send %d: %v", i+1, err)
}
body, readErr := io.ReadAll(response.Body)
response.Body.Close()
if readErr != nil || string(body) != "usable response" || requestbudget.Check(ctx) != nil {
t.Fatal("last allowed response must remain usable")
}
}
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, "https://offline.invalid", nil)
if _, err := transport.RoundTrip(req); !errors.Is(err, requestbudget.ErrStopped) {
t.Fatalf("excess send error = %v", err)
}
if sends != limit || guard.Reservations() != limit || !errors.Is(context.Cause(ctx), requestbudget.ErrStopped) {
t.Fatalf("sends = %d, reservations = %d, cause = %v", sends, guard.Reservations(), context.Cause(ctx))
}
if err := guard.Reserve(context.Background(), "model", requestbudget.GraphPhase); !errors.Is(err, requestbudget.ErrStopped) {
t.Fatal("stopped execution was revived")
}
})
}
}
func TestProbeBudgetPreservesScopeAndDeadline(t *testing.T) {
for _, tc := range []struct{ name, model, phase string }{
{"model", "other", requestbudget.GraphPhase},
{"phase", "model", "other"},
} {
t.Run(tc.name, func(t *testing.T) {
ctx, guard, cancel, err := bindProbeExecution(context.Background(), "offline", "model", 2)
if err != nil {
t.Fatal(err)
}
defer cancel()
if err := guard.Reserve(ctx, tc.model, tc.phase); !errors.Is(err, requestbudget.ErrStopped) {
t.Fatalf("scope mismatch error = %v", err)
}
if guard.Reservations() != 0 || !errors.Is(context.Cause(ctx), requestbudget.ErrStopped) {
t.Fatal("scope mismatch consumed a send or did not stop execution")
}
})
}
t.Run("inherited deadline", func(t *testing.T) {
deadline := time.Now().Add(time.Minute)
parent, parentCancel := context.WithDeadline(context.Background(), deadline)
defer parentCancel()
ctx, guard, cancel, err := bindProbeExecution(parent, "offline", "model", 2)
if err != nil {
t.Fatal(err)
}
defer cancel()
if got, ok := ctx.Deadline(); !ok || !got.Equal(deadline) {
t.Fatal("inherited deadline was reset")
}
if err := guard.Reserve(ctx, "model", requestbudget.GraphPhase); err != nil {
t.Fatal(err)
}
if got, _ := ctx.Deadline(); !got.Equal(deadline) {
t.Fatal("reservation reset the deadline")
}
parentCancel()
if err := guard.Reserve(ctx, "model", requestbudget.GraphPhase); !errors.Is(err, context.Canceled) {
t.Fatalf("cancellation error = %v", err)
}
if guard.Reservations() != 1 {
t.Fatal("canceled send consumed a reservation")
}
})
t.Run("default deadline", func(t *testing.T) {
started := time.Now()
ctx, _, cancel, err := bindProbeExecution(context.Background(), "offline", "model", 2)
if err != nil {
t.Fatal(err)
}
defer cancel()
deadline, ok := ctx.Deadline()
if !ok || deadline.Before(started.Add(requestbudget.GraphTimeout)) || deadline.After(time.Now().Add(requestbudget.GraphTimeout)) {
t.Fatal("probe did not retain the shared absolute timeout")
}
})
t.Run("already canceled", func(t *testing.T) {
parent, cancel := context.WithCancel(context.Background())
cancel()
if _, _, cleanup, err := bindProbeExecution(parent, "offline", "model", 2); err == nil {
cleanup()
t.Fatal("canceled execution was rebound")
}
})
t.Run("nested execution", func(t *testing.T) {
ctx, _, cancel, err := bindProbeExecution(context.Background(), "offline", "model", 2)
if err != nil {
t.Fatal(err)
}
defer cancel()
if _, _, cleanup, err := bindProbeExecution(ctx, "retry", "model", 2); err == nil {
cleanup()
t.Fatal("nested execution reset the allowance")
}
})
}
func TestProbeBudgetConcurrentReservationsStayBounded(t *testing.T) {
ctx, guard, cancel, err := bindProbeExecution(context.Background(), "offline", "model", 2)
if err != nil {
t.Fatal(err)
}
defer cancel()
var successful atomic.Int32
var group sync.WaitGroup
for i := 0; i < 16; i++ {
group.Add(1)
go func() {
defer group.Done()
if guard.Reserve(ctx, "model", requestbudget.GraphPhase) == nil {
successful.Add(1)
}
}()
}
group.Wait()
if successful.Load() != 2 || guard.Reservations() != 2 {
t.Fatalf("successful = %d; reservations = %d", successful.Load(), guard.Reservations())
}
}
+170
View File
@@ -0,0 +1,170 @@
// graph_extraction_probe calls the production extractor without writing graph data.
// The model configuration and AES key arrive on stdin and are never reported.
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"time"
"unicode/utf8"
chatpipeline "github.com/Tencent/WeKnora/internal/application/service/chat_pipeline"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/utils"
"gopkg.in/yaml.v3"
)
type request struct {
ExecutionID string `json:"execution_id"`
MaxHTTPSends *int `json:"max_http_sends"`
AESKey string `json:"aes_key"`
Model types.Model `json:"model"`
Chunk types.Chunk `json:"chunk"`
ExtractConfig types.ExtractConfig `json:"extract_config"`
}
type chatDelegate interface{ chat.Chat }
type probeCall struct {
Number int `json:"number"`
ElapsedMS int64 `json:"elapsed_ms"`
MaxTokens int `json:"max_tokens"`
Thinking *bool `json:"thinking_requested"`
ResponseReceived bool `json:"response_received"`
Failed bool `json:"failed"`
FinishReason string `json:"finish_reason"`
ContentRunes int `json:"content_runes"`
ReasoningRunes int `json:"reasoning_runes"`
Usage types.TokenUsage `json:"usage"`
}
type recordingChat struct {
chatDelegate
response *types.ChatResponse
calls int
records []probeCall
}
func (c *recordingChat) Chat(ctx context.Context, messages []chat.Message, options *chat.ChatOptions) (*types.ChatResponse, error) {
started := time.Now()
response, err := c.chatDelegate.Chat(ctx, messages, options)
c.response = response
c.calls++
record := probeCall{
Number: c.calls, ElapsedMS: time.Since(started).Milliseconds(),
ResponseReceived: response != nil, Failed: err != nil,
}
if options != nil {
record.MaxTokens = options.MaxTokens
if options.Thinking != nil {
thinking := *options.Thinking
record.Thinking = &thinking
}
}
if response != nil {
record.FinishReason = response.FinishReason
record.ContentRunes = utf8.RuneCountInString(response.Content)
record.ReasoningRunes = utf8.RuneCountInString(response.ReasoningContent)
record.Usage = response.Usage
}
c.records = append(c.records, record)
return response, err
}
func run() error {
var input request
if err := json.NewDecoder(os.Stdin).Decode(&input); err != nil {
return fmt.Errorf("invalid probe input")
}
capValue, capSet := os.LookupEnv("GRAPH_PROBE_MAX_SENDS")
maxSends, err := probeSendLimit(input.MaxHTTPSends, capValue, capSet)
if err != nil {
return err
}
if input.ExecutionID == "" || input.Chunk.ID == "" || input.Chunk.Content == "" || input.Model.ID == "" || input.Model.Name == "" {
return fmt.Errorf("probe requires a source chunk and model")
}
if err := os.Setenv("SYSTEM_AES_KEY", input.AESKey); err != nil {
return err
}
input.AESKey = ""
key, err := utils.DecryptStoredSecret(input.Model.Parameters.APIKey)
if err != nil {
return fmt.Errorf("cannot decrypt configured model credential")
}
input.Model.Parameters.APIKey = key
defer os.Unsetenv("SYSTEM_AES_KEY")
chatConfig := chat.ConfigFromModel(&input.Model, "", "")
model, err := chat.NewChat(chatConfig, nil)
if err != nil {
return fmt.Errorf("cannot initialize configured remote model")
}
base, err := os.ReadFile("config/config.yaml")
if err != nil {
return err
}
var config struct {
Extract struct {
Graph struct {
Description string `yaml:"description"`
} `yaml:"extract_graph"`
} `yaml:"extract"`
}
if err := yaml.Unmarshal(base, &config); err != nil {
return err
}
recorder := &recordingChat{chatDelegate: model}
extractor := chatpipeline.NewGraphExtractor(recorder, &types.PromptTemplateStructured{
Description: types.AppendCustomPromptInstructions(config.Extract.Graph.Description, input.ExtractConfig.CustomInstructions, "graph_extraction"),
Tags: input.ExtractConfig.Tags,
})
ctx, guard, cancel, err := bindProbeExecution(context.Background(), input.ExecutionID, input.Model.ID, maxSends)
if err != nil {
return err
}
defer cancel()
started := time.Now()
graph, extractionErr := extractor.Extract(ctx, input.Chunk.Content)
hash := sha256.Sum256([]byte(input.Chunk.Content))
report := map[string]any{
"checked_at": started.UTC().Format(time.RFC3339), "elapsed_ms": time.Since(started).Milliseconds(),
"chunk_id": input.Chunk.ID, "source_sha256": hex.EncodeToString(hash[:]),
"model_name": input.Model.Name, "allowed_relation_types": input.ExtractConfig.Tags,
"passed": extractionErr == nil, "graph": graph, "production_write": false,
"model_calls": recorder.calls,
"execution_id": input.ExecutionID,
"http_send_reservations": guard.Reservations(),
"http_send_limit": maxSends,
"call_diagnostics": recorder.records,
"effective_thinking_control": chat.EffectiveThinkingControl(chatConfig),
"diagnostics_version": 2,
}
if recorder.response != nil {
report["finish_reason"] = recorder.response.FinishReason
report["usage"] = recorder.response.Usage
if extractionErr != nil {
report["rejected_model_content"] = recorder.response.Content
}
}
if extractionErr != nil {
report["error"] = extractionErr.Error()
}
encoded, err := json.Marshal(report)
if err != nil {
return err
}
fmt.Printf("GRAPH_EXTRACTION_PROBE_RESULT=%s\n", encoded)
return nil
}
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
@@ -0,0 +1,77 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
)
type fakeProbeChat struct {
chatDelegate
response *types.ChatResponse
err error
}
func (f *fakeProbeChat) Chat(context.Context, []chat.Message, *chat.ChatOptions) (*types.ChatResponse, error) {
return f.response, f.err
}
func TestRecordingChatPreservesPerCallDiagnosticsWithoutContent(t *testing.T) {
fake := &fakeProbeChat{response: &types.ChatResponse{
Content: "answer", ReasoningContent: "private reasoning", FinishReason: "length",
Usage: types.TokenUsage{PromptTokens: 10, CompletionTokens: 4, TotalTokens: 14},
}}
recorder := &recordingChat{chatDelegate: fake}
thinking := false
options := &chat.ChatOptions{MaxTokens: 4096, Thinking: &thinking}
first, err := recorder.Chat(context.Background(), nil, options)
if err != nil || first != fake.response {
t.Fatal("recorder changed the response")
}
thinking = true
fake.response = &types.ChatResponse{Content: "\u4e2d\u6587", FinishReason: "stop"}
if _, err := recorder.Chat(context.Background(), nil, options); err != nil {
t.Fatal(err)
}
if recorder.calls != 2 || len(recorder.records) != 2 {
t.Fatal("missing call diagnostics")
}
a, b := recorder.records[0], recorder.records[1]
if a.Number != 1 || a.MaxTokens != 4096 || a.Thinking == nil || *a.Thinking ||
!a.ResponseReceived || a.Failed || a.FinishReason != "length" ||
a.ContentRunes != 6 || a.ReasoningRunes != 17 || a.Usage.TotalTokens != 14 {
t.Fatalf("unexpected first call metadata: %+v", a)
}
if b.Number != 2 || b.Thinking == nil || !*b.Thinking || b.ContentRunes != 2 || b.ReasoningRunes != 0 {
t.Fatalf("unexpected second call metadata: %+v", b)
}
encoded, err := json.Marshal(recorder.records)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(encoded), "private reasoning") || strings.Contains(string(encoded), "answer") {
t.Fatal("diagnostics exposed response content")
}
}
func TestRecordingChatRecordsNoResponseWithoutLeakingError(t *testing.T) {
want := errors.New("private upstream error")
recorder := &recordingChat{chatDelegate: &fakeProbeChat{err: want}}
response, err := recorder.Chat(context.Background(), nil, nil)
if response != nil || err != want || len(recorder.records) != 1 {
t.Fatal("recorder changed the failure")
}
record := recorder.records[0]
if record.ResponseReceived || !record.Failed || record.Thinking != nil || record.ContentRunes != 0 || record.ReasoningRunes != 0 {
t.Fatalf("unexpected failure metadata: %+v", record)
}
encoded, _ := json.Marshal(record)
if strings.Contains(string(encoded), want.Error()) {
t.Fatal("diagnostics exposed upstream error")
}
}
+117
View File
@@ -0,0 +1,117 @@
const assert = require('node:assert/strict')
const fs = require('node:fs')
const {execFileSync} = require('node:child_process')
const tenantID = 10059
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
const literal = value => `'${String(value).replaceAll("'", "''")}'`
const arraySQL = values => `ARRAY[${values.map(literal).join(',')}]::text[]`
function validateReport(report, now = Date.now()) {
assert.ok(report && typeof report === 'object', 'Missing indexed fixture report')
assert.equal(report.base, 'https://agent.pducv.cn/api/v1', 'Unexpected indexed API origin')
// Older completed reports omit tenant_id; every database identity is still checked below.
assert.ok(report.tenant_id === undefined || report.tenant_id === tenantID, 'Unexpected indexed tenant')
assert.match(report.prefix || '', /^indexed-isolation-[0-9]+$/, 'Invalid indexed fixture prefix')
for (const key of ['started_at', 'time']) {
assert.ok(typeof report[key] === 'string' && Number.isFinite(Date.parse(report[key])), `Invalid report ${key}`)
assert.equal(new Date(report[key]).toISOString(), report[key], `Noncanonical report ${key}`)
}
const start = Date.parse(report.started_at), end = Date.parse(report.time)
const prefixTime = Number(report.prefix.slice('indexed-isolation-'.length))
assert.ok(start <= end && end <= now && end - start <= 86400000, 'Invalid indexed report window')
assert.ok(Number.isSafeInteger(prefixTime) && prefixTime >= start && prefixTime <= end, 'Prefix outside report window')
const fixtures = report.fixtures
assert.ok(fixtures && typeof fixtures === 'object', 'Missing indexed fixture IDs')
const allIDs = []
for (const [key, maximum] of Object.entries({users: 2, roles: 1, kbs: 1, docs: 1, tags: 2})) {
assert.ok(Array.isArray(fixtures[key]) && fixtures[key].length <= maximum, `Invalid indexed ${key} count`)
for (const id of fixtures[key]) {
assert.ok(typeof id === 'string' && uuid.test(id), `Invalid indexed ${key} UUID`)
allIDs.push(id.toLowerCase())
}
}
assert.equal(new Set(allIDs).size, allIDs.length, 'Reused indexed fixture UUID')
assert.ok(!fixtures.kbs.length || fixtures.users.length > 0, 'Knowledge fixture has no owner')
assert.ok(!(fixtures.tags.length || fixtures.docs.length) || fixtures.kbs.length === 1, 'Child fixture has no knowledge base')
for (const id of fixtures.kbs) {
const proof = report.cleanup?.filter(item => item?.name === `knowledge ${id}`)
assert.ok(proof?.length === 1 && proof[0].passed === true && !proof[0].error, 'Missing successful knowledge deletion proof')
}
return fixtures
}
function buildCleanupSQL(report, now = Date.now()) {
const fixtures = validateReport(report, now)
const ids = Object.fromEntries(Object.entries(fixtures).filter(([, value]) => Array.isArray(value)).map(([key, value]) => [key, arraySQL(value)]))
const window = alias => `${alias}.created_at BETWEEN ${literal(report.started_at)}::timestamptz AND ${literal(report.time)}::timestamptz`
const deleted = alias => `${alias}.deleted_at BETWEEN ${alias}.created_at AND ${literal(report.time)}::timestamptz`
const guards = []
const requireSQL = (predicate, message) => guards.push(` IF (${predicate}) IS DISTINCT FROM TRUE THEN RAISE EXCEPTION ${literal(message)}; END IF;`)
fixtures.users.forEach((id, index) => {
const name = `${report.prefix}-${['owner', 'reader'][index]}`
requireSQL(`SELECT count(*)=1 FROM users u WHERE u.id=${literal(id)} AND u.tenant_id=${tenantID} AND u.username=${literal(name)} AND u.email=${literal(name + '@example.test')} AND ${window('u')}`, 'Indexed fixture user identity mismatch')
})
fixtures.kbs.forEach(id => {
requireSQL(`SELECT count(*)=1 FROM knowledge_bases k WHERE k.id=${literal(id)} AND k.tenant_id=${tenantID} AND k.name=${literal(report.prefix)} AND k.creator_id=${literal(fixtures.users[0])} AND ${window('k')} AND ${deleted('k')}`, 'Indexed fixture knowledge base must match its deleted owner scope')
})
fixtures.docs.forEach(id => {
requireSQL(`SELECT count(*)=1 FROM knowledges d WHERE d.id=${literal(id)} AND d.tenant_id=${tenantID} AND d.knowledge_base_id=ANY(${ids.kbs}) AND d.type='file' AND d.title=${literal(report.prefix + '.txt')} AND d.file_name=${literal(report.prefix + '.txt')} AND ${window('d')} AND ${deleted('d')}`, 'Indexed fixture document must match its deleted knowledge scope')
})
requireSQL(`NOT EXISTS (SELECT 1 FROM knowledges d WHERE d.knowledge_base_id=ANY(${ids.kbs}) AND NOT (d.id=ANY(${ids.docs})))`, 'Unreported document in indexed fixture knowledge base')
requireSQL(`NOT EXISTS (SELECT 1 FROM knowledge_tags t WHERE t.knowledge_base_id=ANY(${ids.kbs}) AND NOT (t.id=ANY(${ids.tags})))`, 'Unreported tag in indexed fixture knowledge base')
fixtures.tags.forEach((id, index) => {
const name = `${report.prefix}-${['category', 'empty-category'][index]}`
// Missing tags are allowed only for an already-cleaned (or partially cleaned) fixture.
requireSQL(`NOT EXISTS (SELECT 1 FROM knowledge_tags t WHERE t.id=${literal(id)} AND (t.tenant_id=${tenantID} AND t.knowledge_base_id=ANY(${ids.kbs}) AND t.name=${literal(name)} AND ${window('t')}) IS DISTINCT FROM TRUE)`, 'Indexed fixture tag identity mismatch')
})
requireSQL(`NOT EXISTS (SELECT 1 FROM knowledge_tag_relations r WHERE (r.tag_id=ANY(${ids.tags}) OR r.knowledge_id=ANY(${ids.docs})) AND (r.tag_id=ANY(${ids.tags}) AND r.knowledge_id=ANY(${ids.docs}) AND EXISTS (SELECT 1 FROM knowledge_tags t WHERE t.id=r.tag_id) AND ${window('r')}) IS DISTINCT FROM TRUE)`, 'Indexed fixture has an unexpected tag relation')
requireSQL(`NOT EXISTS (SELECT 1 FROM chunks c WHERE (c.tag_id=ANY(${ids.tags}) OR c.knowledge_base_id=ANY(${ids.kbs}) OR c.knowledge_id=ANY(${ids.docs})) AND (c.tenant_id=${tenantID} AND c.knowledge_base_id=ANY(${ids.kbs}) AND c.knowledge_id=ANY(${ids.docs}) AND (c.tag_id IS NULL OR c.tag_id='' OR c.tag_id=ANY(${ids.tags})) AND ${window('c')} AND ${deleted('c')}) IS DISTINCT FROM TRUE)`, 'Indexed fixture has an active or unrelated chunk')
return `BEGIN;
SET LOCAL lock_timeout='3s';
SET LOCAL statement_timeout='15s';
SET LOCAL idle_in_transaction_session_timeout='15s';
-- Relations have no foreign keys: table locks also exclude new out-of-scope links during verification.
LOCK TABLE knowledge_tags, knowledge_tag_relations IN SHARE ROW EXCLUSIVE MODE;
LOCK TABLE knowledges, chunks IN SHARE MODE;
DO $indexed_cleanup$
BEGIN
PERFORM 1 FROM users WHERE id=ANY(${ids.users}) FOR UPDATE;
PERFORM 1 FROM knowledge_bases WHERE id=ANY(${ids.kbs}) FOR UPDATE;
${guards.join('\n')}
END
$indexed_cleanup$;
WITH removed AS (
DELETE FROM knowledge_tag_relations WHERE tag_id=ANY(${ids.tags}) AND knowledge_id=ANY(${ids.docs}) RETURNING 1
) SELECT json_build_object('kind','relations','deleted',count(*)) FROM removed;
WITH removed AS (
DELETE FROM knowledge_tags WHERE tenant_id=${tenantID} AND knowledge_base_id=ANY(${ids.kbs}) AND id=ANY(${ids.tags}) RETURNING 1
) SELECT json_build_object('kind','tags','deleted',count(*)) FROM removed;
COMMIT;
`
}
function cleanupIndexedFixtureTags(report, run = execFileSync, now = Date.now()) {
const input = buildCleanupSQL(report, now)
const output = run('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres',
'psql', '-X', '-q', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-At', '-v', 'ON_ERROR_STOP=1'],
{input, encoding: 'utf8', timeout: 45000, maxBuffer: 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe']}).trim()
const results = output.split('\n').map(line => JSON.parse(line))
assert.deepEqual(results.map(result => result.kind), ['relations', 'tags'], 'Unexpected cleanup result')
for (const result of results) assert.ok(Number.isSafeInteger(result.deleted) && result.deleted >= 0 && result.deleted <= report.fixtures.tags.length, 'Invalid cleanup count')
return {relations_deleted: results[0].deleted, tags_deleted: results[1].deleted}
}
function main() {
const args = process.argv.slice(2)
const apply = args[0] === '--apply'
assert.equal(args.length, apply ? 2 : 1, 'Usage: node scripts/indexed_fixture_tag_cleanup.cjs [--apply] REPORT.json')
const report = JSON.parse(fs.readFileSync(args[apply ? 1 : 0], 'utf8'))
if (apply) console.log(JSON.stringify(cleanupIndexedFixtureTags(report)))
else process.stdout.write(buildCleanupSQL(report))
}
module.exports = {validateReport, buildCleanupSQL, cleanupIndexedFixtureTags}
if (require.main === module) {
try { main() } catch (error) { console.error(error.message); process.exitCode = 1 }
}
@@ -0,0 +1,236 @@
const assert = require('node:assert/strict')
const test = require('node:test')
const fs = require('node:fs')
const path = require('node:path')
const {execFileSync} = require('node:child_process')
const {validateReport, buildCleanupSQL, cleanupIndexedFixtureTags} = require('./indexed_fixture_tag_cleanup.cjs')
const id = number => `00000000-0000-4000-8000-${String(number).padStart(12, '0')}`
const literal = value => value === null ? 'NULL' : typeof value === 'number' ? String(value) : `'${String(value).replaceAll("'", "''")}'`
const start = '2026-09-19T17:45:09.219Z'
const created = '2026-09-19T17:45:12.000Z'
const deleted = '2026-09-19T17:45:40.000Z'
const finish = '2026-09-19T17:45:45.637Z'
const now = Date.parse(finish) + 1000
function fixture() {
return {base: 'https://agent.pducv.cn/api/v1', tenant_id: 10059,
started_at: start, time: finish, prefix: 'indexed-isolation-1789839909244',
fixtures: {users: [id(1), id(2)], roles: [id(3)], kbs: [id(4)], docs: [id(5)], tags: [id(6), id(7)]},
cleanup: [{name: `knowledge ${id(4)}`, passed: true}], passed: true}
}
for (const [name, mutate] of [
['wrong tenant', r => {r.tenant_id = 1}],
['wrong API', r => {r.base = 'http://localhost/api/v1'}],
['unsafe prefix', r => {r.prefix += "'; DELETE FROM users;--"}],
['foreign prefix', r => {r.prefix = 'resource-isolation-1789839909244'}],
['prefix outside window', r => {r.prefix = 'indexed-isolation-1'}],
['invalid timestamp', r => {r.time = 'today'}],
['noncanonical timestamp', r => {r.time = '2026-09-19'}],
['inverted window', r => {r.started_at = r.time}],
['future report', r => {r.time = '2099-01-01T00:00:00.000Z'}],
['excessive window', r => {r.started_at = '2026-09-17T00:00:00.000Z'}],
['missing fixture IDs', r => {delete r.fixtures.tags}],
['invalid fixture UUID', r => {r.fixtures.tags[0] = 'all'}],
['nonstring UUID', r => {r.fixtures.tags[0] = 123}],
['duplicate UUID', r => {r.fixtures.tags[1] = r.fixtures.tags[0]}],
['reused UUID across types', r => {r.fixtures.tags[0] = r.fixtures.docs[0]}],
['too many fixtures', r => {r.fixtures.kbs.push(id(8))}],
['missing owner', r => {r.fixtures.users = []}],
['missing parent', r => {r.fixtures.kbs = []}],
['missing deletion proof', r => {delete r.cleanup}],
['failed deletion proof', r => {r.cleanup[0].passed = false}],
['deletion proof with error', r => {r.cleanup[0].error = 'failed'}],
['duplicate deletion proof', r => {r.cleanup.push({...r.cleanup[0]})}],
['wrong deletion proof', r => {r.cleanup[0].name = `knowledge ${id(8)}`}],
]) {
test(`rejects ${name} before executing SQL`, () => {
const report = fixture()
mutate(report)
let calls = 0
assert.throws(() => cleanupIndexedFixtureTags(report, () => {calls++; return ''}, now))
assert.equal(calls, 0)
})
}
test('accepts legacy reports and failed runs with successful knowledge deletion', () => {
const report = fixture()
delete report.tenant_id
report.passed = false
report.failures = ['Search fixture failed']
assert.doesNotThrow(() => validateReport(report, now))
})
test('uses one bounded transaction through stdin and only deletes tags and their relations', () => {
let call
const result = cleanupIndexedFixtureTags(fixture(), (command, args, options) => {
call = {command, args, options}
return '{"kind":"relations","deleted":1}\n{"kind":"tags","deleted":2}\n'
}, now)
assert.deepEqual(result, {relations_deleted: 1, tags_deleted: 2})
assert.equal(call.command, 'docker')
assert.ok(call.args.includes('colima-yawan'))
assert.ok(call.args.includes('yawan_native_acceptance_20260915'))
assert.ok(call.args.includes('ON_ERROR_STOP=1'))
assert.ok(call.args.includes('-X'))
assert.equal(call.options.timeout, 45000)
assert.deepEqual(call.options.stdio, ['pipe', 'pipe', 'pipe'])
const sql = call.options.input
assert.match(sql, /^BEGIN;\nSET LOCAL lock_timeout='3s';/)
assert.match(sql, /SET LOCAL statement_timeout='15s'/)
assert.match(sql, /SET LOCAL idle_in_transaction_session_timeout='15s'/)
assert.match(sql, /LOCK TABLE knowledge_tags, knowledge_tag_relations IN SHARE ROW EXCLUSIVE MODE/)
assert.match(sql, /LOCK TABLE knowledges, chunks IN SHARE MODE/)
assert.match(sql, /FROM users.*FOR UPDATE/)
assert.match(sql, /FROM knowledge_bases.*FOR UPDATE/)
assert.match(sql, /COMMIT;\n$/)
assert.deepEqual([...sql.matchAll(/DELETE FROM (\w+)/g)].map(match => match[1]), ['knowledge_tag_relations', 'knowledge_tags'])
assert.doesNotMatch(sql, /\b(?:UPDATE|INSERT INTO|TRUNCATE|DROP|ALTER)\s+\w+/)
assert.ok(sql.indexOf('RAISE EXCEPTION') < sql.indexOf('DELETE FROM'))
})
test('propagates SQL refusal and rejects malformed command results', () => {
assert.throws(() => cleanupIndexedFixtureTags(fixture(), () => {throw new Error('transaction refused')}, now), /transaction refused/)
for (const output of ['', '{}', '{"kind":"relations","deleted":-1}\n{"kind":"tags","deleted":2}', '{"kind":"relations","deleted":1}\n{"kind":"tags","deleted":3}']) {
assert.throws(() => cleanupIndexedFixtureTags(fixture(), () => output, now))
}
})
const pgBin = process.env.POSTGRES_TEST_BIN || '/opt/homebrew/opt/postgresql@16/bin'
test('cleanup SQL runs against an isolated local PostgreSQL cluster', {skip: !fs.existsSync(path.join(pgBin, 'initdb'))}, async t => {
const directory = fs.mkdtempSync('/tmp/indexed-tag-test-')
const data = path.join(directory, 'data')
let running = false
const run = (binary, args, input) => execFileSync(path.join(pgBin, binary), args, {
input, encoding: 'utf8', timeout: 30000, stdio: ['pipe', 'pipe', 'pipe'],
})
const query = sql => run('psql', ['-X', '-q', '-h', directory, '-p', '55459', '-U', 'fixture', '-d', 'postgres', '-At', '-v', 'ON_ERROR_STOP=1'], sql).trim()
const tables = ['users', 'knowledge_bases', 'knowledges', 'knowledge_tags', 'knowledge_tag_relations', 'chunks']
const snapshot = () => JSON.parse(query(`SELECT json_build_object(${tables.map(name => `${literal(name)},(SELECT COALESCE(json_agg(t ORDER BY row_to_json(t)::text),'[]'::json) FROM ${name} t)`).join(',')});`))
const insert = (table, rows) => rows.length ? `INSERT INTO ${table} (${Object.keys(rows[0]).join(',')}) VALUES ${rows.map(row => `(${Object.values(row).map(literal).join(',')})`).join(',')};` : ''
function seed(report = fixture()) {
const users = report.fixtures.users.map((userID, index) => ({id: userID, tenant_id: 10059,
username: `${report.prefix}-${['owner', 'reader'][index]}`, email: `${report.prefix}-${['owner', 'reader'][index]}@example.test`, created_at: created, deleted_at: deleted}))
const kbs = report.fixtures.kbs.map(kbID => ({id: kbID, tenant_id: 10059, creator_id: report.fixtures.users[0], name: report.prefix, created_at: created, deleted_at: deleted}))
const docs = report.fixtures.docs.map(docID => ({id: docID, tenant_id: 10059, knowledge_base_id: report.fixtures.kbs[0], title: report.prefix + '.txt', file_name: report.prefix + '.txt', type: 'file', created_at: created, deleted_at: deleted}))
const tags = report.fixtures.tags.map((tagID, index) => ({id: tagID, tenant_id: 10059, knowledge_base_id: report.fixtures.kbs[0], name: `${report.prefix}-${['category', 'empty-category'][index]}`, created_at: created}))
const relations = docs.length && tags.length ? [{knowledge_id: docs[0].id, tag_id: tags[0].id, created_at: created}] : []
const chunks = docs.length ? [{id: id(8), tenant_id: 10059, knowledge_base_id: report.fixtures.kbs[0], knowledge_id: docs[0].id, tag_id: '', created_at: created, deleted_at: deleted}] : []
users.push({id: id(101), tenant_id: 10059, username: 'real-user', email: 'real-user@example.org', created_at: created, deleted_at: null})
kbs.push({id: id(104), tenant_id: 10059, creator_id: id(101), name: 'Real knowledge', created_at: created, deleted_at: null})
docs.push({id: id(105), tenant_id: 10059, knowledge_base_id: id(104), title: 'Real file', file_name: 'real.txt', type: 'file', created_at: created, deleted_at: null})
tags.push({id: id(106), tenant_id: 10059, knowledge_base_id: id(104), name: 'Real category', created_at: created})
relations.push({knowledge_id: id(105), tag_id: id(106), created_at: created})
chunks.push({id: id(108), tenant_id: 10059, knowledge_base_id: id(104), knowledge_id: id(105), tag_id: id(106), created_at: created, deleted_at: null})
query(`TRUNCATE ${tables.join(',')};\n` + Object.entries({users, knowledge_bases: kbs, knowledges: docs, knowledge_tags: tags, knowledge_tag_relations: relations, chunks}).map(([table, rows]) => insert(table, rows)).join('\n'))
}
try {
run('initdb', ['-D', data, '-U', 'fixture', '--auth=trust', '--no-locale', '--encoding=UTF8'])
run('pg_ctl', ['-D', data, '-l', path.join(directory, 'postgres.log'), '-o', `-F -c listen_addresses='' -k ${directory} -p 55459`, '-w', 'start'])
running = true
query(`CREATE TABLE users (id text PRIMARY KEY, tenant_id int, username text, email text, created_at timestamptz, deleted_at timestamptz);
CREATE TABLE knowledge_bases (id text PRIMARY KEY, tenant_id int, creator_id text, name text, created_at timestamptz, deleted_at timestamptz);
CREATE TABLE knowledges (id text PRIMARY KEY, tenant_id int, knowledge_base_id text, title text, file_name text, type text, created_at timestamptz, deleted_at timestamptz);
CREATE TABLE knowledge_tags (id text PRIMARY KEY, tenant_id int, knowledge_base_id text, name text, created_at timestamptz);
CREATE TABLE knowledge_tag_relations (knowledge_id text, tag_id text, created_at timestamptz, PRIMARY KEY (knowledge_id,tag_id));
CREATE TABLE chunks (id text PRIMARY KEY, tenant_id int, knowledge_base_id text, knowledge_id text, tag_id text, created_at timestamptz, deleted_at timestamptz);`)
await t.test('removes only two fixture tags and one relation, retaining all soft-delete history', () => {
seed()
const before = snapshot()
const result = query(buildCleanupSQL(fixture(), now)).split('\n').map(JSON.parse)
assert.deepEqual(result, [{kind: 'relations', deleted: 1}, {kind: 'tags', deleted: 2}])
const after = snapshot()
for (const table of ['users', 'knowledge_bases', 'knowledges', 'chunks']) assert.deepEqual(after[table], before[table])
assert.deepEqual(after.knowledge_tags, before.knowledge_tags.filter(row => row.id === id(106)))
assert.deepEqual(after.knowledge_tag_relations, before.knowledge_tag_relations.filter(row => row.tag_id === id(106)))
assert.deepEqual(query(buildCleanupSQL(fixture(), now)).split('\n').map(JSON.parse), [{kind: 'relations', deleted: 0}, {kind: 'tags', deleted: 0}])
assert.deepEqual(snapshot(), after)
})
for (const [name, sql] of [
['wrong user tenant', `UPDATE users SET tenant_id=1 WHERE id='${id(1)}'`],
['wrong owner username', `UPDATE users SET username='real-user-2' WHERE id='${id(1)}'`],
['wrong owner email', `UPDATE users SET email='real@example.org' WHERE id='${id(1)}'`],
['wrong reader identity', `UPDATE users SET username='other-reader' WHERE id='${id(2)}'`],
['owner outside window', `UPDATE users SET created_at='2026-09-01' WHERE id='${id(1)}'`],
['missing owner', `DELETE FROM users WHERE id='${id(1)}'`],
['wrong KB tenant', `UPDATE knowledge_bases SET tenant_id=1 WHERE id='${id(4)}'`],
['wrong KB owner', `UPDATE knowledge_bases SET creator_id='${id(101)}' WHERE id='${id(4)}'`],
['wrong KB name', `UPDATE knowledge_bases SET name='Real knowledge 2' WHERE id='${id(4)}'`],
['active KB', `UPDATE knowledge_bases SET deleted_at=NULL WHERE id='${id(4)}'`],
['KB outside window', `UPDATE knowledge_bases SET created_at='2026-09-01' WHERE id='${id(4)}'`],
['missing KB history', `DELETE FROM knowledge_bases WHERE id='${id(4)}'`],
['wrong document tenant', `UPDATE knowledges SET tenant_id=1 WHERE id='${id(5)}'`],
['wrong document KB', `UPDATE knowledges SET knowledge_base_id='${id(104)}' WHERE id='${id(5)}'`],
['wrong document title', `UPDATE knowledges SET title='Real file' WHERE id='${id(5)}'`],
['wrong document file', `UPDATE knowledges SET file_name='real.txt' WHERE id='${id(5)}'`],
['wrong document type', `UPDATE knowledges SET type='url' WHERE id='${id(5)}'`],
['active document', `UPDATE knowledges SET deleted_at=NULL WHERE id='${id(5)}'`],
['document outside window', `UPDATE knowledges SET created_at='2026-09-01' WHERE id='${id(5)}'`],
['missing document history', `DELETE FROM knowledges WHERE id='${id(5)}'`],
['unreported document', `UPDATE knowledges SET knowledge_base_id='${id(4)}' WHERE id='${id(105)}'`],
['wrong tag tenant', `UPDATE knowledge_tags SET tenant_id=1 WHERE id='${id(6)}'`],
['wrong tag KB', `UPDATE knowledge_tags SET knowledge_base_id='${id(104)}' WHERE id='${id(6)}'`],
['wrong tag name', `UPDATE knowledge_tags SET name='Real category 2' WHERE id='${id(6)}'`],
['null tag metadata', `UPDATE knowledge_tags SET name=NULL WHERE id='${id(6)}'`],
['tag outside window', `UPDATE knowledge_tags SET created_at='2026-09-01' WHERE id='${id(6)}'`],
['unreported tag', `UPDATE knowledge_tags SET knowledge_base_id='${id(4)}' WHERE id='${id(106)}'`],
['unrelated document relation', `INSERT INTO knowledge_tag_relations VALUES ('${id(105)}','${id(6)}','${created}')`],
['unreported relation tag', `INSERT INTO knowledge_tag_relations VALUES ('${id(5)}','${id(106)}','${created}')`],
['relation without retained tag', `DELETE FROM knowledge_tags WHERE id='${id(6)}'`],
['relation outside window', `UPDATE knowledge_tag_relations SET created_at='2026-09-01' WHERE tag_id='${id(6)}'`],
['active chunk', `UPDATE chunks SET deleted_at=NULL WHERE id='${id(8)}'`],
['unrelated chunk with fixture tag', `UPDATE chunks SET tag_id='${id(6)}' WHERE id='${id(108)}'`],
['fixture chunk with external tag', `UPDATE chunks SET tag_id='${id(106)}' WHERE id='${id(8)}'`],
['wrong chunk tenant', `UPDATE chunks SET tenant_id=1 WHERE id='${id(8)}'`],
['chunk outside window', `UPDATE chunks SET created_at='2026-09-01' WHERE id='${id(8)}'`],
]) {
await t.test(`rolls back without deleting anything for ${name}`, () => {
seed()
query(sql)
const before = snapshot()
assert.throws(() => query(buildCleanupSQL(fixture(), now)), /ERROR:/)
assert.deepEqual(snapshot(), before)
})
}
for (const [name, mutate] of [
['no fixtures yet', r => {for (const key of Object.keys(r.fixtures)) r.fixtures[key] = []; r.cleanup = []}],
['owner created only', r => {r.fixtures.users.pop(); r.fixtures.kbs = []; r.fixtures.tags = []; r.fixtures.docs = []; r.cleanup = []}],
['KB created without tags', r => {r.fixtures.tags = []; r.fixtures.docs = []}],
['first tag created before failure', r => {r.fixtures.tags.pop(); r.fixtures.docs = []}],
['two tags before upload failure', r => {r.fixtures.docs = []}],
]) {
await t.test(`supports partial fixture: ${name}`, () => {
const report = fixture()
mutate(report)
seed(report)
assert.doesNotThrow(() => query(buildCleanupSQL(report, now)))
assert.equal(snapshot().knowledge_tags.length, 1)
assert.equal(snapshot().knowledge_tag_relations.length, 1)
})
}
await t.test('supports one previously removed empty tag', () => {
seed()
query(`DELETE FROM knowledge_tags WHERE id='${id(7)}'`)
assert.deepEqual(query(buildCleanupSQL(fixture(), now)).split('\n').map(JSON.parse), [{kind: 'relations', deleted: 1}, {kind: 'tags', deleted: 1}])
})
await t.test('rolls back relation removal when the later tag delete fails', () => {
seed()
const before = snapshot()
query(`CREATE FUNCTION refuse_tag_delete() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'test tag delete failure'; END $$;
CREATE TRIGGER refuse_tag_delete BEFORE DELETE ON knowledge_tags FOR EACH ROW EXECUTE FUNCTION refuse_tag_delete();`)
try {
assert.throws(() => query(buildCleanupSQL(fixture(), now)), /test tag delete failure/)
assert.deepEqual(snapshot(), before)
} finally { query('DROP TRIGGER refuse_tag_delete ON knowledge_tags; DROP FUNCTION refuse_tag_delete();') }
})
} finally {
if (running) run('pg_ctl', ['-D', data, '-m', 'immediate', '-w', 'stop'])
fs.rmSync(directory, {recursive: true, force: true})
}
})
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
# Install WeKnora git hooks (core.hooksPath -> scripts/git-hooks).
#
# Usage (from repo root):
# ./scripts/install-git-hooks.sh
#
# Bypass hooks for one command:
# SKIP_HOOKS=1 git commit ...
# SKIP_HOOKS=1 git push ...
#
# Skip slow go test on pre-push:
# HOOK_SKIP_TEST=1 git push ...
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
HOOKS_DIR="$ROOT/scripts/git-hooks"
if [[ ! -d "$HOOKS_DIR" ]]; then
echo "hooks directory not found: $HOOKS_DIR" >&2
exit 1
fi
chmod +x "$HOOKS_DIR"/common.sh "$HOOKS_DIR"/pre-commit "$HOOKS_DIR"/pre-push
git -C "$ROOT" config core.hooksPath scripts/git-hooks
echo "Installed git hooks -> scripts/git-hooks"
echo " pre-commit: whitespace, gofmt (auto-fix), golangci-lint (if installed)"
echo " pre-push: PR gofmt + full go vet/build (like CI); scoped go test"
echo ""
echo "Optional: SKIP_HOOKS=1 or HOOK_SKIP_TEST=1 — see script header."
+46
View File
@@ -0,0 +1,46 @@
#!/bin/sh
set -eu
# Read-only preflight for the legacy SQLite -> native WeKnora migration.
# It never writes to the source database and never imports data. The output is
# intentionally small enough to attach to a migration review.
DB_PATH=${LEGACY_DB_PATH:-/Volumes/Extreme\ SSD/yawan-storage/yawan-workbench/server/data/workspace.db}
if [ ! -f "$DB_PATH" ]; then
DB_PATH=${LEGACY_DB_PATH_FALLBACK:-/Users/macclaw/Desktop/yawan-workbench/server/data-dev/workspace.db}
fi
if [ ! -f "$DB_PATH" ]; then
echo "legacy database not found; set LEGACY_DB_PATH" >&2
exit 2
fi
command -v sqlite3 >/dev/null 2>&1 || { echo "sqlite3 is required" >&2; exit 2; }
sha=$(shasum -a 256 "$DB_PATH" | awk '{print $1}')
size=$(wc -c < "$DB_PATH" | tr -d ' ')
integrity=$(sqlite3 "$DB_PATH" 'PRAGMA integrity_check;' | tr '\n' ' ' | sed 's/[[:space:]]*$//')
if [ "$integrity" != "ok" ]; then
echo "legacy database integrity check failed: $integrity" >&2
exit 1
fi
count() {
table=$1
# sqlite3 exits non-zero for a missing table; surface that as zero only for
# optional legacy tables so an audit can run against older snapshots.
sqlite3 "$DB_PATH" "SELECT count(*) FROM \"$table\";" 2>/dev/null || echo 0
}
printf '{\n'
printf ' "path": %s,\n' "$(printf '%s' "$DB_PATH" | awk '{gsub(/\\/, "\\\\"); gsub(/"/, "\\\""); printf "\"%s\"", $0}')"
printf ' "bytes": %s,\n' "$size"
printf ' "sha256": "%s",\n' "$sha"
printf ' "integrity": "ok",\n'
printf ' "counts": {\n'
tables='appuser knowledgebase knowledgefile knowledgecategory agent agentskill agentskillbinding canvas trainingmap segment digitalhuman memory knowledgeusage pointaccount pointtransaction auditlog systemlog'
first=1
for table in $tables; do
[ "$first" -eq 1 ] || printf ',\n'
first=0
printf ' "%s": %s' "$table" "$(count "$table")"
done
printf '\n }\n}\n'
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env node
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
const database = process.env.GRAPH_MIGRATION_DATABASE || 'yawan_native_acceptance_20260915';
if (!/^yawan_native_acceptance_[0-9]+$/.test(database)) throw new Error('Only isolated acceptance databases are allowed');
if (!process.argv.includes('--apply')) throw new Error('Usage: node scripts/map_native_graph_evidence.mjs --apply');
const tenant = Number(process.env.GRAPH_MIGRATION_TENANT || 10059);
if (!Number.isSafeInteger(tenant) || tenant < 1) throw new Error('Invalid tenant');
const base = ['--context','colima-yawan','exec','-i','WeKnora-postgres','psql','-X','-v','ON_ERROR_STOP=1','-U','postgres','-d',database,'-At'];
const quote = value => `'${String(value).replaceAll("'", "''")}'`;
function sql(query) {
const result = spawnSync('docker',base,{input:query,encoding:'utf8',maxBuffer:256*1024*1024});
if(result.status!==0) throw new Error(result.stderr);
return result.stdout.trim().split('\n').filter(Boolean);
}
const normalize = value => value.normalize('NFKC').replace(/\s+/gu,'');
const documents = sql(`WITH candidates AS (SELECT k.id,k.knowledge_base_id,s.source_file,count(*) OVER (PARTITION BY s.knowledge_base_id,s.source_file) AS matches FROM (SELECT DISTINCT knowledge_base_id,source_file FROM native_graph_sources WHERE tenant_id=${tenant}) s JOIN knowledges k ON k.tenant_id=${tenant} AND k.knowledge_base_id=s.knowledge_base_id AND k.file_name=regexp_replace(s.source_file,'\\.jsonl$','.md') AND k.deleted_at IS NULL) SELECT json_build_object('id',id,'kb',knowledge_base_id,'file',source_file) FROM candidates WHERE matches=1 ORDER BY id;`).map(JSON.parse);
let mapped = 0, ambiguous = 0;
for (const document of documents) {
const rows=sql(`SELECT json_build_object('id',source_chunk_id,'text',excerpt) FROM native_graph_sources WHERE tenant_id=${tenant} AND knowledge_base_id=${quote(document.kb)} AND source_file=${quote(document.file)} AND native_chunk_id IS NULL;`).map(JSON.parse);
const prefixes=new Map();
for (const row of rows) {
row.normalized=normalize(row.text);row.matches=[];
if([...row.normalized].length<80)continue;
const key=row.normalized.slice(0,16);
if(!prefixes.has(key))prefixes.set(key,[]);
prefixes.get(key).push(row);
}
if(!prefixes.size)continue;
const chunks=sql(`SELECT json_build_object('id',id,'content',content) FROM chunks WHERE tenant_id=${tenant} AND knowledge_base_id=${quote(document.kb)} AND knowledge_id=${quote(document.id)} AND chunk_type='text' AND deleted_at IS NULL AND is_enabled=true ORDER BY id;`).map(JSON.parse);
for(const chunk of chunks) {
const content=normalize(chunk.content);
const seen=new Set();
for(let pos=0;pos<=content.length-16;pos++) {
const candidates=prefixes.get(content.slice(pos,pos+16));
if(!candidates)continue;
for(const row of candidates) {
if(seen.has(row.id)||row.matches.length>1)continue;
if(content.startsWith(row.normalized,pos)) {
row.matches.push({id:chunk.id,hash:createHash('sha256').update(chunk.content).digest('hex')});seen.add(row.id);
}
}
}
}
const matches=rows.filter(row=>row.matches.length===1);
ambiguous+=rows.filter(row=>row.matches.length>1).length;
if(matches.length) {
const values=matches.map(row=>`(${quote(row.id)},${quote(row.matches[0].id)},${quote(row.matches[0].hash)})`).join(',\n');
sql(`BEGIN; UPDATE native_graph_sources s SET native_knowledge_id=${quote(document.id)},native_chunk_id=m.chunk_id,native_content_sha256=m.hash,mapping_state='normalized_full_excerpt' FROM (VALUES ${values}) m(source_id,chunk_id,hash) WHERE s.tenant_id=${tenant} AND s.knowledge_base_id=${quote(document.kb)} AND s.source_file=${quote(document.file)} AND s.source_chunk_id=m.source_id AND s.native_chunk_id IS NULL; COMMIT;`);
}
mapped+=matches.length;
console.log(JSON.stringify({file:document.file,candidates:rows.length,mapped:matches.length}));
}
console.log(JSON.stringify({mapped,ambiguous,normalization:'Unicode NFKC and whitespace removal; entire excerpt >= 80 codepoints; unique native chunk in matching tenant, KB and source document'}));
+122
View File
@@ -0,0 +1,122 @@
#!/bin/bash
set -e
# Get the script directory and project root
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
# Load .env file if it exists (for development mode)
if [ -f "$PROJECT_ROOT/.env" ]; then
echo "Loading .env file from $PROJECT_ROOT/.env"
set -a
source "$PROJECT_ROOT/.env"
set +a
fi
# Database connection details (can be overridden by environment variables)
DB_HOST=${DB_HOST:-localhost}
DB_PORT=${DB_PORT:-5432}
DB_USER=${DB_USER:-postgres}
DB_PASSWORD=${DB_PASSWORD:-postgres}
DB_NAME=${DB_NAME:-WeKnora}
# Use versioned migrations directory
MIGRATIONS_DIR="${MIGRATIONS_DIR:-migrations/versioned}"
# Check if migrate tool is installed
if ! command -v migrate &> /dev/null; then
echo "Error: migrate tool is not installed"
echo "Install it with: go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest"
exit 1
fi
# Construct the database URL
# If DB_URL is already set in .env, use it but ensure sslmode=disable is set
# Otherwise, construct it from individual components
if [ -n "$DB_URL" ]; then
# If DB_URL already exists, ensure sslmode=disable is set (unless sslmode is already specified)
if [[ "$DB_URL" != *"sslmode="* ]]; then
# Add sslmode=disable if not present
if [[ "$DB_URL" == *"?"* ]]; then
DB_URL="${DB_URL}&sslmode=disable"
else
DB_URL="${DB_URL}?sslmode=disable"
fi
elif [[ "$DB_URL" == *"sslmode=require"* ]] || [[ "$DB_URL" == *"sslmode=prefer"* ]]; then
# Replace sslmode=require/prefer with sslmode=disable for local dev
DB_URL="${DB_URL//sslmode=require/sslmode=disable}"
DB_URL="${DB_URL//sslmode=prefer/sslmode=disable}"
fi
else
# Use Python to properly URL encode password if it contains special characters
# This handles special characters in passwords correctly
if command -v python3 &> /dev/null; then
ENCODED_PASSWORD=$(python3 -c "import urllib.parse; print(urllib.parse.quote('$DB_PASSWORD', safe=''))")
else
# Fallback: try to use printf for basic encoding (may not work for all special chars)
ENCODED_PASSWORD="$DB_PASSWORD"
fi
DB_URL="postgres://${DB_USER}:${ENCODED_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable"
fi
# Execute migration based on command
case "$1" in
up)
echo "Running migrations up..."
echo "DB_URL: ${DB_URL}"
echo "DB_USER: ${DB_USER}"
echo "DB_PASSWORD: ${DB_PASSWORD}"
echo "DB_HOST: ${DB_HOST}"
echo "DB_PORT: ${DB_PORT}"
echo "DB_NAME: ${DB_NAME}"
echo "MIGRATIONS_DIR: ${MIGRATIONS_DIR}"
migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} up
;;
down)
echo "Running migrations down..."
migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} down
;;
create)
if [ -z "$2" ]; then
echo "Error: Migration name is required"
echo "Usage: $0 create <migration_name>"
exit 1
fi
echo "Creating migration files for $2..."
migrate create -ext sql -dir ${MIGRATIONS_DIR} -seq $2
echo "Created:"
echo " - ${MIGRATIONS_DIR}/$(ls -t ${MIGRATIONS_DIR} | head -1)"
echo " - ${MIGRATIONS_DIR}/$(ls -t ${MIGRATIONS_DIR} | head -2 | tail -1)"
;;
version)
echo "Checking current migration version..."
migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} version
;;
force)
if [ -z "$2" ]; then
echo "Error: Version number is required"
echo "Usage: $0 force <version>"
echo "Note: Use -1 to reset to no version (allows re-running all migrations)"
exit 1
fi
VERSION="$2"
echo "Forcing migration version to $VERSION..."
# Use env to pass the command, avoiding shell flag parsing issues with negative numbers
env migrate -path "${MIGRATIONS_DIR}" -database "${DB_URL}" force -- "$VERSION"
;;
goto)
if [ -z "$2" ]; then
echo "Error: Version number is required"
echo "Usage: $0 goto <version>"
exit 1
fi
echo "Migrating to version $2..."
migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} goto $2
;;
*)
echo "Usage: $0 {up|down|create <migration_name>|version|force <version>|goto <version>}"
exit 1
;;
esac
echo "Migration command completed successfully"
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env node
import { spawn, spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import { createReadStream } from 'node:fs';
import { once } from 'node:events';
import { createInterface } from 'node:readline';
import { fileURLToPath } from 'node:url';
const snapshot = process.argv[2];
if (!snapshot || !process.argv.includes('--apply')) throw new Error('Usage: node scripts/migrate_native_graph.mjs SNAPSHOT --apply');
const database = process.env.GRAPH_MIGRATION_DATABASE || 'yawan_native_acceptance_20260915';
if (!/^yawan_native_acceptance_[0-9]+$/.test(database)) throw new Error('Only isolated acceptance databases are allowed');
const tenant = Number(process.env.GRAPH_MIGRATION_TENANT || 10059);
if (!Number.isSafeInteger(tenant) || tenant < 1) throw new Error('Invalid tenant');
const docker = ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres', 'psql', '-X', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', database];
const hash = createHash('sha256');
for await (const part of createReadStream(snapshot)) hash.update(part);
const digest = hash.digest('hex');
const sqlite = (sql) => {
const result = spawnSync('sqlite3', ['-readonly', '-json', snapshot, sql], {encoding:'utf8', maxBuffer:8*1024*1024});
if (result.status !== 0) throw new Error(result.stderr);
return JSON.parse(result.stdout || '[]');
};
const quote = value => `'${String(value).replaceAll("'", "''")}'`;
const sourceKBs = sqlite('SELECT id,name FROM knowledgebase ORDER BY id');
const proc = spawn('docker', docker, {stdio:['pipe','pipe','pipe']});
proc.stdout.pipe(process.stdout); proc.stderr.pipe(process.stderr);
const completion = once(proc, 'close');
const write = async data => {if (!proc.stdin.write(data)) await once(proc.stdin,'drain');};
await write('BEGIN; SET statement_timeout = 0;\n');
await write(await (await import('node:fs/promises')).readFile(fileURLToPath(new URL('../migrations/versioned/000200_native_graph.up.sql', import.meta.url)), 'utf8'));
await write(`\nCREATE TEMP TABLE graph_kb_map(legacy_id bigint PRIMARY KEY,native_id varchar(36) NOT NULL);\n`);
for (const kb of sourceKBs) {
await write(`DO $$ BEGIN IF (SELECT count(*) FROM knowledge_bases WHERE tenant_id=${tenant} AND name=${quote(kb.name)} AND deleted_at IS NULL)<>1 THEN RAISE EXCEPTION 'Ambiguous or missing knowledge base ${kb.id}'; END IF; END $$;\nINSERT INTO graph_kb_map SELECT ${kb.id},id FROM knowledge_bases WHERE tenant_id=${tenant} AND name=${quote(kb.name)} AND deleted_at IS NULL;\n`);
}
const tables = ['knowledgegraphversion','knowledgeentity','knowledgerelation','knowledgegraphevidence','knowledgegraphchunkstate','knowledgegraphsourceunit','knowledgegraphclaim','knowledgeentityalias','knowledgegraphcommunity','knowledgegraphcommunityreport'];
const expected = {};
for (const table of tables) {
const columns = sqlite(`PRAGMA table_info(${table})`).map(c=>c.name);
expected[table] = sqlite(`SELECT count(*) AS n FROM ${table}`)[0].n;
const sql = `SELECT json_object(${columns.flatMap(c=>[quote(c), '"'+c+'"']).join(',')}) FROM ${table} ORDER BY id;`;
await write(`CREATE TEMP TABLE graph_import_${table}(payload jsonb);\nCOPY graph_import_${table}(payload) FROM STDIN WITH (FORMAT csv);\n`);
const source = spawn('sqlite3',['-readonly','-noheader',snapshot,sql],{stdio:['ignore','pipe','pipe']});
const sourceCompletion = once(source,'close');
source.stderr.pipe(process.stderr);
for await (const line of createInterface({input:source.stdout,crlfDelay:Infinity})) await write('"'+line.replaceAll('"','""')+'"\n');
const [sourceCode]=await sourceCompletion;
if(sourceCode!==0){proc.kill();throw new Error(`SQLite export failed: ${table}`);}
await write(`\\.\nDO $$ BEGIN IF EXISTS(SELECT 1 FROM graph_import_${table} s JOIN graph_kb_map k ON k.legacy_id=(s.payload->>'kb_id')::bigint JOIN native_graph_records r ON r.knowledge_base_id=k.native_id AND r.tenant_id=${tenant} AND r.source_table=${quote(table)} AND r.legacy_id=(s.payload->>'id')::bigint WHERE r.payload<>s.payload) THEN RAISE EXCEPTION 'Source record differs from preserved graph archive'; END IF; END $$;\nINSERT INTO native_graph_records(tenant_id,knowledge_base_id,source_table,legacy_id,graph_version_id,payload,source_snapshot_sha256) SELECT ${tenant},k.native_id,${quote(table)},(s.payload->>'id')::bigint,COALESCE((s.payload->>'graph_version_id')::bigint,(s.payload->>'id')::bigint),s.payload,${quote(digest)} FROM graph_import_${table} s JOIN graph_kb_map k ON k.legacy_id=(s.payload->>'kb_id')::bigint ON CONFLICT DO NOTHING;\nDO $$ BEGIN IF (SELECT count(*) FROM native_graph_records WHERE tenant_id=${tenant} AND knowledge_base_id IN (SELECT native_id FROM graph_kb_map) AND source_table=${quote(table)})<>${expected[table]} THEN RAISE EXCEPTION 'Graph count mismatch ${table}'; END IF; END $$;\nDROP TABLE graph_import_${table};\n`);
}
await write(`
DO $$ BEGIN IF EXISTS(SELECT 1 FROM native_graph_records WHERE tenant_id=${tenant} AND knowledge_base_id IN (SELECT native_id FROM graph_kb_map) AND source_table='knowledgegraphevidence' GROUP BY knowledge_base_id,payload->>'source_chunk_id' HAVING count(DISTINCT payload->>'source_content_hash')>1) THEN RAISE EXCEPTION 'Conflicting source content hashes'; END IF; END $$;
INSERT INTO native_graph_sources(tenant_id,knowledge_base_id,source_chunk_id,source_file,source_content_hash,excerpt)
SELECT DISTINCT ON (knowledge_base_id,payload->>'source_chunk_id') tenant_id,knowledge_base_id,payload->>'source_chunk_id',payload->>'source_file',payload->>'source_content_hash',payload->>'excerpt'
FROM native_graph_records WHERE tenant_id=${tenant} AND knowledge_base_id IN (SELECT native_id FROM graph_kb_map) AND source_table='knowledgegraphevidence' ORDER BY knowledge_base_id,payload->>'source_chunk_id',length(payload->>'excerpt') DESC,legacy_id ON CONFLICT DO NOTHING;
CREATE TEMP TABLE graph_native_chunks AS SELECT c.id,c.knowledge_id,c.knowledge_base_id,k.file_name,encode(sha256(convert_to(c.content,'UTF8')),'hex') AS content_hash FROM chunks c JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id AND k.deleted_at IS NULL WHERE c.tenant_id=${tenant} AND c.knowledge_base_id IN (SELECT native_id FROM graph_kb_map) AND c.deleted_at IS NULL AND c.is_enabled=true AND c.chunk_type='text';
CREATE INDEX ON graph_native_chunks(knowledge_base_id,content_hash);
ANALYZE graph_native_chunks;
UPDATE native_graph_sources s SET native_chunk_id=m.id,native_knowledge_id=m.knowledge_id,native_content_sha256=m.content_hash,mapping_state='exact_content_hash' FROM (SELECT * FROM (SELECT *,count(*) OVER (PARTITION BY knowledge_base_id,file_name,content_hash) AS matches FROM graph_native_chunks) candidates WHERE matches=1) m WHERE s.tenant_id=${tenant} AND s.knowledge_base_id=m.knowledge_base_id AND regexp_replace(s.source_file,'\\.jsonl$','.md')=m.file_name AND s.source_content_hash=m.content_hash AND s.native_chunk_id IS NULL;
COMMIT;
VACUUM (ANALYZE) native_graph_records, native_graph_sources;
SELECT source_table,count(*) AS preserved FROM native_graph_records WHERE tenant_id=${tenant} GROUP BY source_table ORDER BY source_table;
SELECT mapping_state,count(*) AS source_chunks FROM native_graph_sources WHERE tenant_id=${tenant} GROUP BY mapping_state;
`);
proc.stdin.end();
const [code] = await completion;
if(code!==0)throw new Error(`PostgreSQL import failed (${code}); transaction rolled back`);
console.log(JSON.stringify({snapshot_sha256:digest,tenant,database,expected}));
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env python3
"""Read-only vector copy audit and distinct-query acceptance latency sample.
Reports only counts, checksums, and timings; never knowledge prose or credentials.
"""
import hashlib
import json
import math
import statistics
import time
from pathlib import Path
from milvus_copy_acceptance import FIELDS, post
from milvus_verify_acceptance import (ACCEPTANCE, Client, VerificationError,
inspect_results, read_object, search_payload)
SOURCE = 'weknora_embeddings_512'
TARGET = 'yawan_acceptance_hybrid_512'
MILVUS = 'http://127.0.0.1:19530'
QUERIES = {
'1': ['产品资料应如何整理和解释', '介绍产品时需要说明哪些使用方法'],
'2': ['如何安排由浅入深的练习流程', '基础练习中怎样判断动作是否正确'],
'3': ['怎样通过沟通理解对方的需求', '遇到紧张情绪时如何调整表达方式'],
'4': ['如何描述亲密关系中的感受和期待', '怎样建立双方舒适的互动氛围'],
'5': ['学习新技巧时如何分解具体步骤', '练习过程有哪些常见错误需要纠正'],
'6': ['如何理解客户需求并解释产品价值', '怎样回应疑问并帮助客户作出选择'],
}
def query(collection, filter_text, fields, limit=100):
return post(MILVUS, 'entities/query', {
'collectionName': collection, 'filter': filter_text,
'outputFields': fields, 'limit': limit, 'consistencyLevel': 'Strong',
})
def digest(row):
return hashlib.sha256(json.dumps({field: row[field] for field in FIELDS},
sort_keys=True, ensure_ascii=False,
separators=(',', ':')).encode()).hexdigest()
def run():
state = read_object(ACCEPTANCE / 'legacy-migration-state.json', 'Migration state')
copy = read_object(ACCEPTANCE / 'hybrid-copy-state.json', 'Copy state')
manifest = json.loads((ACCEPTANCE / 'hybrid-chunk-manifest.json').read_text())
report = {'source': SOURCE, 'target': TARGET, 'storage': [], 'queries': [],
'scope': 'Six migrated business knowledge bases only',
'latency_scope': 'Sequential distinct queries, warmed service; cache not cleared or instrumented',
'started_at': time.time()}
business_ids = {kb['id'] for kb in state['knowledge_bases'].values()}
missing_ids = set(copy['missing'])
business_missing = [row['id'] for row in manifest
if row['id'] in missing_ids and row['knowledge_base_id'] in business_ids]
report['missing_business_chunks'] = len(business_missing)
report['missing_historical_test_chunks'] = len(missing_ids) - len(business_missing)
for legacy_id, kb in sorted(state['knowledge_bases'].items()):
entries = [row for row in manifest if row['knowledge_base_id'] == kb['id']]
filter_text = 'knowledge_base_id == ' + json.dumps(kb['id'])
counts = {name: int(query(name, filter_text, ['count(*)'])[0]['count(*)'])
for name in (SOURCE, TARGET)}
positions = sorted({0, len(entries) // 2, len(entries) - 1})
ids = [entries[index]['id'] for index in positions]
samples = {}
for name in (SOURCE, TARGET):
rows = query(name, 'chunk_id in ' + json.dumps(ids), FIELDS, len(ids))
samples[name] = {row['chunk_id']: digest(row) for row in rows}
equal = (set(samples[SOURCE]) == set(ids) and samples[SOURCE] == samples[TARGET])
report['storage'].append({
'legacy_kb': legacy_id, 'kb_id': kb['id'], 'expected_chunks': len(entries),
'source_count': counts[SOURCE], 'target_count': counts[TARGET],
'sample_count': len(ids), 'sample_fields_equal': equal,
'passed': equal and all(count == len(entries) for count in counts.values()),
})
credentials = read_object(ACCEPTANCE / 'migration-token.json', 'Migration credentials')
client = Client(45)
status, login, _ = client.request('/api/v1/auth/login', payload={
'email': credentials['email'], 'password': credentials['password'],
})
if status != 200 or login.get('active_tenant', {}).get('id') != state['tenant_id']:
raise VerificationError('Unexpected login scope')
for legacy_id, queries in sorted(QUERIES.items()):
kb_id = state['knowledge_bases'][legacy_id]['id']
for index, text in enumerate(queries):
status, result, latency = client.request(
'/api/v1/knowledge-bases/' + kb_id + '/hybrid-search', login['token'],
search_payload(text, 'hybrid', 10))
check = inspect_results(result, kb_id, 10, 'hybrid') if status == 200 else {'passed': False}
report['queries'].append({'legacy_kb': legacy_id, 'query_index': index,
'query_sha256': hashlib.sha256(text.encode()).hexdigest(),
'status': status, 'latency_ms': latency, **check})
latencies = sorted(row['latency_ms'] for row in report['queries'])
report['latency_ms'] = {'min': min(latencies), 'median': statistics.median(latencies),
'p95_nearest_rank': latencies[math.ceil(len(latencies) * .95) - 1],
'max': max(latencies), 'count': len(latencies)}
report['passed'] = (not business_missing and all(row['passed'] for row in report['storage'])
and all(row['passed'] for row in report['queries']))
report['finished_at'] = time.time()
output = ACCEPTANCE / 'milvus-storage-and-latency.json'
output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
print(json.dumps({key: report[key] for key in (
'passed', 'missing_business_chunks', 'missing_historical_test_chunks', 'latency_ms')}, indent=2))
return 0 if report['passed'] else 1
if __name__ == '__main__':
raise SystemExit(run())
+96
View File
@@ -0,0 +1,96 @@
#!/usr/bin/env python3
"""Copy existing vectors to an isolated BM25 collection, without re-embedding.
The manifest is an export of the acceptance DB's chunk IDs. Source collections
are read-only; acknowledgements and a manifest digest make retries resumable.
"""
import argparse
import hashlib
import json
import os
import time
import urllib.request
from pathlib import Path
FIELDS = ['id', 'embedding', 'content', 'source_id', 'source_type', 'chunk_id',
'knowledge_id', 'knowledge_base_id', 'tag_id', 'is_enabled']
def post(base, path, payload):
request = urllib.request.Request(base + '/v2/vectordb/' + path,
data=json.dumps(payload).encode(),
headers={'Content-Type': 'application/json'})
with urllib.request.urlopen(request, timeout=120) as response:
result = json.load(response)
if result.get('code') != 0:
raise RuntimeError(f"{path}: {result.get('message', result.get('code'))}")
return result.get('data')
def save(path, data):
temporary = path.with_suffix('.tmp')
temporary.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
os.replace(temporary, path)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--manifest', type=Path, required=True)
parser.add_argument('--state', type=Path, required=True)
parser.add_argument('--target', default='yawan_acceptance_hybrid_512')
parser.add_argument('--source', default='weknora_embeddings_512')
args = parser.parse_args()
if not args.target.startswith('yawan_acceptance_') or args.target == args.source:
parser.error('target must be a distinct acceptance collection')
base = 'http://127.0.0.1:19530'
raw = args.manifest.read_bytes()
manifest = json.loads(raw)
digest = hashlib.sha256(raw).hexdigest()
state = json.loads(args.state.read_text()) if args.state.exists() else {
'manifest_sha256': digest, 'source': args.source, 'target': args.target,
'offset': 0, 'copied': 0, 'missing': [], 'started_at': time.time(),
}
for key, value in [('manifest_sha256', digest), ('source', args.source), ('target', args.target)]:
if state[key] != value:
raise RuntimeError(f'{key} differs from checkpoint')
description = post(base, 'collections/describe', {'collectionName': args.target})
if 'content_sparse' not in {f['name'] for f in description['fields']}:
raise RuntimeError('target must have BM25 schema before copying')
for offset in range(state['offset'], len(manifest), 256):
batch = manifest[offset:offset + 256]
expected = {x['id']: x for x in batch}
records = post(base, 'entities/query', {
'collectionName': args.source,
'filter': 'chunk_id in ' + json.dumps(list(expected)),
'outputFields': FIELDS, 'limit': len(batch) * 2,
'consistencyLevel': 'Strong',
})
found = set()
for row in records:
entry = expected.get(row['chunk_id'])
if not entry or row['knowledge_base_id'] != entry['knowledge_base_id'] or row['knowledge_id'] != entry['knowledge_id']:
raise RuntimeError('source scope differs from acceptance manifest')
if row['chunk_id'] in found:
raise RuntimeError('duplicate source vectors for chunk')
found.add(row['chunk_id'])
if records:
result = post(base, 'entities/upsert', {'collectionName': args.target, 'data': records})
if result.get('upsertCount') != len(records):
raise RuntimeError('target did not acknowledge every vector')
state['missing'].extend(sorted(expected.keys() - found))
state['copied'] += len(records)
state['offset'] = offset + len(batch)
state['updated_at'] = time.time()
save(args.state, state)
if offset % (256 * 20) == 0 or state['offset'] == len(manifest):
print(json.dumps({'processed': state['offset'], 'total': len(manifest),
'copied': state['copied'], 'missing': len(state['missing'])}), flush=True)
state['complete'] = not state['missing']
save(args.state, state)
print(json.dumps({'complete': state['complete'], 'copied': state['copied'],
'missing': len(state['missing'])}), flush=True)
return 0 if state['complete'] else 1
if __name__ == '__main__':
raise SystemExit(main())
+203
View File
@@ -0,0 +1,203 @@
#!/usr/bin/env python3
"""Upload one temporary acceptance document, verify retrieval, then delete it.
Service mutations use only 127.0.0.1:18080 and the KB created by this run.
Milvus access, when enabled, is restricted to read-only entities/query calls.
"""
import argparse
import json
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
from milvus_verify_acceptance import (ACCEPTANCE, BASE, MODES, NoRedirect,
VerificationError, inspect_results,
read_object, search_payload)
class SmokeClient:
def __init__(self):
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect())
self.token = ''
self.kb_id = None
self.knowledge_id = None
def request(self, route, method='GET', payload=None, raw=None, content_type=None):
kb = '/api/v1/knowledge-bases/' + str(self.kb_id)
knowledge = '/api/v1/knowledge/' + str(self.knowledge_id)
allowed = {('POST', '/api/v1/auth/login'), ('POST', '/api/v1/knowledge-bases')}
if self.kb_id:
allowed.update({('POST', kb + '/knowledge/file'), ('POST', kb + '/hybrid-search'),
('DELETE', kb), ('GET', kb)})
if self.knowledge_id:
allowed.add(('GET', knowledge))
if (method, route) not in allowed:
raise VerificationError('Request is outside the temporary resource allowlist')
headers = {'Accept': 'application/json'}
if self.token:
headers['Authorization'] = 'Bearer ' + self.token
if payload is not None:
raw = json.dumps(payload).encode()
content_type = 'application/json'
if content_type:
headers['Content-Type'] = content_type
started = time.monotonic()
request = urllib.request.Request(BASE + route, data=raw, headers=headers, method=method)
try:
try:
response = self.opener.open(request, timeout=45)
except urllib.error.HTTPError as error:
response = error
with response:
status = response.code
body = response.read(8 * 1024 * 1024)
value = json.loads(body) if status in (200, 201, 202) else None
except (OSError, ValueError):
raise VerificationError('Acceptance request failed or timed out') from None
return status, value, round((time.monotonic() - started) * 1000, 2)
def count_vectors(self, collection):
if collection not in ('yawan_acceptance_hybrid_512', 'weknora_embeddings_512') or not self.kb_id:
raise VerificationError('Milvus query is outside the allowed collections')
request = urllib.request.Request('http://127.0.0.1:19530/v2/vectordb/entities/query',
data=json.dumps({'collectionName': collection,
'filter': 'knowledge_base_id == ' + json.dumps(self.kb_id),
'outputFields': ['count(*)'], 'consistencyLevel': 'Strong'}).encode(),
headers={'Content-Type': 'application/json'})
try:
with self.opener.open(request, timeout=30) as response:
value = json.load(response)
if value.get('code') != 0:
raise VerificationError('Milvus count query failed')
return int(value['data'][0]['count(*)'])
except (OSError, ValueError, KeyError, IndexError):
raise VerificationError('Milvus count query failed or returned an invalid shape') from None
def run(args, report):
credentials = read_object(ACCEPTANCE / 'migration-token.json', 'Migration credentials')
state = read_object(ACCEPTANCE / 'legacy-migration-state.json', 'Migration state')
client = SmokeClient()
status, login, _ = client.request('/api/v1/auth/login', 'POST',
{'email': credentials['email'], 'password': credentials['password']})
if status != 200 or login.get('active_tenant', {}).get('id') != state['tenant_id']:
raise VerificationError('Login failed or used an unexpected tenant')
client.token = login['token']
suffix = uuid.uuid4().hex[:12]
kb_name = '临时验收-新上传检索-' + suffix
try:
status, value, latency = client.request('/api/v1/knowledge-bases', 'POST', {
'name': kb_name, 'description': 'Temporary acceptance-only upload smoke test',
'type': 'document', 'embedding_model_id': state['model_id'],
'summary_model_id': '',
'chunking_config': {'chunk_size': 512, 'chunk_overlap': 0},
'indexing_strategy': {'vector_enabled': True, 'keyword_enabled': True,
'wiki_enabled': False, 'graph_enabled': False},
'storage_provider_config': {'provider': 'local'},
'auto_tag_config': {'enabled': False},
})
if status != 201 or not isinstance(value.get('data'), dict):
raise VerificationError('Temporary knowledge base creation failed')
client.kb_id = value['data']['id']
report.update(kb_id=client.kb_id, create_ms=latency)
migrated_ids = {row['id'] for row in state['knowledge_bases'].values()}
if client.kb_id in migrated_ids:
client.kb_id = None
raise VerificationError('Created KB unexpectedly matched protected migration IDs')
if args.verify_milvus:
report['vectors_before'] = {name: client.count_vectors(name) for name in
('yawan_acceptance_hybrid_512', 'weknora_embeddings_512')}
if any(report['vectors_before'].values()):
raise VerificationError('Fresh temporary KB unexpectedly already has vectors')
boundary = 'acceptance' + uuid.uuid4().hex
document = ('# 资料归档演练\n\n编号:' + suffix + '\n\n'
'资料归档采用三步流程:先登记文件名称,再核对目录分类,最后确认文件可以检索。'
'检索测试使用资料归档和目录核对两个关键词。完成演练后删除临时资料,不影响其他知识库。\n')
raw = ('--' + boundary + '\r\nContent-Disposition: form-data; name="file"; '
'filename="acceptance-upload-' + suffix + '.md"\r\n'
'Content-Type: text/markdown; charset=utf-8\r\n\r\n' + document +
'\r\n--' + boundary + '--\r\n').encode('utf-8')
status, value, latency = client.request('/api/v1/knowledge-bases/' + client.kb_id + '/knowledge/file',
'POST', raw=raw, content_type='multipart/form-data; boundary=' + boundary)
if status not in (200, 201) or not isinstance(value.get('data'), dict):
raise VerificationError('Temporary file upload failed')
client.knowledge_id = value['data']['id']
report.update(knowledge_id=client.knowledge_id, upload_ms=latency, file_bytes=len(document.encode('utf-8')))
started = time.monotonic()
while time.monotonic() - started < args.index_timeout:
status, value, _ = client.request('/api/v1/knowledge/' + client.knowledge_id)
parse_status = value.get('data', {}).get('parse_status') if isinstance(value, dict) else None
if parse_status == 'completed':
break
if parse_status in ('failed', 'cancelled', 'deleting') or status != 200:
raise VerificationError('Temporary file indexing failed')
time.sleep(2)
else:
raise VerificationError('Temporary file indexing exceeded its deadline')
report['indexing_seconds'] = round(time.monotonic() - started, 2)
for mode in MODES:
status, value, latency = client.request('/api/v1/knowledge-bases/' + client.kb_id + '/hybrid-search',
'POST', search_payload('资料归档 目录核对', mode, 5))
result = inspect_results(value, client.kb_id, 5, mode) if status == 200 else {'passed': False}
result.update(mode=mode, status=status, latency_ms=latency)
report['retrieval'].append(result)
if args.verify_milvus:
report['vectors_after_upload'] = {name: client.count_vectors(name) for name in
('yawan_acceptance_hybrid_512', 'weknora_embeddings_512')}
report['isolated_vector_write'] = (report['vectors_after_upload']['yawan_acceptance_hybrid_512'] > 0
and report['vectors_after_upload']['weknora_embeddings_512'] == 0)
finally:
if client.kb_id:
status, _, latency = client.request('/api/v1/knowledge-bases/' + client.kb_id, 'DELETE')
report['cleanup'] = {'delete_status': status, 'delete_ms': latency, 'passed': False}
if status not in (200, 202, 204):
raise VerificationError('Temporary KB cleanup request failed')
status, _, _ = client.request('/api/v1/knowledge-bases/' + client.kb_id)
report['cleanup']['kb_get_status'] = status
report['cleanup']['passed'] = status == 404
if args.verify_milvus:
started = time.monotonic()
while time.monotonic() - started < 90:
counts = {name: client.count_vectors(name) for name in
('yawan_acceptance_hybrid_512', 'weknora_embeddings_512')}
if all(count == 0 for count in counts.values()):
break
time.sleep(2)
report['cleanup'].update(vector_counts=counts,
vector_cleanup_seconds=round(time.monotonic() - started, 2))
report['cleanup']['passed'] = report['cleanup']['passed'] and all(count == 0 for count in counts.values())
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--verify-milvus', action='store_true', help='Read counts for this temporary KB from acceptance and production collections')
parser.add_argument('--output', type=Path)
parser.add_argument('--index-timeout', type=int, default=180)
args = parser.parse_args()
if not 10 <= args.index_timeout <= 600:
parser.error('index-timeout must be 10-600 seconds')
if args.output and args.output.resolve() in {file.resolve() for file in
(ACCEPTANCE / 'migration-token.json', ACCEPTANCE / 'legacy-migration-state.json', ACCEPTANCE / 'browser-auth.json')}:
parser.error('Output must not overwrite existing authentication or migration state')
report = {'acceptance_url': BASE, 'retrieval': [], 'failures': []}
try:
run(args, report)
except VerificationError as error:
report['failures'].append(str(error))
except Exception:
report['failures'].append('Unexpected smoke-test failure; no server error or credentials included')
report['passed'] = (not report['failures'] and len(report['retrieval']) == 3
and all(row['passed'] for row in report['retrieval'])
and report.get('cleanup', {}).get('passed', False)
and (not args.verify_milvus or report.get('isolated_vector_write', False)))
text = json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + '\n'
if args.output:
args.output.write_text(text, encoding='utf-8')
print(text, end='')
return 0 if report['passed'] else 1
if __name__ == '__main__':
raise SystemExit(main())
+274
View File
@@ -0,0 +1,274 @@
#!/usr/bin/env python3
"""Verify retrieval in the isolated acceptance API without changing its data.
Only login and read-only search POSTs are allowed. Credentials and response
prose are never included in the report. No files are written unless --output
is supplied, and credential files are never refreshed or overwritten.
"""
import argparse
import datetime
import json
import math
import re
import time
import urllib.error
import urllib.request
from pathlib import Path
BASE = 'http://127.0.0.1:18080'
ACCEPTANCE = Path(__file__).resolve().parents[2] / 'acceptance'
UUID = re.compile(r'^[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$')
MODES = {
'keyword-only': (False, True),
'vector-only': (True, False),
'hybrid': (False, False),
}
QUERIES = {
'1': '产品 使用 说明',
'2': '练习 方法 步骤',
'3': '沟通 情绪 理解',
'4': '关系 感受 表达',
'5': '技巧 步骤 学习',
'6': '需求 价值 选择',
}
class VerificationError(Exception):
"""A fixed, credential-free error suitable for the report."""
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
class Client:
def __init__(self, timeout):
self.timeout = timeout
self.opener = urllib.request.build_opener(
urllib.request.ProxyHandler({}), NoRedirect())
def request(self, route, token=None, payload=None):
search_route = re.fullmatch(
r'/api/v1/knowledge-bases/[0-9a-fA-F-]+/hybrid-search', route)
allowed_read = route in ('/health', '/api/v1/auth/me')
allowed_login = route == '/api/v1/auth/login'
if not ((payload is None and allowed_read) or
(payload is not None and (search_route or allowed_login))):
raise VerificationError('Request outside the read-only acceptance allowlist')
headers = {'Accept': 'application/json'}
if token:
headers['Authorization'] = 'Bearer ' + token
body = None
if payload is not None:
headers['Content-Type'] = 'application/json'
body = json.dumps(payload).encode('utf-8')
request = urllib.request.Request(BASE + route, data=body, headers=headers)
started = time.monotonic()
try:
try:
response = self.opener.open(request, timeout=self.timeout)
except urllib.error.HTTPError as error:
response = error
with response:
status = response.code
raw = response.read(8 * 1024 * 1024 + 1)
if len(raw) > 8 * 1024 * 1024:
raise VerificationError('Response exceeded the verification size limit')
# Denied requests need only a status; never retain their error prose.
value = json.loads(raw) if status == 200 else None
return status, value, round((time.monotonic() - started) * 1000, 2)
except VerificationError:
raise
except (OSError, urllib.error.URLError, ValueError):
raise VerificationError('Request failed, timed out, or returned invalid JSON') from None
def read_object(file, label):
try:
value = json.loads(file.read_text(encoding='utf-8'))
except (OSError, ValueError):
raise VerificationError(label + ' could not be read as JSON') from None
if not isinstance(value, dict):
raise VerificationError(label + ' must contain a JSON object')
return value
def search_payload(query, mode, count):
disable_keywords, disable_vector = MODES[mode]
return {
'query_text': query,
'match_count': count,
'vector_threshold': 0,
'keyword_threshold': 0,
'disable_keywords_match': disable_keywords,
'disable_vector_match': disable_vector,
'skip_context_enrichment': True,
}
def inspect_results(value, kb_id, count, mode):
if not isinstance(value, dict) or value.get('success') is not True:
return {'passed': False, 'failure': 'Invalid search response envelope'}
rows = value.get('data') or []
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
return {'passed': False, 'failure': 'Invalid search results shape'}
scores = [row.get('score') for row in rows]
finite = all(isinstance(score, (int, float)) and not isinstance(score, bool)
and math.isfinite(score) for score in scores)
descending = finite and all(left + 1e-9 >= right
for left, right in zip(scores, scores[1:]))
in_scope = all(row.get('knowledge_base_id') == kb_id for row in rows)
ids = [row.get('id') for row in rows]
unique = all(isinstance(item, str) and item for item in ids) and len(set(ids)) == len(ids)
# Match types 0 and 1 are vector and keyword hits respectively.
expected = {'keyword-only': {1}, 'vector-only': {0}, 'hybrid': {0, 1}}[mode]
correct_mode = all(type(row.get('match_type')) is int and row['match_type'] in expected for row in rows)
return {
'passed': bool(rows) and len(rows) <= count and finite and descending and in_scope and unique and correct_mode,
'count': len(rows),
'nonempty': bool(rows),
'within_requested_count': len(rows) <= count,
'scores': [round(score, 8) for score in scores] if finite else [],
'scores_finite': finite,
'scores_descending': descending,
'kb_scope_valid': in_scope,
'unique_chunk_ids': unique,
'match_types_valid': correct_mode,
}
def verify(args, report):
credentials = read_object(args.credentials, 'Migration credentials')
migration = read_object(args.state, 'Migration state')
foreign = read_object(args.foreign_auth, 'Foreign tenant authentication')
entries = migration.get('knowledge_bases', {})
if not isinstance(entries, dict) or set(entries) != set(QUERIES):
raise VerificationError('Migration state must describe exactly the six expected knowledge bases')
kb_ids = []
for legacy_id, row in entries.items():
if not isinstance(row, dict) or not UUID.fullmatch(str(row.get('id', ''))):
raise VerificationError('Migration state contains an invalid knowledge base ID')
kb_ids.append(row['id'])
if len(set(kb_ids)) != 6:
raise VerificationError('Migration state contains duplicate knowledge base IDs')
if not all(isinstance(credentials.get(key), str) and credentials[key] for key in ('email', 'password')):
raise VerificationError('Migration login credentials are incomplete')
if not isinstance(foreign.get('token'), str) or not foreign['token']:
raise VerificationError('Existing foreign tenant token is required')
client = Client(args.timeout)
status, _, latency = client.request('/health')
report['checks'].append({'check': 'acceptance_health', 'status': status,
'latency_ms': latency, 'passed': status == 200})
if status != 200:
raise VerificationError('Acceptance API is not healthy')
status, login, latency = client.request('/api/v1/auth/login', payload={
'email': credentials['email'], 'password': credentials['password'],
})
if status != 200 or not isinstance(login, dict) or not isinstance(login.get('token'), str):
raise VerificationError('Migration login failed')
token = login['token']
tenant_id = login.get('active_tenant', {}).get('id')
correct_tenant = tenant_id is not None and tenant_id == migration.get('tenant_id') == credentials.get('tenant_id')
report['checks'].append({'check': 'migration_login_scope', 'status': status,
'latency_ms': latency, 'passed': correct_tenant})
if not correct_tenant:
raise VerificationError('Migration login is outside the expected tenant')
status, identity, latency = client.request('/api/v1/auth/me', foreign['token'])
details = identity.get('data', {}) if isinstance(identity, dict) else {}
foreign_tenant = (details.get('tenant') or {}).get('id')
user = details.get('user') or {}
memberships = details.get('memberships') or []
isolated = (status == 200 and foreign_tenant is not None and foreign_tenant != tenant_id
and not user.get('is_system_admin') and not user.get('can_access_all_tenants')
and not any(member.get('tenant_id') == tenant_id for member in memberships if isinstance(member, dict)))
report['checks'].append({'check': 'foreign_identity_valid_and_isolated', 'status': status,
'latency_ms': latency, 'passed': isolated})
if not isolated:
report['failures'].append('Foreign authentication is expired, invalid, or not isolated; denial tests cannot prove tenant isolation')
for legacy_id in sorted(entries):
kb_id = entries[legacy_id]['id']
route = '/api/v1/knowledge-bases/' + kb_id + '/hybrid-search'
payload = search_payload(QUERIES[legacy_id], 'keyword-only', args.top_k)
for check, auth, enabled, statuses in (
('unauthenticated_denied', None, True, (401, 403)),
('foreign_tenant_denied', foreign['token'], isolated, (403, 404)),
):
if not enabled:
continue
result = {'check': check, 'legacy_kb': legacy_id, 'kb_id': kb_id}
try:
status, _, latency = client.request(route, auth, payload)
result.update(status=status, latency_ms=latency, passed=status in statuses)
except VerificationError as error:
result.update(passed=False, failure=str(error))
report['checks'].append(result)
for mode in MODES:
result = {'legacy_kb': legacy_id, 'kb_id': kb_id, 'mode': mode,
'query': QUERIES[legacy_id], 'top_k': args.top_k}
try:
status, value, latency = client.request(route, token, search_payload(QUERIES[legacy_id], mode, args.top_k))
result.update(status=status, latency_ms=latency)
result.update(inspect_results(value, kb_id, args.top_k, mode) if status == 200
else {'passed': False, 'failure': 'Search HTTP status was not 200'})
except VerificationError as error:
result.update(passed=False, failure=str(error))
report['retrieval'].append(result)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--credentials', type=Path, default=ACCEPTANCE / 'migration-token.json')
parser.add_argument('--state', type=Path, default=ACCEPTANCE / 'legacy-migration-state.json')
parser.add_argument('--foreign-auth', type=Path, default=ACCEPTANCE / 'browser-auth.json')
parser.add_argument('--output', type=Path, help='Write the same JSON report to this file only')
parser.add_argument('--timeout', type=float, default=45, help='Timeout in seconds for every HTTP request (1-180)')
parser.add_argument('--top-k', type=int, default=10, help='Requested primary matches (1-50)')
args = parser.parse_args()
if not 1 <= args.timeout <= 180 or not 1 <= args.top_k <= 50:
parser.error('timeout must be 1-180 seconds and top-k must be 1-50')
if args.output and args.output.resolve() in {file.resolve() for file in (args.credentials, args.state, args.foreign_auth)}:
parser.error('output must not overwrite authentication or migration state')
started = time.monotonic()
report = {
'acceptance_url': BASE,
'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
'checks': [], 'retrieval': [], 'failures': [],
'limitations': ['API validation does not independently attest the configured backend or collection',
'Nonempty and ordered matches are smoke checks, not a relevance-quality benchmark'],
}
try:
verify(args, report)
except VerificationError as error:
report['failures'].append(str(error))
except Exception:
# Do not expose response contents, credentials, or traceback locals.
report['failures'].append('Unexpected verification failure; inspect the verifier and response shapes locally')
all_checks = report['checks'] + report['retrieval']
report['passed'] = (not report['failures'] and len(report['retrieval']) == 18
and all(item.get('passed') for item in all_checks))
report['duration_seconds'] = round(time.monotonic() - started, 2)
report['summary'] = {'checks': len(all_checks),
'passed': sum(bool(item.get('passed')) for item in all_checks),
'failed': sum(not item.get('passed') for item in all_checks)}
encoded = json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + '\n'
if args.output:
try:
args.output.write_text(encoded, encoding='utf-8')
except OSError:
report['passed'] = False
report['failures'].append('Report output could not be written')
encoded = json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + '\n'
print(encoded, end='')
return 0 if report['passed'] else 1
if __name__ == '__main__':
raise SystemExit(main())
+835
View File
@@ -0,0 +1,835 @@
// Run explicitly by filename, like scripts/wiki_backfill_trigger.go.
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"net"
"net/url"
"os"
"os/exec"
"os/signal"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
"github.com/hibiken/asynq"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"gorm.io/gorm/logger"
)
type graphRecoveryCandidate struct {
ID string `json:"chunk_id"`
KnowledgeID string `json:"knowledge_id"`
SeqID int64 `json:"seq_id"`
ChunkIndex int `json:"chunk_index"`
ContentHash string `json:"content_sha256"`
}
// Failure records survive Redis retention and watcher restarts. Raw provider
// errors are deliberately excluded: state and stable identities are sufficient
// to find the task's private diagnostics and authorize a bounded manual retry.
type graphRecoveryFailure struct {
TaskID string `gorm:"primaryKey" json:"task_id"`
TenantID uint64 `gorm:"index:idx_graph_backfill_scope;index:idx_graph_backfill_candidate,priority:1" json:"tenant_id"`
KnowledgeBaseID string `gorm:"index:idx_graph_backfill_scope;index:idx_graph_backfill_candidate,priority:2" json:"knowledge_base_id"`
ChunkID string `gorm:"index:idx_graph_backfill_candidate,priority:3" json:"chunk_id"`
KnowledgeID string `json:"knowledge_id"`
ContentHash string `gorm:"index:idx_graph_backfill_candidate,priority:4" json:"content_sha256"`
Revision string `gorm:"index:idx_graph_backfill_candidate,priority:5" json:"revision"`
State string `gorm:"index:idx_graph_backfill_candidate,priority:6" json:"state"`
Reason string `json:"reason"`
ManualRetries int `json:"manual_retries"`
RetryReason string `json:"retry_reason,omitempty"`
FirstFailedAt time.Time `json:"first_failed_at"`
UpdatedAt time.Time `json:"updated_at"`
}
func (graphRecoveryFailure) TableName() string { return "native_graph_backfill_failures" }
func graphFailureFilter(includeFailed bool) string {
if includeFailed {
return ""
}
return ` AND NOT EXISTS(SELECT 1 FROM native_graph_backfill_failures f WHERE f.tenant_id=c.tenant_id AND f.knowledge_base_id=c.knowledge_base_id AND f.chunk_id=c.id AND f.content_hash=encode(sha256(convert_to(c.content,'UTF8')),'hex') AND f.revision=? AND f.state='failed')`
}
const graphCurrentFailureSource = `EXISTS(SELECT 1 FROM chunks c JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id AND k.deleted_at IS NULL WHERE c.tenant_id=f.tenant_id AND c.knowledge_base_id=f.knowledge_base_id AND c.id=f.chunk_id AND c.knowledge_id=f.knowledge_id AND c.deleted_at IS NULL AND c.is_enabled AND c.chunk_type='text' AND encode(sha256(convert_to(c.content,'UTF8')),'hex')=f.content_hash)`
const graphFailureCheckpoint = `EXISTS(SELECT 1 FROM native_graph_records r WHERE r.tenant_id=f.tenant_id AND r.knowledge_base_id=f.knowledge_base_id AND r.source_table='knowledgegraphchunkstate' AND r.payload->>'origin'='native_extraction_v1' AND r.payload->>'extraction_status'='complete' AND CAST(COALESCE(r.payload->>'relation_count','0') AS INTEGER)>0 AND r.payload->>'native_chunk_id'=f.chunk_id AND r.payload->>'native_knowledge_id'=f.knowledge_id AND r.payload->>'source_content_hash'=f.content_hash)`
func graphUnresolvedFailures(db *gorm.DB, tenant uint64, kbID, revision string, reconcile bool) ([]graphRecoveryFailure, error) {
if reconcile {
// Keep historical rows, but a deleted/changed source or a newly verified
// checkpoint must not leave a completed knowledge base blocked forever.
for _, resolution := range []struct{ state, condition string }{
{"superseded", "NOT " + graphCurrentFailureSource},
{"resolved", graphCurrentFailureSource + " AND " + graphFailureCheckpoint},
} {
if err := db.Exec("UPDATE native_graph_backfill_failures AS f SET state=?,updated_at=? WHERE tenant_id=? AND knowledge_base_id=? AND revision=? AND state NOT IN ('resolved','superseded') AND "+resolution.condition, resolution.state, time.Now().UTC(), tenant, kbID, revision).Error; err != nil {
return nil, err
}
}
}
var failures []graphRecoveryFailure
err := db.Table("native_graph_backfill_failures AS f").Where("f.tenant_id=? AND f.knowledge_base_id=? AND f.revision=? AND f.state='failed'", tenant, kbID, revision).Where(graphCurrentFailureSource).Where("NOT " + graphFailureCheckpoint).Order("f.first_failed_at,f.task_id").Find(&failures).Error
return failures, err
}
func graphRecoveryCandidates(db *gorm.DB, tenant uint64, kbID, revision string, after int64, limit int, hasFailures, includeFailed bool, exact ...graphRecoveryCandidate) ([]graphRecoveryCandidate, error) {
var candidates []graphRecoveryCandidate
filter := ""
queryArgs := []any{tenant, kbID, tenant, kbID, after}
if hasFailures && !includeFailed {
filter = graphFailureFilter(false)
queryArgs = append(queryArgs, revision)
}
if len(exact) > 0 {
filter += " AND c.id=? AND c.knowledge_id=? AND encode(sha256(convert_to(c.content,'UTF8')),'hex')=?"
queryArgs = append(queryArgs, exact[0].ID, exact[0].KnowledgeID, exact[0].ContentHash)
}
queryArgs = append(queryArgs, limit)
err := db.Raw(`WITH completed AS NOT MATERIALIZED(SELECT payload FROM native_graph_records WHERE tenant_id=? AND knowledge_base_id=? AND source_table='knowledgegraphchunkstate' AND payload->>'origin'='native_extraction_v1' AND payload->>'extraction_status'='complete' AND CAST(COALESCE(payload->>'relation_count','0') AS INTEGER)>0)
SELECT c.id,c.knowledge_id,c.seq_id,c.chunk_index,encode(sha256(convert_to(c.content,'UTF8')),'hex') AS content_hash
FROM chunks c JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id AND k.deleted_at IS NULL
WHERE c.tenant_id=? AND c.knowledge_base_id=? AND c.deleted_at IS NULL AND c.is_enabled AND c.chunk_type='text' AND c.seq_id>? AND k.parse_status='completed'
AND NOT EXISTS(SELECT 1 FROM completed r WHERE r.payload->>'native_chunk_id'=c.id AND r.payload->>'native_knowledge_id'=c.knowledge_id AND r.payload->>'source_content_hash'=encode(sha256(convert_to(c.content,'UTF8')),'hex'))
`+filter+` ORDER BY c.seq_id LIMIT ?`, queryArgs...).Scan(&candidates).Error
if err == nil && len(exact) > 0 && len(candidates) != 1 {
return nil, errors.New("exact graph source unavailable, changed, completed or excluded by its failure ledger; no substitute was selected")
}
return candidates, err
}
func graphValidateExactScope(chunkID, knowledgeID, hash string, after int64, limit, maxOutstanding int) error {
if chunkID == "" && knowledgeID == "" && hash == "" {
return nil
}
chunk, chunkErr := uuid.Parse(chunkID)
knowledge, knowledgeErr := uuid.Parse(knowledgeID)
decoded, hashErr := hex.DecodeString(hash)
if chunkErr != nil || knowledgeErr != nil || chunk == uuid.Nil || knowledge == uuid.Nil ||
chunk.String() != chunkID || knowledge.String() != knowledgeID ||
hashErr != nil || len(decoded) != sha256.Size || hex.EncodeToString(decoded) != hash ||
after != 0 || limit != 1 || maxOutstanding != 1 {
return errors.New("exact graph scope requires canonical chunk and knowledge UUIDs, lowercase source SHA-256, limit=1, max-outstanding=1 and no cursor")
}
return nil
}
func graphTaskMatchesReviewedSource(payload []byte, tenant uint64, kbID string, c graphRecoveryCandidate) bool {
var task types.ExtractChunkPayload
return json.Unmarshal(payload, &task) == nil && task.TenantID == tenant && task.ChunkID == c.ID &&
(task.KnowledgeID == "" || task.KnowledgeID == c.KnowledgeID) &&
task.ExpectedSource != nil && task.ExpectedSource.KnowledgeBaseID == kbID &&
task.ExpectedSource.KnowledgeID == c.KnowledgeID && task.ExpectedSource.ContentSHA256 == c.ContentHash
}
type graphRecoveryQueue interface {
RunTask(string, string) error
}
func graphQueueSlots(info *asynq.QueueInfo, limit int) int {
if info == nil || info.Paused {
return 0
}
remaining := limit - info.Active - info.Pending - info.Scheduled - info.Retry - info.Aggregating
if remaining < 0 {
return 0
}
return remaining
}
func graphResumeCursor(after int64, candidates []graphRecoveryCandidate, settled map[string]bool) int64 {
for _, c := range candidates {
if !settled[c.ID] {
break
}
after = c.SeqID
}
return after
}
func graphRecoveryTaskID(tenant uint64, kbID string, candidate graphRecoveryCandidate, revision string) string {
identity := sha256.Sum256([]byte(fmt.Sprintf("%d:%s:%s:%s:%s", tenant, kbID, candidate.ID, candidate.ContentHash, revision)))
return "native-graph-" + hex.EncodeToString(identity[:])
}
func recordGraphFailure(db *gorm.DB, tenant uint64, kbID, revision, taskID string, c graphRecoveryCandidate, reason string) error {
var failure graphRecoveryFailure
err := db.First(&failure, "task_id=?", taskID).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
failure = graphRecoveryFailure{TaskID: taskID, TenantID: tenant, KnowledgeBaseID: kbID, ChunkID: c.ID, KnowledgeID: c.KnowledgeID, ContentHash: c.ContentHash, Revision: revision, FirstFailedAt: time.Now().UTC()}
} else if err != nil {
return err
}
if failure.FirstFailedAt.IsZero() {
failure.FirstFailedAt = time.Now().UTC()
}
failure.State, failure.Reason, failure.UpdatedAt = "failed", reason, time.Now().UTC()
return db.Save(&failure).Error
}
func reserveGraphDispatch(db *gorm.DB, tenant uint64, kbID, revision, taskID string, c graphRecoveryCandidate) (bool, error) {
var prior graphRecoveryFailure
err := db.First(&prior, "task_id=?", taskID).Error
if err == nil {
// An absent Redis task is not permission to repeat an earlier attempt.
return false, recordGraphFailure(db, tenant, kbID, revision, taskID, c, "task_missing_after_dispatch")
}
if !errors.Is(err, gorm.ErrRecordNotFound) {
return false, err
}
now := time.Now().UTC()
entry := graphRecoveryFailure{TaskID: taskID, TenantID: tenant, KnowledgeBaseID: kbID, ChunkID: c.ID, KnowledgeID: c.KnowledgeID, ContentHash: c.ContentHash, Revision: revision, State: "dispatch_requested", UpdatedAt: now}
if err := db.Create(&entry).Error; err != nil {
return false, err
}
return true, nil
}
func trackGraphTask(db *gorm.DB, tenant uint64, kbID, revision, taskID string, c graphRecoveryCandidate) error {
entry := graphRecoveryFailure{TaskID: taskID, TenantID: tenant, KnowledgeBaseID: kbID, ChunkID: c.ID, KnowledgeID: c.KnowledgeID, ContentHash: c.ContentHash, Revision: revision, State: "dispatch_requested", UpdatedAt: time.Now().UTC()}
return db.Clauses(clause.OnConflict{DoNothing: true}).Create(&entry).Error
}
func graphArchivedReason(lastError string) string {
message := strings.ToLower(lastError)
if strings.Contains(message, "graph_needs_review") {
return "relation_coverage_review_required"
}
if strings.Contains(message, "read response") && (strings.Contains(message, "context deadline exceeded") || strings.Contains(message, "timeout")) {
return "provider_response_timeout"
}
if strings.Contains(message, "context deadline exceeded") || strings.Contains(message, "deadline exceeded") || strings.Contains(message, "timeout") {
return "task_timeout"
}
if strings.Contains(message, "context canceled") || strings.Contains(message, "context cancelled") {
return "task_cancelled"
}
return "archived"
}
// Reconciliation receives only the Inspector read API, never a queue client.
type graphTaskReader interface {
GetTaskInfo(string, string) (*asynq.TaskInfo, error)
}
type graphReconcileReport struct {
Mode string `json:"mode"`
Apply bool `json:"apply"`
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
Revision string `json:"revision"`
Source graphRecoveryCandidate `json:"source"`
TaskID string `json:"task_id"`
QueueState string `json:"queue_state"`
LedgerState string `json:"ledger_state"`
ProposedState string `json:"proposed_state"`
SourceCurrent bool `json:"source_current"`
CheckpointComplete bool `json:"checkpoint_complete"`
Reason string `json:"reason"`
Blocked bool `json:"blocked"`
WouldUpdate bool `json:"would_update"`
LedgerUpdated bool `json:"ledger_updated"`
Enqueued int `json:"enqueued"`
Retried int `json:"retried"`
ModelCalls int `json:"model_calls"`
}
const graphReconcileCurrentSource = graphCurrentFailureSource + ` AND EXISTS(SELECT 1 FROM knowledges k JOIN knowledge_bases b ON b.id=k.knowledge_base_id AND b.tenant_id=k.tenant_id WHERE k.id=f.knowledge_id AND k.tenant_id=f.tenant_id AND k.knowledge_base_id=f.knowledge_base_id AND k.deleted_at IS NULL AND k.parse_status='completed' AND b.deleted_at IS NULL)`
func graphValidateReconciliation(reconcile bool, chunkID, revision string, retry bool, retryReason string) error {
if reconcile && (chunkID == "" || strings.TrimSpace(revision) == "" || retry || retryReason != "") {
return errors.New("reconcile-only requires one exact source, GRAPH_BACKFILL_REVISION and no retry flags")
}
return nil
}
func graphReconcileSourceStatus(db *gorm.DB, taskID string) (current, complete bool, err error) {
var status struct {
Current, Complete bool
}
result := db.Raw("SELECT ("+graphReconcileCurrentSource+") AS current,("+graphFailureCheckpoint+") AS complete FROM native_graph_backfill_failures f WHERE f.task_id=?", taskID).Scan(&status)
if result.Error != nil {
return false, false, errors.New("cannot validate graph source and checkpoint")
}
if result.RowsAffected != 1 {
return false, false, errors.New("graph reconciliation ledger disappeared")
}
return status.Current, status.Complete, nil
}
func reconcileGraphTask(db *gorm.DB, reader graphTaskReader, tenant uint64, kbID, revision string, c graphRecoveryCandidate, apply bool) (graphReconcileReport, error) {
report := graphReconcileReport{Mode: "reconcile_only", Apply: apply, TenantID: tenant, KnowledgeBaseID: kbID, Revision: revision, Source: c,
TaskID: graphRecoveryTaskID(tenant, kbID, c, revision), QueueState: "unknown", Blocked: true}
var ledger graphRecoveryFailure
err := db.First(&ledger, "task_id=? AND tenant_id=? AND knowledge_base_id=? AND chunk_id=? AND knowledge_id=? AND content_hash=? AND revision=?",
report.TaskID, tenant, kbID, c.ID, c.KnowledgeID, c.ContentHash, revision).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
report.Reason = "ledger_missing_or_identity_mismatch"
return report, nil
}
if err != nil {
return report, errors.New("cannot read exact graph ledger")
}
report.LedgerState, report.ProposedState = ledger.State, ledger.State
report.SourceCurrent, report.CheckpointComplete, err = graphReconcileSourceStatus(db, report.TaskID)
if err != nil {
return report, err
}
if !report.SourceCurrent {
report.Reason = "source_changed_or_unavailable"
return report, nil
}
if ledger.State == "failed" && strings.HasPrefix(ledger.Reason, "semantic_review_rejected") {
report.Reason = "manual_semantic_review_required"
return report, nil
}
info, err := reader.GetTaskInfo(types.QueueGraph, report.TaskID)
if errors.Is(err, asynq.ErrTaskNotFound) || errors.Is(err, asynq.ErrQueueNotFound) || (err == nil && info == nil) {
report.QueueState, report.Reason = "missing", "task_missing_after_dispatch"
return report, nil
}
if err != nil {
report.Reason = "queue_inspection_failed"
return report, errors.New("cannot inspect graph task; no reconciliation applied")
}
report.QueueState = info.State.String()
if info.ID != report.TaskID || info.Queue != types.QueueGraph || info.Type != types.TypeChunkExtract ||
!graphTaskMatchesReviewedSource(info.Payload, tenant, kbID, c) || info.MaxRetry != 0 {
report.Reason = "queue_task_identity_mismatch"
return report, nil
}
if info.State != asynq.TaskStateArchived && info.State != asynq.TaskStateCompleted {
report.Blocked, report.Reason = false, "task_not_terminal"
return report, nil
}
// A worker can commit between the first source read and terminal queue read.
report.SourceCurrent, report.CheckpointComplete, err = graphReconcileSourceStatus(db, report.TaskID)
if err != nil {
return report, err
}
if !report.SourceCurrent {
report.Reason = "source_changed_or_unavailable"
return report, nil
}
if report.CheckpointComplete {
report.ProposedState, report.Reason, report.Blocked = "resolved", "native_checkpoint_verified", false
} else {
report.ProposedState, report.Reason = "failed", "completed_without_native_checkpoint"
if info.State == asynq.TaskStateArchived {
report.Reason = graphArchivedReason(info.LastErr)
}
}
if ledger.State == "superseded" || (ledger.State == "resolved" && report.ProposedState != "resolved") {
report.ProposedState, report.Reason, report.Blocked = ledger.State, "ledger_terminal_conflict", true
return report, nil
}
// Preserve earlier failure and retry history; refine only an empty/generic
// reason. Resolution never erases the original failure reason or timestamp.
reason := ledger.Reason
if report.ProposedState == "failed" && (reason == "" || reason == "archived") {
reason = report.Reason
}
report.WouldUpdate = ledger.State != report.ProposedState || ledger.Reason != reason ||
(report.ProposedState == "failed" && ledger.FirstFailedAt.IsZero())
if !apply || !report.WouldUpdate {
return report, nil
}
updates := map[string]any{"state": report.ProposedState, "reason": reason, "updated_at": time.Now().UTC()}
if report.ProposedState == "failed" && ledger.FirstFailedAt.IsZero() {
failedAt := info.LastFailedAt
if failedAt.IsZero() {
failedAt = time.Now().UTC()
}
updates["first_failed_at"] = failedAt
}
checkpoint := graphFailureCheckpoint
if !report.CheckpointComplete {
checkpoint = "NOT " + checkpoint
}
// Revalidate source, checkpoint, and ledger state in the update itself so a
// concurrent source edit or recovery cannot be overwritten by this snapshot.
result := db.Table("native_graph_backfill_failures AS f").Where("f.task_id=? AND f.state=? AND f.reason=? AND f.manual_retries=? AND f.updated_at=?",
report.TaskID, ledger.State, ledger.Reason, ledger.ManualRetries, ledger.UpdatedAt).
Where("f.tenant_id=? AND f.knowledge_base_id=? AND f.chunk_id=? AND f.knowledge_id=? AND f.content_hash=? AND f.revision=?", tenant, kbID, c.ID, c.KnowledgeID, c.ContentHash, revision).
Where(graphReconcileCurrentSource).Where(checkpoint).Updates(updates)
if result.Error != nil {
return report, errors.New("cannot update exact graph ledger")
}
if result.RowsAffected != 1 {
report.Reason, report.Blocked = "reconciliation_state_changed", true
return report, nil
}
report.LedgerUpdated = true
return report, nil
}
func graphRecoveryRedisOptions() (asynq.RedisClientOpt, error) {
redisDB, err := strconv.Atoi(os.Getenv("REDIS_DB"))
if err != nil || os.Getenv("REDIS_ADDR") == "" {
return asynq.RedisClientOpt{}, errors.New("runtime Redis configuration is missing")
}
return asynq.RedisClientOpt{Addr: os.Getenv("REDIS_ADDR"), Username: os.Getenv("REDIS_USERNAME"), Password: os.Getenv("REDIS_PASSWORD"), DB: redisDB, TLSConfig: common.RedisTLSConfig()}, nil
}
// The checkpoint is queried again after observing a completed task because a
// worker may have committed between candidate selection and Redis inspection.
func graphCandidateCompleted(db *gorm.DB, tenant uint64, kbID string, c graphRecoveryCandidate) (bool, error) {
var count int64
err := db.Table("native_graph_records").Where("tenant_id=? AND knowledge_base_id=? AND source_table='knowledgegraphchunkstate' AND payload->>'origin'='native_extraction_v1' AND payload->>'extraction_status'='complete' AND CAST(COALESCE(payload->>'relation_count','0') AS INTEGER)>0 AND payload->>'native_chunk_id'=? AND payload->>'native_knowledge_id'=? AND payload->>'source_content_hash'=?", tenant, kbID, c.ID, c.KnowledgeID, c.ContentHash).Count(&count).Error
return count > 0, err
}
func graphFailureRetry(db *gorm.DB, queue graphRecoveryQueue, taskID, reason string, slots int) (bool, error) {
if slots <= 0 {
return false, nil
}
if len(strings.TrimSpace(reason)) < 12 {
return false, errors.New("manual retry requires a reason describing the investigated failure and fix")
}
var failure graphRecoveryFailure
if err := db.First(&failure, "task_id=?", taskID).Error; err != nil {
return false, err
}
if failure.ManualRetries >= 1 {
return false, nil
}
// Reserve the one allowed manual retry before sending to Redis. A crash
// here fails closed and leaves a visible record instead of retrying forever.
failure.ManualRetries++
failure.State, failure.RetryReason, failure.UpdatedAt = "retry_requested", reason, time.Now().UTC()
if err := db.Save(&failure).Error; err != nil {
return false, err
}
if err := queue.RunTask(types.QueueGraph, taskID); err != nil {
return false, err
}
return true, nil
}
func graphRecoveryDSN() (string, error) {
if dsn := os.Getenv("GRAPH_BACKFILL_DSN"); dsn != "" {
return dsn, nil
}
if os.Getenv("DB_HOST") == "" || os.Getenv("DB_NAME") == "" || os.Getenv("DB_USER") == "" {
return "", errors.New("runtime PostgreSQL configuration is missing")
}
port := os.Getenv("DB_PORT")
if port == "" {
port = "5432"
}
u := url.URL{Scheme: "postgresql", User: url.UserPassword(os.Getenv("DB_USER"), os.Getenv("DB_PASSWORD")), Host: net.JoinHostPort(os.Getenv("DB_HOST"), port), Path: "/" + os.Getenv("DB_NAME")}
q := u.Query()
ssl := os.Getenv("DB_SSL_MODE")
if ssl == "" {
ssl = "disable"
}
q.Set("sslmode", ssl)
u.RawQuery = q.Encode()
return u.String(), nil
}
func runGraphRecovery() error {
tenant := flag.Uint64("tenant", 0, "required tenant")
kbID := flag.String("kb", "", "required knowledge base UUID")
limit := flag.Int("limit", 20, "bounded chunks to enqueue (1-200)")
after := flag.Int64("after", 0, "resume after this chunk sequence ID")
apply := flag.Bool("apply", false, "apply the selected mode; otherwise report only")
reconcileOnly := flag.Bool("reconcile-only", false, "reconcile one existing exact-source ledger only; never enqueue or retry")
retry := flag.Bool("retry-failed", false, "one audited manual retry for tasks still archived in Redis")
retryReason := flag.String("retry-reason", "", "audited reason for the single permitted manual retry")
maxOutstanding := flag.Int("max-outstanding", 2, "maximum outstanding tasks in the shared graph queue (1-20)")
chunkID := flag.String("chunk", "", "only this reviewed chunk; requires knowledge and source-sha256")
knowledgeID := flag.String("knowledge", "", "reviewed source document UUID")
sourceHash := flag.String("source-sha256", "", "reviewed exact source SHA-256")
flag.Parse()
revision := os.Getenv("GRAPH_BACKFILL_REVISION")
if flag.NArg() != 0 {
return errors.New("unexpected positional arguments")
}
if err := graphValidateReconciliation(*reconcileOnly, *chunkID, revision, *retry, *retryReason); err != nil {
return err
}
if *tenant == 0 || *limit < 1 || *limit > 200 || *after < 0 || *maxOutstanding < 1 || *maxOutstanding > 20 {
return errors.New("tenant, limit 1-200 and non-negative cursor are required")
}
if *retry && len(strings.TrimSpace(*retryReason)) < 12 {
return errors.New("retry-failed requires an audited retry-reason describing the investigated failure and fix")
}
if _, err := uuid.Parse(*kbID); err != nil {
return errors.New("invalid knowledge base UUID")
}
if err := graphValidateExactScope(*chunkID, *knowledgeID, *sourceHash, *after, *limit, *maxOutstanding); err != nil {
return err
}
var exact []graphRecoveryCandidate
if *chunkID != "" {
exact = append(exact, graphRecoveryCandidate{ID: *chunkID, KnowledgeID: *knowledgeID, ContentHash: *sourceHash})
}
dsn, err := graphRecoveryDSN()
if err != nil {
return err
}
db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
if err != nil {
return errors.New("cannot open graph recovery database")
}
sqlDB, err := db.DB()
if err != nil {
return err
}
defer sqlDB.Close()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
db = db.WithContext(ctx)
if *apply {
if *reconcileOnly && os.Getenv("GRAPH_LEDGER_RECONCILE_VERIFIED") != "1" {
return errors.New("review the reconciliation preview first; then set GRAPH_LEDGER_RECONCILE_VERIFIED=1")
}
if !*reconcileOnly && os.Getenv("GRAPH_NATIVE_WRITE_VERIFIED") != "1" {
return errors.New("deploy and verify native graph persistence first; then set GRAPH_NATIVE_WRITE_VERIFIED=1")
}
// Hold a session lock on a dedicated connection while ledger writes use
// autocommit. Redis calls must never precede a durable attempt record.
lockConn, err := sqlDB.Conn(ctx)
if err != nil {
return err
}
defer lockConn.Close()
var locked bool
if err := lockConn.QueryRowContext(ctx, "SELECT pg_try_advisory_lock(20260919, 7351)").Scan(&locked); err != nil {
return err
}
if !locked {
return errors.New("another graph backfill dispatcher is active")
}
defer func() {
unlockCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, _ = lockConn.ExecContext(unlockCtx, "SELECT pg_advisory_unlock(20260919, 7351)")
}()
}
if *reconcileOnly {
if !db.Migrator().HasTable(&graphRecoveryFailure{}) {
return errors.New("graph failure ledger is missing; reconciliation will not create it")
}
redisOpt, err := graphRecoveryRedisOptions()
if err != nil {
return err
}
inspector := asynq.NewInspector(redisOpt)
defer inspector.Close()
report, err := reconcileGraphTask(db, inspector, *tenant, *kbID, revision, exact[0], *apply)
if err != nil {
return err
}
return json.NewEncoder(os.Stdout).Encode(report)
}
var kb struct {
Name, SummaryModelID string
Enabled bool
}
result := db.Raw(`SELECT name,summary_model_id,COALESCE((extract_config->>'enabled')::boolean,false) AS enabled FROM knowledge_bases WHERE tenant_id=? AND id=? AND deleted_at IS NULL`, *tenant, *kbID).Scan(&kb)
if result.Error != nil {
return result.Error
}
if result.RowsAffected != 1 {
return errors.New("knowledge base not found in tenant")
}
if *apply && (!strings.EqualFold(os.Getenv("LEGACY_GRAPH_ENABLE"), "true") || !kb.Enabled || kb.SummaryModelID == "") {
return errors.New("native graph backend, knowledge base extraction and summary model must be enabled first")
}
hasFailures := db.Migrator().HasTable(&graphRecoveryFailure{})
if *apply && !hasFailures {
if err := db.AutoMigrate(&graphRecoveryFailure{}); err != nil {
return err
}
hasFailures = true
}
if *apply {
if _, err := graphUnresolvedFailures(db, *tenant, *kbID, revision, true); err != nil {
return err
}
}
var coverage struct{ Documents, TextChunks, NativeCompleted, MappedChunks int64 }
err = db.Raw(`WITH completed AS NOT MATERIALIZED(SELECT payload FROM native_graph_records WHERE tenant_id=? AND knowledge_base_id=? AND source_table='knowledgegraphchunkstate' AND payload->>'origin'='native_extraction_v1' AND payload->>'extraction_status'='complete' AND CAST(COALESCE(payload->>'relation_count','0') AS INTEGER)>0)
SELECT count(DISTINCT c.knowledge_id) AS documents,count(*) AS text_chunks,
count(*) FILTER(WHERE EXISTS(SELECT 1 FROM completed r WHERE r.payload->>'native_chunk_id'=c.id AND r.payload->>'native_knowledge_id'=c.knowledge_id AND r.payload->>'source_content_hash'=encode(sha256(convert_to(c.content,'UTF8')),'hex'))) AS native_completed,
count(*) FILTER(WHERE EXISTS(SELECT 1 FROM native_graph_sources s WHERE s.tenant_id=c.tenant_id AND s.knowledge_base_id=c.knowledge_base_id AND s.native_chunk_id=c.id AND s.native_content_sha256=encode(sha256(convert_to(c.content,'UTF8')),'hex'))) AS mapped_chunks
FROM chunks c JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id AND k.deleted_at IS NULL
WHERE c.tenant_id=? AND c.knowledge_base_id=? AND c.deleted_at IS NULL AND c.is_enabled AND c.chunk_type='text'`, *tenant, *kbID, *tenant, *kbID).Scan(&coverage).Error
if err != nil {
return err
}
candidates, err := graphRecoveryCandidates(db, *tenant, *kbID, revision, *after, *limit, hasFailures, *retry, exact...)
if err != nil {
return err
}
report := map[string]any{"tenant_id": *tenant, "knowledge_base_id": *kbID, "knowledge_base_name": kb.Name, "graph_enabled": kb.Enabled, "coverage": coverage, "candidate_count": len(candidates), "after": *after, "next_cursor": *after, "apply": *apply, "enqueued": 0, "already_queued": 0, "retried": 0, "archived": 0, "candidates": candidates}
if !*apply {
if hasFailures {
failures, err := graphUnresolvedFailures(db, *tenant, *kbID, revision, false)
if err != nil {
return err
}
report["failures"], report["failure_count"] = failures, len(failures)
}
return json.NewEncoder(os.Stdout).Encode(report)
}
redisOpt, err := graphRecoveryRedisOptions()
if err != nil {
return err
}
client := asynq.NewClient(redisOpt)
defer client.Close()
inspector := asynq.NewInspector(redisOpt)
defer inspector.Close()
enqueued, existing, retried, archived, deferred := 0, 0, 0, 0, 0
queueInfo, err := inspector.GetQueueInfo(types.QueueGraph)
if err != nil && !errors.Is(err, asynq.ErrQueueNotFound) {
return err
}
if queueInfo == nil {
queueInfo = &asynq.QueueInfo{}
}
slots := graphQueueSlots(queueInfo, *maxOutstanding)
settled := make(map[string]bool)
for _, c := range candidates {
if len(exact) > 0 {
// Recheck immediately before dispatch. The worker checks the same
// immutable identity before calling the model if a source changes later.
if _, err := graphRecoveryCandidates(db, *tenant, *kbID, revision, 0, 1, hasFailures, *retry, exact...); err != nil {
return err
}
}
taskID := graphRecoveryTaskID(*tenant, *kbID, c, revision)
info, inspectErr := inspector.GetTaskInfo(types.QueueGraph, taskID)
if inspectErr != nil && !errors.Is(inspectErr, asynq.ErrTaskNotFound) && !errors.Is(inspectErr, asynq.ErrQueueNotFound) {
return inspectErr
}
if info != nil {
if len(exact) > 0 && (!graphTaskMatchesReviewedSource(info.Payload, *tenant, *kbID, c) || info.MaxRetry != 0) {
return errors.New("existing graph task does not have the reviewed source and single-execution guard; inspect it without resetting the failure ledger")
}
existing++
if err := trackGraphTask(db, *tenant, *kbID, revision, taskID, c); err != nil {
return err
}
if info.State == asynq.TaskStateArchived || info.State == asynq.TaskStateCompleted {
verified, err := graphCandidateCompleted(db, *tenant, *kbID, c)
if err != nil {
return err
}
if verified {
settled[c.ID] = true
continue
}
reason := "completed_without_native_checkpoint"
if info.State == asynq.TaskStateArchived {
reason = graphArchivedReason(info.LastErr)
archived++
}
if err := recordGraphFailure(db, *tenant, *kbID, revision, taskID, c, reason); err != nil {
return err
}
settled[c.ID] = true
if *retry && info.State == asynq.TaskStateArchived {
didRetry, err := graphFailureRetry(db, inspector, taskID, *retryReason, slots)
if err != nil {
return err
}
if didRetry {
retried++
slots--
settled[c.ID] = false
}
}
}
continue
}
if slots == 0 {
deferred++
continue
}
reserved, err := reserveGraphDispatch(db, *tenant, *kbID, revision, taskID, c)
if err != nil {
return err
}
if !reserved {
settled[c.ID] = true
continue
}
// KnowledgeID is intentionally empty: recovery has no ingestion slot
// to decrement and must not finalize a concurrent document upload.
payload, err := json.Marshal(types.ExtractChunkPayload{TenantID: *tenant, ChunkID: c.ID, ModelID: kb.SummaryModelID, ChunkIndex: c.ChunkIndex})
if len(exact) > 0 {
payload, err = json.Marshal(types.ExtractChunkPayload{TenantID: *tenant, ChunkID: c.ID, ModelID: kb.SummaryModelID, ChunkIndex: c.ChunkIndex,
ExpectedSource: &types.GraphSourceIdentity{KnowledgeBaseID: *kbID, KnowledgeID: c.KnowledgeID, ContentSHA256: c.ContentHash}})
}
if err != nil {
return err
}
maxRetry, timeout := 3, 10*time.Minute
if len(exact) > 0 {
maxRetry, timeout = 0, 3*time.Minute
}
_, err = client.EnqueueContext(ctx, asynq.NewTask(types.TypeChunkExtract, payload), asynq.Queue(types.QueueGraph), asynq.TaskID(taskID), asynq.MaxRetry(maxRetry), asynq.Timeout(timeout), asynq.Retention(24*time.Hour))
if errors.Is(err, asynq.ErrTaskIDConflict) {
existing++
// Another producer raced with this dispatcher. Count the slot
// conservatively; the next poll will inspect its actual state.
slots--
continue
}
if err != nil {
return err
}
enqueued++
slots--
}
report["enqueued"], report["already_queued"], report["retried"] = enqueued, existing, retried
// Do not let a manual resume skip deferred or still-running work. Watch
// always starts at zero and additionally uses persisted failure exclusions.
report["next_cursor"] = graphResumeCursor(*after, candidates, settled)
report["archived"] = archived
report["deferred_for_queue_limit"], report["max_outstanding"] = deferred, *maxOutstanding
report["queue_paused"] = queueInfo.Paused
report["queue_limit_scope"] = "serialized_backfill_dispatchers; other graph producers may enqueue concurrently"
failures, err := graphUnresolvedFailures(db, *tenant, *kbID, revision, true)
if err != nil {
return err
}
report["failures"], report["failure_count"] = failures, len(failures)
return json.NewEncoder(os.Stdout).Encode(report)
}
func graphBatchStatus(report map[string]any) (complete, blocked bool) {
coverage, _ := report["coverage"].(map[string]any)
textChunks, totalOK := coverage["TextChunks"].(float64)
completedChunks, completedOK := coverage["NativeCompleted"].(float64)
candidates, candidatesOK := report["candidate_count"].(float64)
archived, _ := report["archived"].(float64)
failures, _ := report["failure_count"].(float64)
paused, _ := report["queue_paused"].(bool)
valid := totalOK && completedOK && candidatesOK && textChunks >= 0 && completedChunks >= 0 && completedChunks <= textChunks && candidates >= 0 && archived >= 0 && failures >= 0
complete = valid && completedChunks == textChunks && candidates == 0 && failures == 0 && archived == 0
blocked = archived > 0 || failures > 0 || !valid || (!complete && (candidates == 0 || paused))
return
}
// A restart resumes from verified database checkpoints. Each knowledge base has
// only one bounded window in flight; failed windows require explicit recovery.
func watchGraphRecovery() error {
flags := flag.NewFlagSet("watch", flag.ContinueOnError)
tenant := flags.Uint64("tenant", 0, "required tenant")
kbs := flags.String("kbs", "", "comma-separated knowledge base UUIDs")
limit := flags.Int("limit", 2, "maximum candidate window per knowledge base (1-20)")
maxOutstanding := flags.Int("max-outstanding", 2, "maximum outstanding tasks across the shared graph queue (1-20)")
interval := flags.Duration("interval", time.Minute, "poll interval (at least 30s)")
stateFile := flags.String("state", "/data/progress.json", "durable progress report")
if err := flags.Parse(os.Args[2:]); err != nil {
return err
}
if *tenant == 0 || *limit < 1 || *limit > 20 || *interval < 30*time.Second || *kbs == "" || *maxOutstanding < 1 || *maxOutstanding > 20 {
return errors.New("watch requires tenant, knowledge bases, limit 1-20 and interval >=30s")
}
ids := strings.Split(*kbs, ",")
if len(ids) > 10 {
return errors.New("watch supports at most 10 knowledge bases")
}
for _, id := range ids {
if _, err := uuid.Parse(id); err != nil {
return errors.New("invalid knowledge base in watch scope")
}
}
executable, err := os.Executable()
if err != nil {
return err
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
for {
reports := []map[string]any{}
complete, blocked := true, false
for _, id := range ids {
commandCtx, cancel := context.WithTimeout(ctx, 6*time.Minute)
command := exec.CommandContext(commandCtx, executable, "-tenant", strconv.FormatUint(*tenant, 10), "-kb", id, "-limit", strconv.Itoa(*limit), "-max-outstanding", strconv.Itoa(*maxOutstanding), "-apply")
output, commandErr := command.Output()
cancel()
var report map[string]any
if commandErr != nil || json.Unmarshal(output, &report) != nil {
report = map[string]any{"knowledge_base_id": id, "error": "batch command failed; check runtime and model configuration"}
blocked, complete = true, false
} else {
delete(report, "candidates")
batchComplete, batchBlocked := graphBatchStatus(report)
complete = complete && batchComplete
blocked = blocked || batchBlocked
if batchBlocked {
report["attention"] = "Native coverage is incomplete or extraction failures need recovery"
}
}
reports = append(reports, report)
if ctx.Err() != nil {
return nil
}
}
status := "running"
if complete {
status = "complete"
} else if blocked {
status = "needs_attention"
}
state := map[string]any{"status": status, "checked_at": time.Now().UTC(), "tenant_id": *tenant, "batch_limit_per_kb": *limit, "max_outstanding": *maxOutstanding, "knowledge_bases": reports}
data, err := json.MarshalIndent(state, "", " ")
if err != nil {
return err
}
if err = os.MkdirAll(filepath.Dir(*stateFile), 0700); err != nil {
return err
}
if err = os.WriteFile(*stateFile+".tmp", data, 0600); err != nil {
return err
}
if err = os.Rename(*stateFile+".tmp", *stateFile); err != nil {
return err
}
fmt.Printf("{\"status\":%q,\"knowledge_bases\":%d}\n", status, len(reports))
if complete {
// Keep the successful checkpoint visible without restart-policy loops.
<-ctx.Done()
return nil
}
select {
case <-ctx.Done():
return nil
case <-time.After(*interval):
}
}
}
func main() {
var err error
if len(os.Args) > 1 && os.Args[1] == "--watch" {
err = watchGraphRecovery()
} else {
err = runGraphRecovery()
}
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
+848
View File
@@ -0,0 +1,848 @@
package main
import (
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
"github.com/Tencent/WeKnora/internal/types"
"github.com/hibiken/asynq"
"github.com/mattn/go-sqlite3"
"gorm.io/driver/postgres"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
func TestGraphExactTaskCannotReuseAnUnpinnedPayload(t *testing.T) {
candidate := graphRecoveryCandidate{ID: "chunk", KnowledgeID: "doc", ContentHash: strings.Repeat("a", 64)}
for _, change := range []string{"unchanged", "legacy", "tenant", "chunk", "kb", "document", "hash", "invalid"} {
t.Run(change, func(t *testing.T) {
task := types.ExtractChunkPayload{TenantID: 1, ChunkID: candidate.ID, ExpectedSource: &types.GraphSourceIdentity{KnowledgeBaseID: "kb", KnowledgeID: candidate.KnowledgeID, ContentSHA256: candidate.ContentHash}}
switch change {
case "legacy":
task.ExpectedSource = nil
case "tenant":
task.TenantID = 2
case "chunk":
task.ChunkID = "other"
case "kb":
task.ExpectedSource.KnowledgeBaseID = "other"
case "document":
task.ExpectedSource.KnowledgeID = "other"
case "hash":
task.ExpectedSource.ContentSHA256 = strings.Repeat("b", 64)
}
payload, _ := json.Marshal(task)
if change == "invalid" {
payload = []byte("invalid")
}
if graphTaskMatchesReviewedSource(payload, 1, "kb", candidate) != (change == "unchanged") {
t.Fatal("unsafe existing task accepted")
}
})
}
}
func TestGraphExactScopeValidation(t *testing.T) {
const chunk = "e40b74eb-0ac7-4ff6-a107-00ce5adfe610"
const knowledge = "9669e1b1-e28e-49a4-ae13-e97e37e21b11"
const hash = "4c4d84679328fc6dd3c17add7a91d13669c2d0cc2bc507098ed4fcc1c0dd84b2"
for _, tc := range []struct {
name, chunk, knowledge, hash string
after int64
limit, outstanding int
valid bool
}{
{"normal batch", "", "", "", 2, 20, 2, true},
{"exact", chunk, knowledge, hash, 0, 1, 1, true},
{"missing chunk", "", knowledge, hash, 0, 1, 1, false},
{"missing knowledge", chunk, "", hash, 0, 1, 1, false},
{"missing hash", chunk, knowledge, "", 0, 1, 1, false},
{"invalid chunk", "other", knowledge, hash, 0, 1, 1, false},
{"nil uuid", "00000000-0000-0000-0000-000000000000", knowledge, hash, 0, 1, 1, false},
{"noncanonical chunk", strings.ToUpper(chunk), knowledge, hash, 0, 1, 1, false},
{"invalid hash", chunk, knowledge, strings.Repeat("z", 64), 0, 1, 1, false},
{"uppercase hash", chunk, knowledge, strings.ToUpper(hash), 0, 1, 1, false},
{"short hash", chunk, knowledge, hash[:62], 0, 1, 1, false},
{"cursor", chunk, knowledge, hash, 1, 1, 1, false},
{"batch", chunk, knowledge, hash, 0, 2, 1, false},
{"parallel", chunk, knowledge, hash, 0, 1, 2, false},
} {
t.Run(tc.name, func(t *testing.T) {
err := graphValidateExactScope(tc.chunk, tc.knowledge, tc.hash, tc.after, tc.limit, tc.outstanding)
if (err == nil) != tc.valid {
t.Fatalf("valid=%t err=%v", tc.valid, err)
}
})
}
}
func TestGraphExactScopeNeverSubstitutesNextCandidate(t *testing.T) {
for _, change := range []string{"unchanged", "hash", "document", "disabled", "deleted", "document deleted", "unparsed", "completed", "failed", "other tenant", "other kb", "missing"} {
t.Run(change, func(t *testing.T) {
db := graphTestDB(t, filepath.Join(t.TempDir(), "exact.db"))
wanted := seedGraphChunk(t, db, 1, "kb", "wanted", "reviewed source", 1)
seedGraphChunk(t, db, 1, "kb", "next", "next source", 2)
switch change {
case "hash":
db.Exec("UPDATE chunks SET content='changed' WHERE id='wanted'")
case "document":
wanted.KnowledgeID = "other"
case "disabled":
db.Exec("UPDATE chunks SET is_enabled=false WHERE id='wanted'")
case "deleted":
db.Exec("UPDATE chunks SET deleted_at='2026-09-20' WHERE id='wanted'")
case "document deleted":
db.Exec("UPDATE knowledges SET deleted_at='2026-09-20' WHERE id=?", wanted.KnowledgeID)
case "unparsed":
db.Exec("UPDATE knowledges SET parse_status='pending' WHERE id=?", wanted.KnowledgeID)
case "completed":
saveGraphCheckpoint(t, db, 1, "kb", wanted, wanted.ContentHash)
case "failed":
if err := recordGraphFailure(db, 1, "kb", "grounded-v6", graphRecoveryTaskID(1, "kb", wanted, "grounded-v6"), wanted, "review_required"); err != nil {
t.Fatal(err)
}
case "other tenant":
db.Exec("UPDATE chunks SET tenant_id=2 WHERE id='wanted'")
case "other kb":
db.Exec("UPDATE chunks SET knowledge_base_id='other' WHERE id='wanted'")
case "missing":
wanted.ID = "absent"
}
got, err := graphRecoveryCandidates(db, 1, "kb", "grounded-v6", 0, 1, true, false, wanted)
if change == "unchanged" {
if err != nil || len(got) != 1 || got[0].ID != wanted.ID {
t.Fatalf("exact source: %v %v", got, err)
}
} else if err == nil || len(got) != 0 {
t.Fatalf("changed source selected a candidate: %v %v", got, err)
}
})
}
}
func TestGraphWatcherRequiresVerifiedCoverage(t *testing.T) {
for _, test := range []struct {
name, report string
complete, blocked bool
}{
{"all source chunks verified", `{"coverage":{"TextChunks":51,"NativeCompleted":51},"candidate_count":0}`, true, false},
{"empty knowledge base", `{"coverage":{"TextChunks":0,"NativeCompleted":0},"candidate_count":0}`, true, false},
{"unparsed source excludes remaining candidates", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":0}`, false, true},
{"queued work is not complete", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":2,"already_queued":2}`, false, false},
{"exhausted retries need attention", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":2,"archived":1}`, false, true},
{"old mapping is not extraction", `{"coverage":{"TextChunks":51,"MappedChunks":51,"NativeCompleted":0},"candidate_count":0}`, false, true},
{"missing coverage fails closed", `{"candidate_count":0}`, false, true},
{"persistent failures survive candidate exclusion", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":0,"failure_count":2}`, false, true},
{"failures remain visible while later work progresses", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":1,"failure_count":1}`, false, true},
{"stale failure report must not claim complete", `{"coverage":{"TextChunks":51,"NativeCompleted":51},"candidate_count":0,"failure_count":1}`, false, true},
{"impossible coverage fails closed", `{"coverage":{"TextChunks":51,"NativeCompleted":52},"candidate_count":1}`, false, true},
{"paused queue needs attention", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":2,"queue_paused":true}`, false, true},
} {
t.Run(test.name, func(t *testing.T) {
var report map[string]any
if err := json.Unmarshal([]byte(test.report), &report); err != nil {
t.Fatal(err)
}
complete, blocked := graphBatchStatus(report)
if complete != test.complete || blocked != test.blocked {
t.Fatalf("got complete=%t blocked=%t; want %t %t", complete, blocked, test.complete, test.blocked)
}
})
}
}
func TestGraphArchivedReasonDoesNotExposeProviderDetails(t *testing.T) {
for _, tc := range []struct{ message, reason string }{
{"graph_needs_review: no source-grounded relationships: skip retry for the task", "relation_coverage_review_required"},
{"provider rejected secret credential", "archived"},
{"read response: context deadline exceeded; credential=secret", "provider_response_timeout"},
{"read response: net/http: timeout awaiting response headers", "provider_response_timeout"},
{"context deadline exceeded", "task_timeout"},
{"context canceled", "task_cancelled"},
{"", "archived"},
} {
if got := graphArchivedReason(tc.message); got != tc.reason {
t.Fatalf("got %q; want %q", got, tc.reason)
}
}
}
func init() {
// The production candidate SQL is exercised unchanged. These deterministic
// functions provide PostgreSQL's content-hash operations to the test database.
sql.Register("graph_backfill_test", &sqlite3.SQLiteDriver{ConnectHook: func(conn *sqlite3.SQLiteConn) error {
if err := conn.RegisterFunc("convert_to", func(content, _ string) string { return content }, true); err != nil {
return err
}
if err := conn.RegisterFunc("sha256", func(content string) []byte { hash := sha256.Sum256([]byte(content)); return hash[:] }, true); err != nil {
return err
}
return conn.RegisterFunc("encode", func(content []byte, _ string) string { return hex.EncodeToString(content) }, true)
}})
}
func graphTestDB(t *testing.T, path string) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Dialector{DriverName: "graph_backfill_test", DSN: path}, &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
if err != nil {
t.Fatal(err)
}
conn, err := db.DB()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = conn.Close() })
if err := db.AutoMigrate(&graphRecoveryFailure{}); err != nil {
t.Fatal(err)
}
for _, statement := range []string{
`CREATE TABLE IF NOT EXISTS knowledge_bases(id TEXT,tenant_id INTEGER,deleted_at TEXT)`,
`CREATE TABLE IF NOT EXISTS chunks(id TEXT,tenant_id INTEGER,knowledge_base_id TEXT,knowledge_id TEXT,seq_id INTEGER,chunk_index INTEGER,content TEXT,deleted_at TEXT,is_enabled BOOLEAN,chunk_type TEXT)`,
`CREATE TABLE IF NOT EXISTS knowledges(id TEXT,tenant_id INTEGER,knowledge_base_id TEXT,deleted_at TEXT,parse_status TEXT)`,
`CREATE TABLE IF NOT EXISTS native_graph_records(tenant_id INTEGER,knowledge_base_id TEXT,source_table TEXT,payload JSON)`,
} {
if err := db.Exec(statement).Error; err != nil {
t.Fatal(err)
}
}
return db
}
func seedGraphChunk(t *testing.T, db *gorm.DB, tenant uint64, kbID, id, content string, seq int) graphRecoveryCandidate {
t.Helper()
if err := db.Exec(`INSERT INTO knowledge_bases(id,tenant_id) SELECT ?,? WHERE NOT EXISTS(SELECT 1 FROM knowledge_bases WHERE id=? AND tenant_id=?)`, kbID, tenant, kbID, tenant).Error; err != nil {
t.Fatal(err)
}
knowledgeID := "knowledge-" + id
if err := db.Exec(`INSERT INTO knowledges(id,tenant_id,knowledge_base_id,parse_status) VALUES(?,?,?,'completed')`, knowledgeID, tenant, kbID).Error; err != nil {
t.Fatal(err)
}
if err := db.Exec(`INSERT INTO chunks(id,tenant_id,knowledge_base_id,knowledge_id,seq_id,chunk_index,content,is_enabled,chunk_type) VALUES(?,?,?,?,?,0,?,true,'text')`, id, tenant, kbID, knowledgeID, seq, content).Error; err != nil {
t.Fatal(err)
}
hash := sha256.Sum256([]byte(content))
return graphRecoveryCandidate{ID: id, KnowledgeID: knowledgeID, SeqID: int64(seq), ContentHash: hex.EncodeToString(hash[:])}
}
type graphTestReader struct {
info *asynq.TaskInfo
err error
reads int
onRead func()
}
func (r *graphTestReader) GetTaskInfo(_, _ string) (*asynq.TaskInfo, error) {
r.reads++
if r.onRead != nil {
r.onRead()
}
return r.info, r.err
}
func seedGraphReconciliation(t *testing.T) (*gorm.DB, graphRecoveryCandidate, *graphTestReader) {
t.Helper()
db := graphTestDB(t, filepath.Join(t.TempDir(), "reconcile.db"))
c := seedGraphChunk(t, db, 1, "kb", "source", "reviewed source", 1)
id := graphRecoveryTaskID(1, "kb", c, "grounded-v6")
if ok, err := reserveGraphDispatch(db, 1, "kb", "grounded-v6", id, c); err != nil || !ok {
t.Fatalf("reserve: %t %v", ok, err)
}
payload, err := json.Marshal(types.ExtractChunkPayload{TenantID: 1, ChunkID: c.ID, ExpectedSource: &types.GraphSourceIdentity{KnowledgeBaseID: "kb", KnowledgeID: c.KnowledgeID, ContentSHA256: c.ContentHash}})
if err != nil {
t.Fatal(err)
}
return db, c, &graphTestReader{info: &asynq.TaskInfo{ID: id, Queue: types.QueueGraph, Type: types.TypeChunkExtract, Payload: payload, State: asynq.TaskStateArchived, LastErr: "read response: context deadline exceeded; credential=must-not-leak", LastFailedAt: time.Date(2026, 9, 20, 8, 46, 33, 0, time.UTC)}}
}
func graphReadLedger(t *testing.T, db *gorm.DB, id string) graphRecoveryFailure {
t.Helper()
var ledger graphRecoveryFailure
if err := db.First(&ledger, "task_id=?", id).Error; err != nil {
t.Fatal(err)
}
return ledger
}
func TestGraphReconciliationRequiresExactNonDispatchMode(t *testing.T) {
for _, tc := range []struct {
name, chunk, revision, retryReason string
retry, valid bool
}{
{name: "exact", chunk: "chunk", revision: "grounded-v6", valid: true},
{name: "missing source", revision: "grounded-v6"},
{name: "missing revision", chunk: "chunk"},
{name: "blank revision", chunk: "chunk", revision: " "},
{name: "retry requested", chunk: "chunk", revision: "grounded-v6", retry: true},
{name: "retry reason supplied", chunk: "chunk", revision: "grounded-v6", retryReason: "retry attempt"},
} {
t.Run(tc.name, func(t *testing.T) {
err := graphValidateReconciliation(true, tc.chunk, tc.revision, tc.retry, tc.retryReason)
if (err == nil) != tc.valid {
t.Fatalf("valid=%t err=%v", tc.valid, err)
}
})
}
}
func TestGraphReconciliationCLIRejectsMixedModesBeforeOpeningDatabase(t *testing.T) {
const chunk = "edf40fb3-f09d-4532-a6fa-533408fe19cb"
const knowledge = "c3db6397-6b50-4d1e-a667-2e3efe503cc7"
const hash = "63bdc854af31ea50dd0360d93a8b2340391e15e5a1c251db714ca1e9533b2e5b"
for _, tc := range []struct {
name, flag, value, want string
}{
{"retry", "--retry-failed", "true", "no retry flags"},
{"retry reason", "--retry-reason", "previous investigation", "no retry flags"},
{"cursor", "--after", "1", "no cursor"},
{"batch", "--limit", "2", "limit=1"},
{"parallel", "--max-outstanding", "2", "max-outstanding=1"},
{"missing hash", "--source-sha256", "", "source SHA-256"},
} {
t.Run(tc.name, func(t *testing.T) {
originalFlags, originalArgs := flag.CommandLine, os.Args
defer func() { flag.CommandLine, os.Args = originalFlags, originalArgs }()
flag.CommandLine = flag.NewFlagSet("reconcile-test", flag.ContinueOnError)
t.Setenv("GRAPH_BACKFILL_REVISION", "grounded-v6")
t.Setenv("GRAPH_BACKFILL_DSN", "invalid connection must not be opened")
os.Args = []string{"reconcile-test", "--reconcile-only", "--tenant", "10059", "--kb", "16c3e59c-4bb6-41cf-ae0a-6d0941b3d032", "--chunk", chunk, "--knowledge", knowledge, "--source-sha256", hash, "--limit", "1", "--max-outstanding", "1", tc.flag + "=" + tc.value}
if err := runGraphRecovery(); err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("expected validation %q before connection, got %v", tc.want, err)
}
})
}
}
func TestGraphReconciliationSanitizesPostgresCheckpointErrors(t *testing.T) {
conn, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
}
defer conn.Close()
db, err := gorm.Open(postgres.New(postgres.Config{Conn: conn}), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
if err != nil {
t.Fatal(err)
}
mock.ExpectQuery("SELECT .* AS current.* FROM native_graph_backfill_failures").WithArgs("task").WillReturnError(errors.New(`invalid input syntax for type integer: "private-source-value"`))
current, complete, err := graphReconcileSourceStatus(db, "task")
if err == nil || current || complete || strings.Contains(err.Error(), "private-source-value") {
t.Fatalf("unsafe checkpoint query outcome: %t %t %v", current, complete, err)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal(err)
}
}
func TestGraphReconciliationPreviewThenApplyIsBoundedAndIdempotent(t *testing.T) {
db, c, reader := seedGraphReconciliation(t)
original := graphReadLedger(t, db, reader.info.ID)
other := seedGraphChunk(t, db, 1, "kb", "other", "untouched source", 2)
otherID := graphRecoveryTaskID(1, "kb", other, "grounded-v6")
if _, err := reserveGraphDispatch(db, 1, "kb", "grounded-v6", otherID, other); err != nil {
t.Fatal(err)
}
unrelated := graphReadLedger(t, db, otherID)
preview, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, false)
if err != nil || !preview.WouldUpdate || preview.LedgerUpdated || !preview.Blocked || preview.Reason != "provider_response_timeout" || preview.ProposedState != "failed" {
t.Fatalf("preview: %+v %v", preview, err)
}
if got := graphReadLedger(t, db, reader.info.ID); !reflect.DeepEqual(got, original) {
t.Fatal("preview changed durable state")
}
result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
if err != nil || !result.LedgerUpdated || result.Enqueued != 0 || result.Retried != 0 || result.ModelCalls != 0 {
t.Fatalf("apply: %+v %v", result, err)
}
updated := graphReadLedger(t, db, reader.info.ID)
if updated.State != "failed" || updated.Reason != "provider_response_timeout" || !updated.FirstFailedAt.Equal(reader.info.LastFailedAt) || updated.ManualRetries != 0 {
t.Fatalf("failure not durably recorded: %+v", updated)
}
encoded, _ := json.Marshal(result)
if strings.Contains(string(encoded), "must-not-leak") {
t.Fatal("report exposed provider details")
}
result, err = reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
if err != nil || result.WouldUpdate || result.LedgerUpdated || !reflect.DeepEqual(updated, graphReadLedger(t, db, reader.info.ID)) {
t.Fatalf("idempotence: %+v %v", result, err)
}
if !reflect.DeepEqual(unrelated, graphReadLedger(t, db, otherID)) {
t.Fatal("reconciliation changed another source ledger")
}
}
func TestGraphReconciliationRefinesArchivedReasonWithoutLosingHistory(t *testing.T) {
db, c, reader := seedGraphReconciliation(t)
failedAt := time.Date(2026, 9, 18, 1, 2, 3, 0, time.UTC)
if err := db.Model(&graphRecoveryFailure{}).Where("task_id=?", reader.info.ID).Updates(map[string]any{"state": "failed", "reason": "archived", "first_failed_at": failedAt, "manual_retries": 1, "retry_reason": "previous authorized recovery"}).Error; err != nil {
t.Fatal(err)
}
result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
after := graphReadLedger(t, db, reader.info.ID)
if err != nil || !result.LedgerUpdated || after.Reason != "provider_response_timeout" || !after.FirstFailedAt.Equal(failedAt) || after.ManualRetries != 1 || after.RetryReason != "previous authorized recovery" {
t.Fatalf("failure history changed: %+v %+v %v", result, after, err)
}
}
func TestGraphReconciliationRequiresTerminalQueueAndPinnedIdentity(t *testing.T) {
for _, change := range []string{"missing task", "missing queue", "nil task", "redis error", "active", "pending", "retry", "payload", "top document", "type", "queue", "task id", "max retry", "source hash", "source disabled", "source deleted", "document deleted", "unparsed", "kb deleted", "tenant", "document identity", "resolved", "superseded"} {
t.Run(change, func(t *testing.T) {
db, c, reader := seedGraphReconciliation(t)
id := reader.info.ID
switch change {
case "missing task":
reader.err = asynq.ErrTaskNotFound
case "missing queue":
reader.err = asynq.ErrQueueNotFound
case "nil task":
reader.info = nil
case "redis error":
reader.err = errors.New("secret connection details")
case "active":
reader.info.State = asynq.TaskStateActive
case "pending":
reader.info.State = asynq.TaskStatePending
case "retry":
reader.info.State = asynq.TaskStateRetry
case "payload":
reader.info.Payload = []byte(`{"tenant_id":2}`)
case "top document":
var p types.ExtractChunkPayload
_ = json.Unmarshal(reader.info.Payload, &p)
p.KnowledgeID = "other-document"
reader.info.Payload, _ = json.Marshal(p)
case "type":
reader.info.Type = "other"
case "queue":
reader.info.Queue = "other"
case "task id":
reader.info.ID = "other"
case "max retry":
reader.info.MaxRetry = 1
case "source hash":
db.Exec("UPDATE chunks SET content='changed'")
case "source disabled":
db.Exec("UPDATE chunks SET is_enabled=false")
case "source deleted":
db.Exec("UPDATE chunks SET deleted_at='2026-09-20'")
case "document deleted":
db.Exec("UPDATE knowledges SET deleted_at='2026-09-20'")
case "unparsed":
db.Exec("UPDATE knowledges SET parse_status='processing'")
case "kb deleted":
db.Exec("UPDATE knowledge_bases SET deleted_at='2026-09-20'")
case "tenant":
db.Exec("UPDATE chunks SET tenant_id=2")
case "document identity":
c.KnowledgeID = "other-document"
case "resolved", "superseded":
db.Model(&graphRecoveryFailure{}).Where("task_id=?", id).Update("state", change)
}
before := graphReadLedger(t, db, id)
result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
if (err != nil) != (change == "redis error") || result.LedgerUpdated || result.WouldUpdate || result.Enqueued != 0 || result.Retried != 0 || result.ModelCalls != 0 {
t.Fatalf("unsafe reconcile: %+v %v", result, err)
}
if err != nil && strings.Contains(err.Error(), "secret") {
t.Fatal("inspection error exposed connection details")
}
if change != "active" && change != "pending" && change != "retry" && !result.Blocked {
t.Fatalf("unsafe case not blocked: %+v", result)
}
if !reflect.DeepEqual(before, graphReadLedger(t, db, id)) {
t.Fatal("blocked reconciliation changed ledger")
}
})
}
}
func TestGraphReconciliationRequiresCurrentNonemptyCheckpointAndPreservesHistory(t *testing.T) {
for _, checkpoint := range []string{"current", "stale hash", "other tenant", "other kb", "other document", "empty", "none", "missing redis"} {
t.Run(checkpoint, func(t *testing.T) {
db, c, reader := seedGraphReconciliation(t)
reader.info.State = asynq.TaskStateCompleted
failedAt := time.Date(2026, 9, 18, 0, 0, 0, 0, time.UTC)
if err := db.Model(&graphRecoveryFailure{}).Where("task_id=?", reader.info.ID).Updates(map[string]any{"state": "failed", "reason": "provider_response_timeout", "first_failed_at": failedAt, "manual_retries": 1, "retry_reason": "previous authorized recovery"}).Error; err != nil {
t.Fatal(err)
}
if checkpoint != "none" {
hash := c.ContentHash
if checkpoint == "stale hash" {
hash = "old-hash"
}
saveGraphCheckpoint(t, db, 1, "kb", c, hash)
}
switch checkpoint {
case "other tenant":
db.Exec("UPDATE native_graph_records SET tenant_id=2")
case "other kb":
db.Exec("UPDATE native_graph_records SET knowledge_base_id='other'")
case "other document":
db.Exec(`UPDATE native_graph_records SET payload=json_set(payload,'$.native_knowledge_id','other')`)
case "empty":
db.Exec(`UPDATE native_graph_records SET payload=json_set(payload,'$.relation_count',0)`)
case "missing redis":
reader.err = asynq.ErrTaskNotFound
}
before := graphReadLedger(t, db, reader.info.ID)
result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
if err != nil || result.LedgerUpdated != (checkpoint == "current") || result.Blocked == (checkpoint == "current") {
t.Fatalf("checkpoint reconciliation: %+v %v", result, err)
}
after := graphReadLedger(t, db, reader.info.ID)
if checkpoint == "current" && after.State != "resolved" {
t.Fatal("verified checkpoint not resolved")
}
if after.Reason != before.Reason || !after.FirstFailedAt.Equal(before.FirstFailedAt) || after.ManualRetries != before.ManualRetries || after.RetryReason != before.RetryReason {
t.Fatal("reconciliation discarded prior failure history")
}
})
}
}
func TestGraphReconciliationRechecksSourceAfterQueueRead(t *testing.T) {
db, c, reader := seedGraphReconciliation(t)
before := graphReadLedger(t, db, reader.info.ID)
reader.onRead = func() { db.Exec("UPDATE chunks SET content='concurrent edit'") }
result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
if err != nil || !result.Blocked || result.LedgerUpdated || result.Reason != "source_changed_or_unavailable" || !reflect.DeepEqual(before, graphReadLedger(t, db, reader.info.ID)) {
t.Fatalf("source edit lost: %+v %v", result, err)
}
}
func TestGraphReconciliationCannotOverrideManualSemanticRejection(t *testing.T) {
db, c, reader := seedGraphReconciliation(t)
saveGraphCheckpoint(t, db, 1, "kb", c, c.ContentHash)
if err := recordGraphFailure(db, 1, "kb", "grounded-v6", reader.info.ID, c, "semantic_review_rejected_product_scope"); err != nil {
t.Fatal(err)
}
before := graphReadLedger(t, db, reader.info.ID)
result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
if err != nil || !result.Blocked || result.LedgerUpdated || result.Reason != "manual_semantic_review_required" || !reflect.DeepEqual(before, graphReadLedger(t, db, reader.info.ID)) {
t.Fatalf("manual rejection was overridden: %+v %v", result, err)
}
}
func TestGraphReconciliationAtomicUpdateChecksConcurrentChanges(t *testing.T) {
for _, change := range []string{"source", "checkpoint", "ledger", "ledger identity"} {
t.Run(change, func(t *testing.T) {
db, c, reader := seedGraphReconciliation(t)
before := graphReadLedger(t, db, reader.info.ID)
fired := false
if err := db.Callback().Update().Before("gorm:update").Register("test:concurrent_change", func(tx *gorm.DB) {
if fired {
return
}
fired = true
tx = tx.Session(&gorm.Session{NewDB: true})
switch change {
case "source":
tx.Exec("UPDATE chunks SET content='concurrent edit'")
case "checkpoint":
saveGraphCheckpoint(t, tx, 1, "kb", c, c.ContentHash)
case "ledger":
tx.Exec("UPDATE native_graph_backfill_failures SET state='retry_requested', manual_retries=1 WHERE task_id=?", reader.info.ID)
case "ledger identity":
tx.Exec("UPDATE native_graph_backfill_failures SET knowledge_id='other-document' WHERE task_id=?", reader.info.ID)
}
}); err != nil {
t.Fatal(err)
}
result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
if err != nil || !fired || !result.Blocked || result.LedgerUpdated || result.Reason != "reconciliation_state_changed" {
t.Fatalf("concurrent change overwritten: %+v %v", result, err)
}
after := graphReadLedger(t, db, reader.info.ID)
if after.Reason != before.Reason || !after.FirstFailedAt.Equal(before.FirstFailedAt) {
t.Fatal("failed compare-and-update mutated failure history")
}
})
}
}
func saveGraphCheckpoint(t *testing.T, db *gorm.DB, tenant uint64, kbID string, c graphRecoveryCandidate, hash string) {
t.Helper()
payload, err := json.Marshal(map[string]string{"origin": "native_extraction_v1", "extraction_status": "complete", "relation_count": "1", "native_chunk_id": c.ID, "native_knowledge_id": c.KnowledgeID, "source_content_hash": hash})
if err != nil {
t.Fatal(err)
}
if err := db.Exec(`INSERT INTO native_graph_records(tenant_id,knowledge_base_id,source_table,payload) VALUES(?,?,'knowledgegraphchunkstate',?)`, tenant, kbID, string(payload)).Error; err != nil {
t.Fatal(err)
}
}
func TestGraphFailureAllowsLaterCandidateAfterRestart(t *testing.T) {
path := filepath.Join(t.TempDir(), "recovery.db")
db := graphTestDB(t, path)
first := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
second := seedGraphChunk(t, db, 1, "kb", "second", "second content", 2)
taskID := graphRecoveryTaskID(1, "kb", first, "revision")
reserved, err := reserveGraphDispatch(db, 1, "kb", "revision", taskID, first)
if err != nil || !reserved {
t.Fatalf("reserve: %t %v", reserved, err)
}
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, first, "archived"); err != nil {
t.Fatal(err)
}
sqlDB, _ := db.DB()
if err := sqlDB.Close(); err != nil {
t.Fatal(err)
}
db = graphTestDB(t, path)
candidates, err := graphRecoveryCandidates(db, 1, "kb", "revision", 0, 1, true, false)
if err != nil || len(candidates) != 1 || candidates[0].ID != second.ID {
t.Fatalf("blocked behind archived task: %+v %v", candidates, err)
}
failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", false)
if err != nil || len(failures) != 1 || failures[0].Reason != "archived" || failures[0].FirstFailedAt.IsZero() {
t.Fatalf("lost failure accounting: %+v %v", failures, err)
}
// A new extraction revision and another tenant must not inherit this exclusion.
candidates, err = graphRecoveryCandidates(db, 1, "kb", "new-revision", 0, 1, true, false)
if err != nil || len(candidates) != 1 || candidates[0].ID != first.ID {
t.Fatalf("revision scope: %+v %v", candidates, err)
}
seedGraphChunk(t, db, 2, "kb", "first", "first content", 1)
candidates, err = graphRecoveryCandidates(db, 2, "kb", "revision", 0, 1, true, false)
if err != nil || len(candidates) != 1 || candidates[0].ID != first.ID {
t.Fatalf("tenant scope: %+v %v", candidates, err)
}
}
func TestGraphLegacyEmptyCheckpointIsNotCompletedOrResolved(t *testing.T) {
db := graphTestDB(t, filepath.Join(t.TempDir(), "recovery.db"))
c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
saveGraphCheckpoint(t, db, 1, "kb", c, c.ContentHash)
if err := db.Exec(`UPDATE native_graph_records SET payload=json_set(payload,'$.relation_count',0)`).Error; err != nil {
t.Fatal(err)
}
taskID := graphRecoveryTaskID(1, "kb", c, "revision")
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "empty_result"); err != nil {
t.Fatal(err)
}
complete, err := graphCandidateCompleted(db, 1, "kb", c)
if err != nil || complete {
t.Fatalf("empty checkpoint completed: %t %v", complete, err)
}
failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", true)
if err != nil || len(failures) != 1 {
t.Fatalf("empty checkpoint resolved failure: %+v %v", failures, err)
}
candidates, err := graphRecoveryCandidates(db, 1, "kb", "new-revision", 0, 1, true, false)
if err != nil || len(candidates) != 1 || candidates[0].ID != c.ID {
t.Fatalf("empty checkpoint skipped rebuild: %+v %v", candidates, err)
}
}
func TestGraphDispatchDoesNotRepeatAfterRedisRecordDisappears(t *testing.T) {
path := filepath.Join(t.TempDir(), "recovery.db")
db := graphTestDB(t, path)
c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
taskID := graphRecoveryTaskID(1, "kb", c, "revision")
reserved, err := reserveGraphDispatch(db, 1, "kb", "revision", taskID, c)
if err != nil || !reserved {
t.Fatalf("reserve: %t %v", reserved, err)
}
sqlDB, _ := db.DB()
if err := sqlDB.Close(); err != nil {
t.Fatal(err)
}
db = graphTestDB(t, path)
reserved, err = reserveGraphDispatch(db, 1, "kb", "revision", taskID, c)
if err != nil || reserved {
t.Fatalf("replayed unknown dispatch: %t %v", reserved, err)
}
failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", false)
if err != nil || len(failures) != 1 || failures[0].Reason != "task_missing_after_dispatch" {
t.Fatalf("lost missing task: %+v %v", failures, err)
}
}
type graphTestQueue struct {
runs int
err error
}
func (q *graphTestQueue) RunTask(_, _ string) error { q.runs++; return q.err }
func TestGraphManualRetryIsBoundedAndAuditedAcrossRestart(t *testing.T) {
path := filepath.Join(t.TempDir(), "recovery.db")
db := graphTestDB(t, path)
c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
taskID := graphRecoveryTaskID(1, "kb", c, "revision")
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
t.Fatal(err)
}
queue := &graphTestQueue{}
if retried, err := graphFailureRetry(db, queue, taskID, "too short", 1); err == nil || retried || queue.runs != 0 {
t.Fatal("unaudited retry was accepted")
}
if retried, err := graphFailureRetry(db, queue, taskID, "Provider configuration investigated and fixed", 0); err != nil || retried || queue.runs != 0 {
t.Fatal("queue cap was ignored")
}
retried, err := graphFailureRetry(db, queue, taskID, "Provider configuration investigated and fixed", 1)
if err != nil || !retried || queue.runs != 1 {
t.Fatalf("first retry: %t %v %+v", retried, err, queue)
}
sqlDB, _ := db.DB()
if err := sqlDB.Close(); err != nil {
t.Fatal(err)
}
db = graphTestDB(t, path)
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
t.Fatal(err)
}
retried, err = graphFailureRetry(db, queue, taskID, "Retry repeatedly after restart", 1)
if err != nil || retried || queue.runs != 1 {
t.Fatalf("retry was repeated: %t %v %+v", retried, err, queue)
}
failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", false)
if err != nil || len(failures) != 1 || failures[0].ManualRetries != 1 || failures[0].RetryReason != "Provider configuration investigated and fixed" {
t.Fatalf("lost retry audit: %+v %v", failures, err)
}
}
func TestGraphRetryDispatchErrorKeepsReservation(t *testing.T) {
db := graphTestDB(t, filepath.Join(t.TempDir(), "recovery.db"))
c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
taskID := graphRecoveryTaskID(1, "kb", c, "revision")
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
t.Fatal(err)
}
queue := &graphTestQueue{err: errors.New("lost response")}
if retried, err := graphFailureRetry(db, queue, taskID, "Investigated timeout and fixed provider", 1); err == nil || retried {
t.Fatal("dispatch error not propagated")
}
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
t.Fatal(err)
}
if retried, err := graphFailureRetry(db, queue, taskID, "Investigated timeout and fixed provider", 1); err != nil || retried || queue.runs != 1 {
t.Fatal("ambiguous dispatch was replayed")
}
}
func TestGraphFailureReconciliationRequiresCurrentNativeCheckpoint(t *testing.T) {
db := graphTestDB(t, filepath.Join(t.TempDir(), "recovery.db"))
c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
taskID := graphRecoveryTaskID(1, "kb", c, "revision")
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "completed_without_native_checkpoint"); err != nil {
t.Fatal(err)
}
saveGraphCheckpoint(t, db, 1, "kb", c, "stale-content-hash")
complete, err := graphCandidateCompleted(db, 1, "kb", c)
if err != nil || complete {
t.Fatalf("stale checkpoint was accepted: %t %v", complete, err)
}
failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", true)
if err != nil || len(failures) != 1 {
t.Fatalf("prematurely resolved: %+v %v", failures, err)
}
saveGraphCheckpoint(t, db, 1, "kb", c, c.ContentHash)
complete, err = graphCandidateCompleted(db, 1, "kb", c)
if err != nil || !complete {
t.Fatalf("racing checkpoint not detected: %t %v", complete, err)
}
failures, err = graphUnresolvedFailures(db, 1, "kb", "revision", true)
if err != nil || len(failures) != 0 {
t.Fatalf("completed source remained blocked: %+v %v", failures, err)
}
var historical graphRecoveryFailure
if err := db.First(&historical, "task_id=?", taskID).Error; err != nil || historical.State != "resolved" || historical.Reason != "completed_without_native_checkpoint" {
t.Fatalf("failure history lost: %+v %v", historical, err)
}
}
func TestGraphChangedSourceSupersedesOldFailure(t *testing.T) {
db := graphTestDB(t, filepath.Join(t.TempDir(), "recovery.db"))
c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
taskID := graphRecoveryTaskID(1, "kb", c, "revision")
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
t.Fatal(err)
}
if err := db.Exec("UPDATE chunks SET content='new content'").Error; err != nil {
t.Fatal(err)
}
failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", true)
if err != nil || len(failures) != 0 {
t.Fatalf("obsolete failure blocked new content: %+v %v", failures, err)
}
candidates, err := graphRecoveryCandidates(db, 1, "kb", "revision", 0, 1, true, false)
if err != nil || len(candidates) != 1 || candidates[0].ContentHash == c.ContentHash {
t.Fatalf("new content unavailable: %+v %v", candidates, err)
}
var historical graphRecoveryFailure
if err := db.First(&historical, "task_id=?", taskID).Error; err != nil || historical.State != "superseded" {
t.Fatalf("history not preserved: %+v %v", historical, err)
}
}
func TestGraphQueueWindowCountsEveryOutstandingState(t *testing.T) {
for _, test := range []struct {
name string
info *asynq.QueueInfo
limit, want int
}{
{"empty", &asynq.QueueInfo{}, 2, 2},
{"pending", &asynq.QueueInfo{Pending: 1}, 2, 1},
{"active", &asynq.QueueInfo{Active: 2}, 2, 0},
{"scheduled", &asynq.QueueInfo{Scheduled: 2}, 2, 0},
{"retry", &asynq.QueueInfo{Retry: 2}, 2, 0},
{"aggregating", &asynq.QueueInfo{Aggregating: 2}, 2, 0},
{"combined", &asynq.QueueInfo{Active: 1, Pending: 1, Retry: 1}, 4, 1},
{"exceeded", &asynq.QueueInfo{Active: 4}, 2, 0},
{"terminal records do not block later work", &asynq.QueueInfo{Archived: 100, Completed: 100}, 2, 2},
{"paused", &asynq.QueueInfo{Paused: true}, 2, 0},
{"unknown", nil, 2, 0},
} {
t.Run(test.name, func(t *testing.T) {
if got := graphQueueSlots(test.info, test.limit); got != test.want {
t.Fatalf("slots=%d want=%d", got, test.want)
}
})
}
}
func TestGraphTaskIdentityScopesContentTenantAndRevision(t *testing.T) {
c := graphRecoveryCandidate{ID: "chunk", ContentHash: "sha256"}
id := graphRecoveryTaskID(1, "kb", c, "r1")
if id != graphRecoveryTaskID(1, "kb", c, "r1") {
t.Fatal("identity changed across restart")
}
for _, other := range []string{
graphRecoveryTaskID(2, "kb", c, "r1"), graphRecoveryTaskID(1, "other", c, "r1"), graphRecoveryTaskID(1, "kb", c, "r2"),
graphRecoveryTaskID(1, "kb", graphRecoveryCandidate{ID: "chunk", ContentHash: "changed"}, "r1"),
} {
if id == other {
t.Fatal("unrelated attempts share task identity")
}
}
}
func TestGraphResumeCursorDoesNotSkipUndispatchedOrUnverifiedWork(t *testing.T) {
candidates := []graphRecoveryCandidate{{ID: "first", SeqID: 10}, {ID: "second", SeqID: 20}, {ID: "third", SeqID: 30}}
for _, test := range []struct {
name string
settled map[string]bool
want int64
}{
{"queued but not yet verified", map[string]bool{}, 5},
{"second deferred by queue limit", map[string]bool{"first": true, "third": true}, 10},
{"all durably handled", map[string]bool{"first": true, "second": true, "third": true}, 30},
} {
t.Run(test.name, func(t *testing.T) {
if got := graphResumeCursor(5, candidates, test.settled); got != test.want {
t.Fatalf("cursor=%d want=%d", got, test.want)
}
})
}
}
+335
View File
@@ -0,0 +1,335 @@
#!/usr/bin/env node
'use strict'
const assert = require('node:assert/strict')
const http = require('node:http')
const crypto = require('node:crypto')
const { readFileSync, writeFileSync, statSync } = require('node:fs')
const path = require('node:path')
// Requires two disposable users and an already-enabled disposable workspace.
// It never changes workspace switches, registers users, or resets a database.
const base = (process.env.MEMORY_SMOKE_BASE || 'http://127.0.0.1:18080/api/v1').replace(/\/$/, '')
const run = process.env.MEMORY_SMOKE_RUN || `memory-smoke-${crypto.randomBytes(8).toString('hex')}`
assert.match(run, /^memory-smoke-[a-f0-9]{16}$/)
const manifestPath = path.resolve(process.env.MEMORY_SMOKE_MANIFEST || `${run}.json`)
const manifest = { run, base, started_at: new Date().toISOString(), resources: [], jobs: [], memories: [], candidates: [], checks: [], cleanup_errors: [] }
const captured = []
let owner, outsider, authorID, server, task, flow, model, author
const persist = () => writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 })
const pause = ms => new Promise(resolve => setTimeout(resolve, ms))
const sqlString = value => `'${String(value).replaceAll("'", "''")}'`
function credentials(file) {
assert.ok(file, 'Set MEMORY_SMOKE_OWNER_AUTH and MEMORY_SMOKE_OUTSIDER_AUTH to local JSON auth files')
assert.equal(statSync(file).mode & 0o077, 0, 'Auth file must not be readable by other users')
const value = JSON.parse(readFileSync(file, 'utf8'))
assert.equal(typeof value.token, 'string', 'Auth file requires token')
return value
}
async function request(route, body, method = body === undefined ? 'GET' : 'POST', statuses = [200], identity = owner) {
const headers = { 'Content-Type': 'application/json', Authorization: `Bearer ${identity.token}` }
if (identity.tenantID) headers['X-Tenant-ID'] = String(identity.tenantID)
const response = await fetch(base + route, { method, headers, body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(15000) })
const text = await response.text()
if (response.status === 204 && statuses.includes(204)) return {}
let value
try { value = JSON.parse(text) } catch { throw new Error(`${method} ${route}: non-JSON HTTP ${response.status}`) }
assert.ok(statuses.includes(response.status), `${method} ${route}: HTTP ${response.status}; ${JSON.stringify(value.error || value.message || '').slice(0, 300)}`)
return value
}
async function listCandidates() {
const response = await request('/workbench/operations/records/memories?page_size=100')
const rows = (response.data || []).filter(row => task && row.source_ref.startsWith(`内容任务/${task.id}/`))
for (const row of rows) if (!manifest.candidates.includes(row.id)) manifest.candidates.push(row.id)
persist()
return rows
}
async function listMemories() {
const response = await request('/memory/items?limit=200')
for (const row of response.data || []) {
if (row.content.includes(run) && !manifest.memories.includes(row.id)) manifest.memories.push(row.id)
}
persist()
return response.data || []
}
async function reconfirmLegacyFixture() {
if (process.env.MEMORY_SMOKE_LEGACY !== '1') return null
assert.equal(typeof process.send, 'function', 'Legacy fixture requires the isolated workspace wrapper')
const seeded = await new Promise((resolve, reject) => {
const timer = setTimeout(() => { process.off('message', onMessage); reject(new Error('Legacy fixture seed timed out')) }, 15000)
function onMessage(message) {
if (message?.type !== 'legacy-memory-seeded') return
clearTimeout(timer)
process.off('message', onMessage)
if (message.error) reject(new Error(message.error))
else resolve(message)
}
process.on('message', onMessage)
process.send({ type: 'seed-legacy-memory', run, agent_id: authorID })
})
assert.match(seeded.id, /^[a-f0-9-]{36}$/)
manifest.candidates.push(seeded.id)
manifest.legacy_history_id = seeded.history_id
persist()
const find = async () => (await request('/workbench/operations/records/memories?page_size=100')).data.find(row => row.id === seeded.id)
const before = await find()
assert.equal(before.status, 'accepted')
assert.equal(before.legacy_history_id, seeded.history_id)
assert.equal(before.can_reconfirm, true)
assert.ok(!before.native_memory_id)
const content = `${run}:解释方案时先给判断标准,再给两条示例。`
const body = { status: 'accepted', content, reconfirm_legacy: true, confirm_personal_scope: true }
const route = `/workbench/operations/memories/${seeded.id}`
await request(route, body, 'POST', [403, 404], outsider)
await request(route, { ...body, confirm_personal_scope: false }, 'POST', [400])
await request(route, body)
await request(route, body)
const after = await find()
assert.equal(after.content, before.content, 'Legacy original text was overwritten')
assert.equal(after.source_ref, before.source_ref, 'Legacy provenance was overwritten')
assert.equal(after.reconfirmed_content, content)
assert.equal(after.can_reconfirm, false)
assert.ok(after.reconfirmed_at && after.native_memory_id)
const linked = (await listMemories()).filter(row => row.source_message_id === seeded.id)
assert.equal(linked.length, 1, 'Legacy reconfirmation was not idempotent')
assert.equal(linked[0].id, after.native_memory_id)
assert.equal(linked[0].content, content)
assert.equal(linked[0].subject_id, manifest.subject_id)
assert.equal(linked[0].status, 'active')
await request(`/memory/items/${linked[0].id}`, undefined, 'DELETE', [404], outsider)
assert.ok(!(await request('/memory/items?limit=200', undefined, 'GET', [200], outsider)).data.some(row => row.id === linked[0].id))
manifest.checks.push('Legacy owner reconfirmation requires consent, preserves provenance, stores one native preference and rejects foreign access')
persist()
return { id: linked[0].id, candidate_id: seeded.id, route, body, content, find }
}
async function finishLegacyFixture(legacy) {
if (!legacy) return
assert.ok(captured.at(-1).system.includes(legacy.content), 'Reconfirmed legacy preference was missing from actual generation prompt')
await request(`/memory/items/${legacy.id}`, undefined, 'DELETE', [200, 204])
await request(legacy.route, legacy.body)
assert.equal((await legacy.find()).native_memory_id, legacy.id, 'Legacy link was removed after native deletion')
assert.ok(!(await listMemories()).some(row => row.id === legacy.id || row.source_message_id === legacy.candidate_id), 'Repeated confirmation resurrected deleted native memory')
manifest.checks.push('Reconfirmed legacy preference reaches actual generation prompt; deliberate native deletion remains deleted after repeat confirmation')
persist()
}
async function act(action, extra = {}, expectedStatus = 'succeeded') {
const job = (await request(`/workbench/resources/content-tasks/${task.id}/actions/${action}`, { version: task.version, ...extra }, 'POST', [202])).data
manifest.jobs.push(job.id)
persist()
const deadline = Date.now() + 90000
while (Date.now() < deadline) {
const current = (await request(`/workbench/jobs/${job.id}`)).data
if (!['queued', 'running'].includes(current.status)) {
assert.equal(current.status, expectedStatus, `${action}: ${current.error || current.status}`)
task = (await request(`/workbench/resources/content-tasks/${task.id}`)).data
return current
}
await pause(350)
}
throw new Error(`Job timeout: ${job.id}; cleanup must wait until the job stops`)
}
function startModel() {
server = http.createServer(async (req, res) => {
try {
let text = ''
for await (const chunk of req) {
text += chunk
assert.ok(text.length < 2 * 1024 * 1024, 'Model request exceeds fixture limit')
}
const body = JSON.parse(text)
const messages = body.messages || []
const system = messages.filter(message => message.role === 'system').map(message => message.content).join('\n')
const prompt = String(messages.at(-1)?.content || '')
captured.push({ system, prompt })
let content = `测试正文 ${run}:先说明判断依据,再讨论应用步骤。`
if (system.includes('内容编排师')) content = JSON.stringify({ segments: [1, 2, 3].map(n => ({ id: `s${n}`, title: `验证步骤${n}`, node_id: 'writer', agent_id: authorID })) })
if (system.includes('内置跨稿质检员')) content = JSON.stringify({ passed: true, summary: '本地固定质检通过', issues: [] })
res.writeHead(200, { 'Content-Type': 'application/json' })
res.end(JSON.stringify({ id: run, object: 'chat.completion', created: Math.floor(Date.now() / 1000), model: run,
choices: [{ index: 0, message: { role: 'assistant', content }, finish_reason: 'stop' }], usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 } }))
} catch {
res.writeHead(400, { 'Content-Type': 'application/json' })
res.end(JSON.stringify({ error: { message: 'Invalid smoke-model request' } }))
}
})
return new Promise((resolve, reject) => {
server.once('error', reject)
server.listen(0, '0.0.0.0', () => resolve(server.address().port))
})
}
async function preflight() {
const url = new URL(base)
assert.ok(['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname), 'Use the loopback origin; public URLs are refused')
owner = credentials(process.env.MEMORY_SMOKE_OWNER_AUTH)
outsider = credentials(process.env.MEMORY_SMOKE_OUTSIDER_AUTH)
const me = (await request('/auth/me')).data
const other = (await request('/auth/me', undefined, 'GET', [200], outsider)).data
assert.ok(me.user.username.startsWith('memory-smoke-'), 'Owner must be a dedicated memory-smoke- user')
assert.ok(me.tenant.name.startsWith('memory-smoke-'), 'Workspace must be a dedicated memory-smoke- workspace')
assert.ok(other.user.username.startsWith('memory-smoke-'), 'Outsider must be a dedicated memory-smoke- user')
assert.notEqual(me.user.id, other.user.id)
manifest.tenant_id = me.tenant.id
manifest.owner_id = me.user.id
manifest.subject_id = `web_user:${me.user.id}`
const settings = (await request('/memory/settings')).data
assert.equal(settings.effective, true, 'Enable memory on the disposable workspace/user before running')
assert.equal((await request('/memory/items?limit=1')).total, 0, 'Owner already has memory; refuse to alter it')
assert.equal((await request('/workbench/operations/records/memories?page_size=1')).total, 0, 'Owner already has review history; use another fixture user')
for (const kind of ['flows', 'content-tasks', 'training-maps']) {
const rows = (await request(`/workbench/resources/${kind}`)).data || []
assert.equal(rows.length, 0, `Fixture workspace already has ${kind}`)
}
persist()
}
async function main() {
await preflight()
const port = await startModel()
const modelHost = process.env.MEMORY_SMOKE_MODEL_HOST || 'host.docker.internal'
assert.ok(['host.docker.internal', '127.0.0.1', 'localhost'].includes(modelHost), 'Fake-model host must resolve to this machine')
model = (await request('/models', { name: run, type: 'KnowledgeQA', source: 'remote', parameters: { base_url: `http://${modelHost}:${port}/v1`, api_key: 'local-smoke-only', provider: 'generic' } }, 'POST', [201])).data
manifest.model_id = model.id
persist()
author = (await request('/agents', { name: run, config: { model_id: model.id, system_prompt: '测试内容作者', role: 'production', agent_mode: 'quick-answer', memory_enabled: true } }, 'POST', [201])).data
authorID = author.id
manifest.agent_id = authorID
persist()
flow = (await request('/workbench/resources/flows', { name: run, data: { nodes: [{ id: 'writer', agent_id: authorID, label: '测试作者' }], edges: [] } }, 'POST', [201])).data
manifest.resources.push({ id: flow.id, kind: 'flows' })
persist()
task = (await request('/workbench/resources/content-tasks', { name: run, data: { day: 1, flow_id: flow.id, agent_id: authorID, source_text: '判断依据、实际应用、总结回顾', knowledge_base_ids: [], duration_minutes: 135 } }, 'POST', [201])).data
manifest.resources.push({ id: task.id, kind: 'content-tasks' })
persist()
assert.equal(task.data.duration_minutes, 135, 'Create lost duration')
task = (await request(`/workbench/resources/content-tasks/${task.id}`, { name: task.name, version: task.version, data: { ...task.data, duration_minutes: 90 } }, 'PUT')).data
assert.equal(task.data.duration_minutes, 90, 'Update lost duration')
for (const invalid of [-1, 0.5, 1441]) {
await request(`/workbench/resources/content-tasks/${task.id}`, { name: task.name, version: task.version, data: { ...task.data, duration_minutes: invalid } }, 'PUT', [400])
assert.equal((await request(`/workbench/resources/content-tasks/${task.id}`)).data.data.duration_minutes, 90, 'Invalid duration changed persisted state')
}
manifest.checks.push('Duration persists 135-minute create and 90-minute update; negative/fractional/over-1440 updates are rejected')
await act('decompose')
assert.equal(task.data.segments.length, 3)
const advice = `${run}:回答先列要点,再给例子。`
await act('generate', { segment_id: 's1', advice })
assert.equal((await listCandidates()).length, 0, 'Unaccepted draft created feedback memory')
assert.equal((await listMemories()).length, 0, 'AI draft became a user memory')
await act('accept', { segment_id: 's1' })
let rows = await listCandidates()
assert.equal(rows.length, 1)
const candidate = rows[0]
assert.equal(candidate.content, advice)
assert.equal(candidate.status, 'pending')
await act('accept', { segment_id: 's1' })
assert.equal((await listCandidates()).length, 1, 'Repeated adoption duplicated feedback')
await request(`/workbench/operations/memories/${candidate.id}`, { status: 'accepted' }, 'POST', [403, 404], outsider)
await request(`/workbench/operations/memories/${candidate.id}`, { status: 'accepted' })
const memories = await listMemories()
assert.equal(memories.length, 1)
assert.equal(memories[0].content, advice)
assert.equal(memories[0].status, 'active')
manifest.subject_id = memories[0].subject_id
await request(`/memory/items/${memories[0].id}`, undefined, 'DELETE', [404], outsider)
const otherRows = (await request('/memory/items?limit=200', undefined, 'GET', [200], outsider)).data || []
assert.ok(!otherRows.some(row => row.id === memories[0].id))
manifest.checks.push('Draft is excluded; accepted human advice is durable, idempotent and owner isolated; review writes active memory')
const legacy = await reconfirmLegacyFixture()
await act('generate', { segment_id: 's2', advice: `${run}:第二段使用简短段落。` })
assert.ok(captured.at(-1).system.includes(advice), 'Approved memory did not reach the actual fake-model prompt')
await finishLegacyFixture(legacy)
await act('accept', { segment_id: 's2' })
const skipped = (await listCandidates()).find(row => row.source_ref.endsWith('/内容段/s2'))
assert.equal(skipped.status, 'pending')
await act('skip', { segment_id: 's2' })
assert.equal((await listCandidates()).find(row => row.id === skipped.id).status, 'invalidated')
await request(`/workbench/operations/memories/${skipped.id}`, { status: 'accepted' }, 'POST', [409])
await act('generate', { segment_id: 's3', advice: `${run}:总结保留清晰步骤。` })
await act('accept', { segment_id: 's3' })
const deleted = (await listCandidates()).find(row => row.source_ref.endsWith('/内容段/s3'))
const originalProse = task.data.segments.map(row => row.content)
await act('review')
assert.equal(task.data.quality.duration_minutes, 90)
await act('compose')
assert.equal(task.data.composed, task.data.segments.filter(row => row.status === 'accepted').map(row => row.content).join('\n\n'))
const callsBeforeDurationChange = captured.length
task = (await request(`/workbench/resources/content-tasks/${task.id}`, { name: task.name, version: task.version, data: { ...task.data, duration_minutes: 225 } }, 'PUT')).data
task = (await request(`/workbench/resources/content-tasks/${task.id}`)).data
assert.equal(task.data.duration_minutes, 225)
assert.deepEqual(task.data.segments.map(row => row.content), originalProse, 'Duration change lost saved prose')
assert.deepEqual(task.data.segments.map(row => row.status), ['needs_review', 'skipped', 'needs_review'])
assert.ok(!task.data.quality && !task.data.composed && !task.data.review, 'Duration change reused old quality approval')
const blockedCompose = await act('compose', {}, 'failed')
assert.ok(blockedCompose.error.includes('重新确认'), 'Composition failed for an unexpected reason')
assert.equal(captured.length, callsBeforeDurationChange, 'Duration update or rejected composition invoked a model')
assert.equal((await listCandidates()).find(row => row.id === deleted.id).status, 'invalidated')
await act('accept', { segment_id: 's1' })
await act('accept', { segment_id: 's3' })
assert.equal((await listCandidates()).find(row => row.id === deleted.id).status, 'pending', 'Reacceptance did not restore pending source')
assert.equal((await listCandidates()).find(row => row.id === candidate.id).status, 'accepted', 'Reacceptance reset an approved memory')
manifest.checks.push('Changing a composed task duration preserves prose, invalidates quality and pending sources, blocks stale composition, and reacceptance restores only pending feedback')
await request(`/workbench/resources/content-tasks/${task.id}`, undefined, 'DELETE')
assert.equal((await listCandidates()).find(row => row.id === deleted.id).status, 'invalidated')
assert.equal((await listMemories()).length, 1, 'Source deletion altered separately approved memory')
manifest.checks.push('Approved preference is recalled; skip/deletion invalidate pending feedback without undoing explicit approval')
manifest.model_calls = captured.length
manifest.passed = true
persist()
}
async function cleanup() {
if (!owner || !manifest.owner_id) return
// Never delete while a worker can still write to the fixture resources.
for (const id of manifest.jobs) {
try {
const row = (await request(`/workbench/jobs/${id}`)).data
if (['queued', 'running'].includes(row.status)) {
manifest.cleanup_errors.push(`Job ${id} is still ${row.status}; fixture retained for bounded cleanup`)
persist()
return
}
} catch (error) { manifest.cleanup_errors.push(error.message); persist(); return }
}
try { await listMemories(); await listCandidates() } catch (error) { manifest.cleanup_errors.push(error.message) }
const remove = async route => {
try { await request(route, undefined, 'DELETE', [200, 204, 404]) } catch (error) { manifest.cleanup_errors.push(error.message) }
}
for (const id of manifest.memories) await remove(`/memory/items/${id}`)
for (const resource of [...manifest.resources].reverse()) await remove(`/workbench/resources/${resource.kind}/${resource.id}`)
if (author) await remove(`/agents/${author.id}`)
if (model) await remove(`/models/${model.id}`)
const tenant = Number(manifest.tenant_id)
assert.ok(Number.isSafeInteger(tenant) && tenant > 0)
const ownerID = sqlString(manifest.owner_id)
const resourceIDs = manifest.resources.map(row => sqlString(row.id)).join(',') || "''"
const candidateIDs = manifest.candidates.map(sqlString).join(',') || "''"
const subject = sqlString(manifest.subject_id)
// API deletion intentionally retains audit/history and rejection tombstones.
// This optional exact-ID SQL removes only this run's workbench history.
const cleanupSQL = `BEGIN;\nSET LOCAL lock_timeout = '3s';\nDO $$ BEGIN\n IF EXISTS (SELECT 1 FROM workbench_jobs WHERE tenant_id=${tenant} AND owner_id=${ownerID} AND resource_id IN (${resourceIDs}) AND status IN ('queued','running')) THEN RAISE EXCEPTION 'Fixture jobs still active'; END IF;\nEND $$;\nDELETE FROM workbench_pending_memories WHERE tenant_id=${tenant} AND owner_id=${ownerID} AND id IN (${candidateIDs});\nDELETE FROM workbench_revisions WHERE tenant_id=${tenant} AND resource_id IN (${resourceIDs});\nDELETE FROM workbench_events WHERE tenant_id=${tenant} AND owner_id=${ownerID} AND resource_id IN (${resourceIDs});\nDELETE FROM workbench_jobs WHERE tenant_id=${tenant} AND owner_id=${ownerID} AND resource_id IN (${resourceIDs});\nDELETE FROM memory_tombstones WHERE tenant_id=${tenant} AND subject_id=${subject} AND source_message_id IN (${candidateIDs});\nCOMMIT;\n`
writeFileSync(`${manifestPath}.cleanup.sql`, cleanupSQL, { mode: 0o600 })
manifest.finished_at = new Date().toISOString()
manifest.cleanup_note = 'Created resources and native memories are API-deleted. Exact-ID cleanup SQL is supplied for review history/tombstones; audit/usage logs and fixture identities remain for verification. No original user data was deleted.'
persist()
}
if (process.argv.includes('--run')) {
main().catch(error => { manifest.error = error.message; manifest.passed = false; process.exitCode = 1 }).finally(async () => {
try { await cleanup() } catch (error) { manifest.cleanup_errors.push(error.message); process.exitCode = 1; persist() }
server?.close()
if (process.connected) process.disconnect()
if (manifest.cleanup_errors.length) process.exitCode = 1
console.log(JSON.stringify({ passed: manifest.passed || false, checks: manifest.checks, error: manifest.error, cleanup_errors: manifest.cleanup_errors, manifest: manifestPath }, null, 2))
})
} else {
console.log('Set MEMORY_SMOKE_OWNER_AUTH and MEMORY_SMOKE_OUTSIDER_AUTH (token JSON files), optionally MEMORY_SMOKE_BASE/MEMORY_SMOKE_MODEL_HOST/MEMORY_SMOKE_MANIFEST, then run with --run. Requires a disposable memory-smoke- workspace and two memory-smoke- users; never registers or resets anything. See scripts/native_memory_api_smoke.md.')
}
+111
View File
@@ -0,0 +1,111 @@
# Native memory API smoke fixture
This script verifies the actual deployed HTTP API, workbench job leases/worker,
database persistence, memory review and the prompt sent to a deterministic local
fake model. It does not benchmark AI quality or invoke any existing model.
## Automatic disposable workspace
Use the wrapper to provision and clean up the required fixture:
```sh
node scripts/native_memory_fixture.cjs --run
```
It reads the existing private admin login file at
`/Volumes/Extreme SSD/yawan-storage/weknora/acceptance/super-admin-login.json`.
Override that path with `MEMORY_SMOKE_ADMIN_AUTH`. The file must have no group or
other read permissions. The wrapper creates two uniquely named disposable users
and an empty workspace, enables explicit-only memory with vector recall disabled,
and gives only the fixture owner a disposable balance. The owner has workspace
owner privileges to create the local fake model; the outsider remains an ordinary
contributor. Neither user is a system administrator.
Credentials are written only into mode-0600 files within a mode-0700 run directory.
The manifest is credential-free and is saved at each setup stage. The wrapper runs
the smoke script, then checks no fixture workbench job remains active and removes
only the new workspace, fixture users, auth tokens, balance, model, Agent, memory
and workbench rows. It validates both workspace name/business marker and exact
user identities before cleanup. A surviving smoke process, unexpected tables or foreign memberships stop
cleanup for inspection. Administrator audit entries in the original workspace
remain as evidence. There is no database reset, and no real provider is called.
If interrupted, rerun just the exact fixture cleanup:
```sh
node scripts/native_memory_fixture.cjs --cleanup /absolute/path/memory-smoke-RUN/fixture.json
```
The wrapper defaults to Docker context `colima-yawan`, container
`WeKnora-postgres`, database `yawan_native_acceptance_20260915` and source tenant
`10059`. This database is production despite its historical name. Connection
overrides are `MEMORY_SMOKE_DOCKER`, `MEMORY_SMOKE_DOCKER_CONTEXT`,
`MEMORY_SMOKE_DB_CONTAINER`, `MEMORY_SMOKE_DB_NAME` and
`MEMORY_SMOKE_SOURCE_TENANT`. Cleanup requires the same recorded connection.
`MEMORY_SMOKE_DIRECTORY` changes the parent directory for fixture reports.
The wrapper is intended to be run after deployment by the release operator.
The wrapper also seeds one synthetic accepted legacy history row through a
private child-process channel, after checking the new Agent's exact owner,
workspace, name and memory setting. This is a new fixture record, never a real
user's imported history. After migration 217, the actual API must require owner
scope consent, preserve the original text/source, create exactly one linked native
preference, include it in the next real generation request to the local stub, and
reject access from the other user. Deleting that native preference then repeating
confirmation must not restore it. The dedicated workspace cleanup removes this
synthetic history too. Provenance matching during migration is tested separately
with an isolated PostgreSQL database. Standalone existing-fixture mode does not
seed or claim coverage of legacy reconfirmation.
## Existing fixture mode
Requirements:
- Node 20+ and the local backend origin after deployment.
- A dedicated empty workspace whose name starts with `memory-smoke-`, with memory enabled.
- Two disposable users whose usernames start with `memory-smoke-`. The owner must be able to create a model, Agent, flow and content task. The outsider must not be a system administrator. The outsider may share the fixture workspace or use another empty fixture workspace.
- Two local mode-0600 JSON auth files with `{"token":"..."}`. Optional `tenantID` selects the active workspace through `X-Tenant-ID`. Do not put tokens in commands, source control or the result report.
```sh
MEMORY_SMOKE_OWNER_AUTH=/absolute/path/owner-auth.json \
MEMORY_SMOKE_OUTSIDER_AUTH=/absolute/path/outsider-auth.json \
MEMORY_SMOKE_MANIFEST=/absolute/path/memory-smoke-result.json \
node scripts/native_memory_api_smoke.cjs --run
```
The default origin is `http://127.0.0.1:18080/api/v1`; override with
`MEMORY_SMOKE_BASE`. Public hostnames are refused. The model stub listens on a
random local host port so the backend container can reach it through
`host.docker.internal`. Set `MEMORY_SMOKE_MODEL_HOST=127.0.0.1` for a backend
running directly on the host. The stub receives only fixture content.
The script checks 135-minute task creation, 90-minute update, persistence, and
rejection of negative/fractional/over-1440 durations without mutating saved state.
It also checks that draft content does not become memory, adopting human
advice creates one pending candidate, repeated adoption deduplicates it,
another account cannot approve or delete it, explicit review writes an active
memory, the next generation actually recalls that preference, and skip/source
deletion invalidate only pending candidates. It also runs quality review and
exact ordered composition, changes the duration to 225 minutes, verifies the
original prose is retained while old review/composition results are cleared,
and checks that composition is blocked until renewed confirmation and review.
Reacceptance restores invalidated pending feedback without resetting explicitly
approved memories. It refuses nonempty owner memory
or content fixtures and does not alter workspace settings.
Cleanup uses exact created resource/memory IDs through APIs. It waits for every
recorded job to be terminal first. On job timeout it deliberately retains the
fixture and reports the IDs instead of deleting beneath a running worker. The
manifest is written after every created object, contains no token, and supports
manual recovery. API deletion leaves workbench review/history records and
memory tombstones; the companion `.cleanup.sql` removes only this run's exact
candidate/resource IDs within its tenant/owner scope. It refuses active jobs.
Run that SQL against the matching database after reviewing the manifest. Audit
and usage logs and the disposable accounts/workspace remain as test evidence;
remove the dedicated fixture identities through the normal administration
workflow when the evidence is no longer needed. No broad database reset is used.
This fixture tests native content feedback and recall. Chat extraction retry,
timestamp pagination, per-session checkpoints and pending conflict rollback
are covered by the offline Go integration suites; this script does not claim
to simulate those asynchronous chat failures.
+278
View File
@@ -0,0 +1,278 @@
#!/usr/bin/env node
'use strict'
const assert = require('node:assert/strict')
const crypto = require('node:crypto')
const fs = require('node:fs')
const path = require('node:path')
const { execFileSync, spawn } = require('node:child_process')
const acceptance = process.env.MEMORY_SMOKE_DIRECTORY || '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
const database = {
context: process.env.MEMORY_SMOKE_DOCKER_CONTEXT || 'colima-yawan',
container: process.env.MEMORY_SMOKE_DB_CONTAINER || 'WeKnora-postgres',
name: process.env.MEMORY_SMOKE_DB_NAME || 'yawan_native_acceptance_20260915',
}
const literal = value => `'${String(value).replaceAll("'", "''")}'`
let manifest, manifestPath, admin
function privateJSON(file, value) {
const temporary = `${file}.${crypto.randomBytes(4).toString('hex')}.tmp`
const fd = fs.openSync(temporary, 'wx', 0o600)
try {
fs.writeFileSync(fd, `${JSON.stringify(value, null, 2)}\n`)
fs.fsyncSync(fd)
} finally { fs.closeSync(fd) }
fs.renameSync(temporary, file)
}
const persist = () => privateJSON(manifestPath, manifest)
function sql(query) {
try {
return execFileSync(process.env.MEMORY_SMOKE_DOCKER || '/opt/homebrew/bin/docker', [
'--context', database.context, 'exec', '-i', database.container,
'psql', '-X', '-q', '-U', 'postgres', '-d', database.name,
'-At', '-v', 'ON_ERROR_STOP=1',
], { input: query, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 30000 }).trim()
} catch {
// SQL stderr may contain request fields. Keep credentials out of reports.
throw new Error('Fixture database operation failed; inspect the bounded fixture SQL and database locally')
}
}
async function api(route, body, identity = admin, tenant = manifest.source_tenant_id) {
const response = await fetch(manifest.base + route, {
method: body === undefined ? 'GET' : 'POST',
headers: { 'Content-Type': 'application/json', ...(identity ? { Authorization: `Bearer ${identity.token}`, 'X-Tenant-ID': String(tenant) } : {}) },
body: body === undefined ? undefined : JSON.stringify(body),
signal: AbortSignal.timeout(15000),
})
assert.ok(response.ok, `${route}: HTTP ${response.status}`)
return response.json()
}
function validateManifest() {
assert.equal(manifest.format, 'native-memory-fixture-v1')
assert.match(manifest.run, /^memory-smoke-[a-f0-9]{16}$/)
assert.deepEqual(manifest.database, database, 'Use the same database connection as the fixture manifest')
assert.ok(Number.isSafeInteger(manifest.source_tenant_id) && manifest.source_tenant_id > 0)
if (manifest.tenant_id !== undefined) {
assert.ok(Number.isSafeInteger(manifest.tenant_id) && manifest.tenant_id > 0 && manifest.tenant_id !== manifest.source_tenant_id)
}
assert.equal(manifest.users.length, 2)
for (const [index, label] of ['owner', 'outsider'].entries()) {
assert.equal(manifest.users[index].username, `${manifest.run}-${label}`)
assert.equal(manifest.users[index].email, `${manifest.run}-${label}@example.test`)
}
const url = new URL(manifest.base)
assert.ok(['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname), 'Only loopback API origins are permitted')
}
async function provision() {
const adminFile = process.env.MEMORY_SMOKE_ADMIN_AUTH || path.join(acceptance, 'super-admin-login.json')
assert.equal(fs.statSync(adminFile).mode & 0o077, 0, 'Admin credentials must be a private file (mode 0600)')
const credentials = JSON.parse(fs.readFileSync(adminFile, 'utf8'))
admin = await api('/auth/login', { email: credentials.email, password: credentials.password }, null)
assert.equal(typeof admin.token, 'string')
// The random name and business marker allow exact recovery after interruption.
manifest.tenant_id = Number(sql(`INSERT INTO tenants(name,business,status,memory_config)
VALUES (${literal(manifest.run)},'native-memory-smoke-fixture','active',
'{"enabled":true,"write_mode":"explicit_only","vector_recall":false,"embedding_model_id":""}') RETURNING id;`))
assert.ok(Number.isSafeInteger(manifest.tenant_id) && manifest.tenant_id > 0 && manifest.tenant_id !== manifest.source_tenant_id)
persist()
for (const user of manifest.users) {
const password = crypto.randomBytes(18).toString('base64url')
const row = (await api('/access/users', { username: user.username, email: user.email, password, role: 'contributor', status: 'active' })).data
user.id = row.id
persist()
const role = user === manifest.users[0] ? 'owner' : 'contributor'
sql(`BEGIN;
UPDATE users SET tenant_id=${manifest.tenant_id} WHERE id=${literal(user.id)} AND email=${literal(user.email)} AND tenant_id=${manifest.source_tenant_id};
UPDATE tenant_members SET tenant_id=${manifest.tenant_id},role=${literal(role)} WHERE user_id=${literal(user.id)} AND tenant_id=${manifest.source_tenant_id};
DELETE FROM workbench_access_assignments WHERE user_id=${literal(user.id)} AND tenant_id=${manifest.source_tenant_id};
COMMIT;`)
const identity = await api('/auth/login', { email: user.email, password }, null)
assert.equal(Number(identity.user.tenant_id), manifest.tenant_id, 'Fixture login selected a different workspace')
assert.equal(identity.user.is_system_admin, false, 'Fixture user must not be a system administrator')
privateJSON(path.join(path.dirname(manifestPath), `${user === manifest.users[0] ? 'owner' : 'outsider'}-auth.json`), { token: identity.token, tenantID: manifest.tenant_id })
if (role === 'owner') {
await api('/workbench/operations/grants', { user_id: user.id, amount: 1_000_000_000, reason: manifest.run, idempotency_key: manifest.run }, identity, manifest.tenant_id)
}
}
manifest.provisioned_at = new Date().toISOString()
persist()
}
function runSmoke() {
const directory = path.dirname(manifestPath)
return new Promise((resolve, reject) => {
const child = spawn(process.execPath, [path.join(__dirname, 'native_memory_api_smoke.cjs'), '--run'], {
env: { ...process.env, MEMORY_SMOKE_BASE: manifest.base,
MEMORY_SMOKE_RUN: manifest.run, MEMORY_SMOKE_LEGACY: '1',
MEMORY_SMOKE_OWNER_AUTH: path.join(directory, 'owner-auth.json'),
MEMORY_SMOKE_OUTSIDER_AUTH: path.join(directory, 'outsider-auth.json'),
MEMORY_SMOKE_MANIFEST: path.join(directory, 'smoke.json') },
stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
})
manifest.smoke_pid = child.pid
persist()
const timer = setTimeout(() => child.kill('SIGTERM'), 300000)
child.on('message', message => {
if (message?.type !== 'seed-legacy-memory') return
try {
validateManifest()
assert.equal(message.run, manifest.run)
assert.match(message.agent_id, /^[a-f0-9-]{36}$/)
assert.ok(!manifest.legacy_candidate, 'Only one synthetic legacy candidate is permitted')
const ownerID = manifest.users[0].id
const id = crypto.randomUUID(), historyID = crypto.randomUUID()
const original = `${manifest.run}: original legacy advice`
const source = `${manifest.run}/legacy/${historyID}`
// Seed only a synthetic accepted history row in this run's empty workspace.
// Migration provenance matching itself is verified by PostgreSQL unit tests.
sql(`BEGIN;
DO $seed$ BEGIN
IF NOT EXISTS (SELECT 1 FROM tenants WHERE id=${manifest.tenant_id} AND name=${literal(manifest.run)} AND business='native-memory-smoke-fixture') OR
NOT EXISTS (SELECT 1 FROM custom_agents WHERE id=${literal(message.agent_id)} AND tenant_id=${manifest.tenant_id} AND created_by=${literal(ownerID)} AND name=${literal(manifest.run)} AND deleted_at IS NULL AND config->>'memory_enabled'='true')
THEN RAISE EXCEPTION 'Legacy fixture scope mismatch'; END IF;
END $seed$;
INSERT INTO workbench_history(id,tenant_id,owner_id,source_kind,source_id,payload)
VALUES(${literal(historyID)},${manifest.tenant_id},${literal(ownerID)},'pendingmemory',${literal(id)},${literal(JSON.stringify({ fixture: manifest.run, content: original }))}::jsonb);
INSERT INTO workbench_pending_memories(id,tenant_id,owner_id,agent_id,content,source_ref,status,reviewed_by,legacy_history_id)
VALUES(${literal(id)},${manifest.tenant_id},${literal(ownerID)},${literal(message.agent_id)},${literal(original)},${literal(source)},'accepted',${literal(ownerID)},${literal(historyID)});
COMMIT;`)
manifest.legacy_candidate = { id, history_id: historyID, agent_id: message.agent_id }
persist()
child.send({ type: 'legacy-memory-seeded', id, history_id: historyID })
} catch (error) {
manifest.legacy_seed_error = error.message
persist()
if (child.connected) child.send({ type: 'legacy-memory-seeded', error: error.message })
}
})
child.once('error', error => { clearTimeout(timer); reject(error) })
child.once('close', (code, signal) => {
clearTimeout(timer)
manifest.smoke_exit_code = code
manifest.smoke_signal = signal
manifest.smoke_finished_at = new Date().toISOString()
manifest.passed = code === 0
persist()
resolve()
})
})
}
function cleanup() {
validateManifest()
if (manifest.smoke_pid && !manifest.smoke_finished_at) {
let running = true
try { process.kill(manifest.smoke_pid, 0) } catch (error) {
if (error.code === 'ESRCH') running = false
}
assert.equal(running, false, 'Fixture smoke process is still alive; wait for it to finish before cleanup')
}
const discovered = sql(`SELECT id FROM tenants WHERE name=${literal(manifest.run)} AND business='native-memory-smoke-fixture';`)
if (discovered) {
const tenant = Number(discovered)
assert.ok(Number.isSafeInteger(tenant) && tenant > 0 && tenant !== manifest.source_tenant_id)
if (manifest.tenant_id) assert.equal(tenant, manifest.tenant_id)
manifest.tenant_id = tenant
}
const tenant = manifest.tenant_id || 0
const identities = manifest.users.map(user => `(username=${literal(user.username)} AND email=${literal(user.email)})`).join(' OR ')
const rows = JSON.parse(sql(`SELECT COALESCE(json_agg(json_build_object('id',id,'username',username,'tenant_id',tenant_id)),'[]') FROM users WHERE ${identities};`))
for (const row of rows) {
const user = manifest.users.find(value => value.username === row.username)
if (user.id) assert.equal(user.id, row.id)
assert.ok([manifest.source_tenant_id, tenant].includes(Number(row.tenant_id)), 'Fixture user moved to an unexpected workspace')
user.id = row.id
}
persist()
const userIDs = manifest.users.filter(user => user.id).map(user => literal(user.id)).join(',') || "''"
// Unknown nonempty tables stop cleanup. This prevents silently orphaning new
// fixture dependencies when future API changes add another storage surface.
const tables = [
'workbench_pending_memories', 'workbench_revisions', 'workbench_events', 'workbench_jobs',
'workbench_resources', 'workbench_balance_holds', 'workbench_usage', 'workbench_transactions',
'workbench_balances', 'workbench_operation_logs', 'workbench_consumption', 'workbench_history',
'workbench_usage_quotas', 'workbench_resource_forks', 'workbench_resource_submissions',
'workbench_resource_hidden', 'workbench_resource_version_history',
'workbench_access_grants', 'workbench_access_publications', 'workbench_access_assignments', 'workbench_access_roles',
'memory_item_embeddings', 'memory_tombstones', 'memory_items', 'memory_topic_stats', 'memory_doc_affinity', 'memory_subjects',
'custom_agents', 'models', 'audit_logs', 'tenant_members',
]
const tableList = tables.map(literal).join(',')
const query = `BEGIN;
SET LOCAL lock_timeout = '3s';
SET LOCAL statement_timeout = '20s';
DO $fixture$
DECLARE entry record; count_rows bigint;
BEGIN
IF ${tenant} <> 0 AND EXISTS (SELECT 1 FROM tenants WHERE id=${tenant} AND (name<>${literal(manifest.run)} OR business<>'native-memory-smoke-fixture')) THEN RAISE EXCEPTION 'Fixture tenant identity mismatch'; END IF;
IF EXISTS (SELECT 1 FROM users WHERE id IN (${userIDs}) AND (is_system_admin OR can_access_all_tenants OR NOT (${identities}))) THEN RAISE EXCEPTION 'Unexpected user identity'; END IF;
IF ${tenant} > 0 AND (EXISTS (SELECT 1 FROM users WHERE tenant_id=${tenant} AND id NOT IN (${userIDs})) OR EXISTS (SELECT 1 FROM tenant_members WHERE tenant_id=${tenant} AND user_id NOT IN (${userIDs}))) THEN RAISE EXCEPTION 'Fixture has other users'; END IF;
IF EXISTS (SELECT 1 FROM tenant_members WHERE user_id IN (${userIDs}) AND tenant_id NOT IN (${tenant},${manifest.source_tenant_id})) THEN RAISE EXCEPTION 'Fixture user has foreign memberships'; END IF;
IF ${tenant} > 0 AND EXISTS (SELECT 1 FROM workbench_jobs WHERE tenant_id=${tenant} AND status IN ('queued','running')) THEN RAISE EXCEPTION 'Fixture jobs are still active'; END IF;
FOR entry IN SELECT table_name FROM information_schema.columns WHERE table_schema='public' AND column_name='tenant_id' AND table_name NOT IN (${tableList},'users') LOOP
EXECUTE format('SELECT count(*) FROM %I WHERE tenant_id=$1 AND $1>0',entry.table_name) INTO count_rows USING ${tenant};
IF count_rows > 0 THEN RAISE EXCEPTION 'Unknown fixture dependency: %',entry.table_name; END IF;
END LOOP;
${tables.map(table => `IF ${tenant} > 0 AND to_regclass('public.${table}') IS NOT NULL THEN DELETE FROM ${table} WHERE tenant_id=${tenant}; END IF;`).join('\n ')}
DELETE FROM auth_tokens WHERE user_id IN (${userIDs});
DELETE FROM workbench_access_assignments WHERE user_id IN (${userIDs}) AND tenant_id=${manifest.source_tenant_id};
DELETE FROM workbench_access_grants WHERE user_id IN (${userIDs}) AND tenant_id=${manifest.source_tenant_id};
DELETE FROM tenant_members WHERE user_id IN (${userIDs}) AND tenant_id=${manifest.source_tenant_id};
DELETE FROM users WHERE id IN (${userIDs}) AND (${identities}) AND tenant_id IN (${tenant},${manifest.source_tenant_id});
DELETE FROM tenants WHERE id=${tenant} AND name=${literal(manifest.run)} AND business='native-memory-smoke-fixture';
END $fixture$;
COMMIT;`
fs.writeFileSync(path.join(path.dirname(manifestPath), 'fixture-cleanup.sql'), query + '\n', { mode: 0o600 })
sql(query)
assert.equal(sql(`SELECT (SELECT count(*) FROM tenants WHERE id=${tenant} AND ${tenant}>0) + (SELECT count(*) FROM users WHERE id IN (${userIDs}));`), '0')
for (const name of ['owner-auth.json', 'outsider-auth.json']) fs.rmSync(path.join(path.dirname(manifestPath), name), { force: true })
manifest.cleaned_at = new Date().toISOString()
manifest.cleanup_error = null
manifest.cleanup_note = 'Dedicated tenant, fixture users, tokens, balances, model, Agent and fixture data removed. Source workspace administrator audit entries are retained.'
persist()
}
async function main() {
if (process.argv[2] === '--cleanup') {
assert.ok(process.argv[3], '--cleanup requires the fixture manifest path')
manifestPath = path.resolve(process.argv[3])
manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'))
cleanup()
return
}
if (process.argv[2] !== '--run') {
console.log('Use --run to create an isolated fixture, run native_memory_api_smoke.cjs and remove all fixture data; use --cleanup /absolute/path/fixture.json to retry cleanup. See native_memory_api_smoke.md.')
return
}
const run = `memory-smoke-${crypto.randomBytes(8).toString('hex')}`
const directory = path.join(acceptance, run)
fs.mkdirSync(directory, { mode: 0o700, recursive: true })
manifestPath = path.join(directory, 'fixture.json')
manifest = { format: 'native-memory-fixture-v1', run, database,
base: (process.env.MEMORY_SMOKE_BASE || 'http://127.0.0.1:18080/api/v1').replace(/\/$/, ''),
source_tenant_id: Number(process.env.MEMORY_SMOKE_SOURCE_TENANT || 10059),
started_at: new Date().toISOString(), passed: false,
users: ['owner', 'outsider'].map(label => ({ username: `${run}-${label}`, email: `${run}-${label}@example.test` })) }
validateManifest()
persist()
try {
await provision()
await runSmoke()
} catch (error) {
manifest.error = error.message
process.exitCode = 1
persist()
} finally {
try { cleanup() } catch (error) { manifest.cleanup_error = error.message; process.exitCode = 1; persist() }
}
if (!manifest.passed || !manifest.cleaned_at) process.exitCode = 1
}
main().catch(error => {
process.exitCode = 1
console.error(error.message)
}).finally(() => {
if (manifest) console.log(JSON.stringify({ passed: manifest.passed, cleaned: !!manifest.cleaned_at, error: manifest.error, cleanup_error: manifest.cleanup_error, manifest: manifestPath }, null, 2))
})
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env bash
set -euo pipefail
#
# 本地构建 + 打包 WeKnora Lite 发行包
#
# 用法:
# ./scripts/package-lite.sh # 自动检测版本
# ./scripts/package-lite.sh v0.2.0 # 指定版本号
# SKIP_FRONTEND=1 ./scripts/package-lite.sh # 跳过前端构建(使用已有 web/)
#
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
cd "${ROOT_DIR}"
# Resolve version
if [ -n "${1:-}" ]; then
VERSION="$1"
elif command -v git >/dev/null 2>&1; then
VERSION=$(git describe --tags --abbrev=0 2>/dev/null || echo "dev")
else
VERSION="dev"
fi
GOOS=$(go env GOOS)
GOARCH=$(go env GOARCH)
ARCHIVE="WeKnora-lite_${VERSION}_${GOOS}_${GOARCH}"
DIST_DIR="dist/${ARCHIVE}"
echo "=== WeKnora Lite Packager ==="
echo " Version : ${VERSION}"
echo " Platform: ${GOOS}/${GOARCH}"
echo " Output : dist/${ARCHIVE}.tar.gz"
echo ""
# ── Step 1: Build frontend (if not skipped) ──
if [ "${SKIP_FRONTEND:-}" != "1" ]; then
if [ -f frontend/package.json ]; then
echo ">> Building frontend..."
(cd frontend && npm ci --prefer-offline && npm run build)
rm -rf web
cp -r frontend/dist web
else
echo ">> No frontend/package.json found, skipping frontend build"
fi
fi
if [ ! -f web/index.html ]; then
echo "WARNING: web/index.html not found — package will not include frontend"
fi
# ── Step 2: Build Go binary ──
echo ">> Building WeKnora-lite binary..."
export EDITION=lite
eval "$(./scripts/get_version.sh env)"
LDFLAGS="-w -s $(./scripts/get_version.sh ldflags)"
export CGO_CFLAGS="-Wno-deprecated-declarations"
if [ "$(uname)" = "Darwin" ]; then
export CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries"
fi
CGO_ENABLED=1 go build -tags "sqlite_fts5" -ldflags="${LDFLAGS}" \
-o WeKnora-lite ./cmd/server
# ── Step 3: Assemble package ──
echo ">> Assembling package..."
rm -rf "${DIST_DIR}"
mkdir -p "${DIST_DIR}/web"
cp WeKnora-lite "${DIST_DIR}/"
if [ -d web ] && [ -f web/index.html ]; then
cp -r web/* "${DIST_DIR}/web/"
fi
cp .env.lite.example "${DIST_DIR}/"
cp docs/LITE.md "${DIST_DIR}/README.md"
if [ -d config ]; then
cp -r config "${DIST_DIR}/config"
fi
if [ -d migrations/sqlite ]; then
mkdir -p "${DIST_DIR}/migrations/sqlite"
cp -r migrations/sqlite/* "${DIST_DIR}/migrations/sqlite/"
fi
if [ -f deploy/weknora-lite.service ]; then
cp deploy/weknora-lite.service "${DIST_DIR}/"
fi
# ── Step 4: Create tarball ──
echo ">> Creating tarball..."
(cd dist && tar czf "${ARCHIVE}.tar.gz" "${ARCHIVE}")
(cd dist && shasum -a 256 "${ARCHIVE}.tar.gz" > "${ARCHIVE}.tar.gz.sha256")
echo ""
echo "=== Done ==="
echo " dist/${ARCHIVE}.tar.gz"
echo " dist/${ARCHIVE}.tar.gz.sha256"
SIZE=$(du -h "dist/${ARCHIVE}.tar.gz" | cut -f1)
echo " Size: ${SIZE}"
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env bash
set -euo pipefail
#
# 本地构建 + 打包 WeKnora macOS 桌面应用 (.app)
#
# 用法:
# ./scripts/package-mac-app.sh
# SKIP_FRONTEND=1 ./scripts/package-mac-app.sh # 跳过前端构建
#
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
cd "${ROOT_DIR}"
APP_NAME="WeKnora Lite"
APP_BUNDLE="${APP_NAME}.app"
DIST_DIR="dist/${APP_BUNDLE}"
echo "=== WeKnora Mac App Packager ==="
echo " Output: dist/${APP_BUNDLE}"
echo ""
# ── Step 1: Build frontend (if not skipped) ──
if [ "${SKIP_FRONTEND:-}" != "1" ]; then
if [ -f frontend/package.json ]; then
echo ">> Building frontend..."
(cd frontend && npm ci --prefer-offline && npm run build)
# Lite 后端从 ./web 提供 SPA(见 internal/router serveFrontendStatic),与 package-lite.sh 一致须用 dist 更新 web
echo ">> Sync frontend/dist -> web/"
rm -rf web
cp -r frontend/dist web
else
echo ">> No frontend/package.json found, skipping frontend build"
fi
fi
if [ ! -f web/index.html ]; then
echo "WARNING: web/index.html not found. Lite 桌面会从 Resources/web 提供前端;请先完整构建前端(勿使用 SKIP_FRONTEND),或手动: cp -r frontend/dist web"
fi
# ── Step 2: Build with Wails ──
echo ">> Building Wails Desktop App..."
# 如果没有 wails 命令行工具,提醒安装
if ! command -v wails >/dev/null 2>&1; then
echo "Wails CLI not found. Please install it first:"
echo "go install github.com/wailsapp/wails/v2/cmd/wails@latest"
exit 1
fi
# 使用 Wails 打包 (需要先处理依赖代理问题)
export GONOSUMDB="git.sr.ht/*"
export GOPROXY="https://goproxy.cn,direct"
export CGO_CFLAGS="-Wno-deprecated-declarations"
export CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries"
export EDITION=lite
# Milvus 与 Qdrant 的 gRPC 生成代码均注册名为 "common.proto" 的描述符,同一进程内会冲突。
# -ldflags -X conflictPolicy=warn 只作用于最终链接,无法覆盖 Wails「生成绑定」时单独启动的 go 子进程。
# 官方做法:见 https://protobuf.dev/reference/go/faq#namespace-conflict
export GOLANG_PROTOBUF_REGISTRATION_CONFLICT=warn
# 获取版本号并配置 LDFLAGS
eval "$(./scripts/get_version.sh env)"
LDFLAGS="$(./scripts/get_version.sh ldflags) -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"
# 默认生成 Wails 绑定(供 window.go.main.App.* 等);若加 -skipbindings 则需在 OnDomReady 注入 __WEKNORA_API_BASE__
(cd cmd/desktop && wails build -clean -tags "sqlite_fts5" -ldflags="$LDFLAGS" -o "${APP_NAME}")
# ── Step 3: Copy generated .app to dist ──
echo ">> Assembling package..."
mkdir -p dist
rm -rf "${DIST_DIR}"
cp -R "cmd/desktop/build/bin/${APP_BUNDLE}" "dist/"
# 将配置文件和初始数据库迁移脚本塞进 .app 内部资源里
RESOURCES_DIR="${DIST_DIR}/Contents/Resources"
mkdir -p "${RESOURCES_DIR}/config"
mkdir -p "${RESOURCES_DIR}/migrations/sqlite"
if [ -f .env.lite.example ]; then
cp .env.lite.example "${RESOURCES_DIR}/.env"
fi
if [ -d migrations/sqlite ]; then
cp -r migrations/sqlite/* "${RESOURCES_DIR}/migrations/sqlite/"
fi
if [ -d config ]; then
cp -r config/* "${RESOURCES_DIR}/config/"
fi
if [ -d web ]; then
cp -r web "${RESOURCES_DIR}/"
fi
# 注意:Wails build 生成的二进制文件工作目录默认是 app 的 Contents/MacOS 目录
# 后续可能需要调整代码中对配置文件的路径读取逻辑。
echo ""
echo "=== Done ==="
echo " App generated at: dist/${APP_BUNDLE}"
echo " You can double click it to run!"
+49
View File
@@ -0,0 +1,49 @@
const fs = require('node:fs')
const path = require('node:path')
const { execFileSync } = require('node:child_process')
// Snapshot only this build, before adding older hashed chunks for open tabs.
function preserveFrontendAssets(root, revision) {
if (!/^[a-zA-Z0-9-]+$/.test(revision)) throw new Error('Invalid asset revision')
const assets = path.join(root, 'src/frontend/dist/assets')
const archive = path.join(root, 'frontend-asset-archives')
fs.mkdirSync(archive, { recursive: true })
const previous = fs.readdirSync(archive).filter(name => fs.statSync(path.join(archive, name)).isDirectory())
.sort((a, b) => fs.statSync(path.join(archive, b)).mtimeMs - fs.statSync(path.join(archive, a)).mtimeMs).slice(0, 3)
const snapshot = path.join(archive, revision)
if (fs.existsSync(snapshot)) throw new Error('Asset snapshot already exists')
fs.cpSync(assets, snapshot, { recursive: true, errorOnExist: true, force: false })
if (!previous.length) {
const docker = (...args) => execFileSync('docker', ['--context', 'colima-yawan', ...args], { encoding: 'utf8' }).trim()
const containers = docker('ps', '-a', '--filter', 'name=yawan-public-frontend', '--format', '{{.Names}}').split('\n').filter(Boolean)
for (const container of containers) {
if (!/^yawan-public-frontend(?:-before-[a-zA-Z0-9-]+)?$/.test(container)) continue
const name = `bootstrap-${container}`
const destination = path.join(archive, name)
if (!fs.existsSync(destination)) docker('cp', `${container}:/usr/share/nginx/html/assets`, destination)
previous.push(name)
}
}
let retained = 0
for (const name of previous) retained += mergeAssets(path.join(archive, name), assets)
const now = new Date()
fs.utimesSync(snapshot, now, now)
return { retained, previous_builds: previous.length }
}
function mergeAssets(source, destination) {
let copied = 0
for (const entry of fs.readdirSync(source, { withFileTypes: true })) {
const from = path.join(source, entry.name), to = path.join(destination, entry.name)
if (entry.isSymbolicLink()) throw new Error('Asset symlinks are not supported')
if (entry.isDirectory()) {
fs.mkdirSync(to, { recursive: true })
copied += mergeAssets(from, to)
} else if (entry.isFile() && !fs.existsSync(to)) {
fs.copyFileSync(from, to, fs.constants.COPYFILE_EXCL)
copied++
}
}
return copied
}
module.exports = { preserveFrontendAssets, mergeAssets }
+21
View File
@@ -0,0 +1,21 @@
const { test } = require('node:test')
const assert = require('node:assert/strict')
const fs = require('node:fs')
const os = require('node:os')
const path = require('node:path')
const { mergeAssets } = require('./preserve_frontend_assets.cjs')
test('keeps old lazy chunks without replacing current build files', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'assets-test-'))
try {
const old = path.join(dir, 'old'), current = path.join(dir, 'current')
fs.mkdirSync(old); fs.mkdirSync(current)
fs.writeFileSync(path.join(old, 'Agent-oldhash.js'), 'old import')
fs.writeFileSync(path.join(old, 'shared.css'), 'old style')
fs.writeFileSync(path.join(current, 'shared.css'), 'current style')
assert.equal(mergeAssets(old, current), 1)
assert.equal(fs.readFileSync(path.join(current, 'Agent-oldhash.js'), 'utf8'), 'old import')
assert.equal(fs.readFileSync(path.join(current, 'shared.css'), 'utf8'), 'current style')
assert.equal(mergeAssets(old, current), 0)
} finally { fs.rmSync(dir, { recursive: true, force: true }) }
})
+109
View File
@@ -0,0 +1,109 @@
const assert = require('node:assert/strict')
const test = require('node:test')
const fs = require('node:fs')
const os = require('node:os')
const path = require('node:path')
const vm = require('node:vm')
const { stagePublicAPIContext } = require('./stage_public_api_context.cjs')
const release = 'public-20260921-040700'
function publisher(t, action, failBuild = false) {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'public-api-publisher-'))
t.after(() => fs.rmSync(root, { recursive: true, force: true }))
const source = path.join(root, 'src')
for (const directory of ['scripts', 'migrations', 'frontend/dist']) fs.mkdirSync(path.join(source, directory), { recursive: true })
fs.mkdirSync(path.join(root, 'acceptance'))
fs.writeFileSync(path.join(source, 'scripts/Dockerfile.public-api'), fs.readFileSync(path.join(__dirname, 'Dockerfile.public-api')))
fs.writeFileSync(path.join(source, 'WeKnora-native-acceptance-next'), 'reviewed-api', { mode: 0o755 })
fs.writeFileSync(path.join(source, 'migrations/001.sql'), 'SELECT 1;\n')
fs.writeFileSync(path.join(root, 'acceptance/runtime.env'), 'DB_NAME=yawan_native_acceptance_20260915\n')
const directory = path.join(root, 'releases', release)
if (action === 'refreshAPI') {
fs.mkdirSync(directory, { recursive: true })
fs.writeFileSync(path.join(directory, 'manifest.json'), JSON.stringify({ release, stage: 'origin-ready',
api_image: 'yawan-api:previous', api_image_id: 'sha256:previous' }))
}
const calls = []
const module = { exports: {} }
const injectedRequire = name => {
if (name === 'node:fs') return { ...fs, copyFileSync(from, to, flags) {
if (from === '/Users/macclaw/.cloudflared/config.yml') {
calls.push({ kind: 'tunnel-backup', to })
return fs.writeFileSync(to, 'previous tunnel configuration\n')
}
return fs.copyFileSync(from, to, flags)
} }
if (name === 'node:child_process') return { execFileSync(binary, args) {
assert.equal(binary, 'docker')
assert.deepEqual(Array.from(args.slice(0, 2)), ['--context', 'colima-yawan'])
const command = Array.from(args.slice(2))
calls.push({ kind: 'docker', command })
if (command[0] === 'build' && command[2].startsWith('yawan-api:')) {
if (failBuild) throw new Error('stubbed API build failure')
assert.deepEqual(fs.readdirSync(command.at(-1)).sort(), ['Dockerfile', 'WeKnora-native-acceptance-next', 'migrations'])
}
return command[0] === 'image' ? 'sha256:built-image\n' : ''
} }
if (name === './preserve_frontend_assets.cjs') return { preserveFrontendAssets() { calls.push({ kind: 'retain-assets' }); return { retained: 7 } } }
if (name === './stage_public_api_context.cjs') return { stagePublicAPIContext }
return require(name)
}
const context = { require: injectedRequire, module, __dirname: path.join(source, 'scripts'), URL, AbortSignal,
fetch: async () => ({ ok: true }), console: { log() {} }, recordIdle: () => calls.push({ kind: 'idle' }) }
vm.runInNewContext(fs.readFileSync(path.join(__dirname, 'publish_public_runtime.cjs'), 'utf8')
+ '\n;idle = recordIdle; globalThis.actions = { prepare, refreshAPI };', context)
return { root, source, directory, calls, run: () => context.actions[action](release) }
}
test('prepare builds its API from the release snapshot after backups and retains frontend behavior', t => {
const value = publisher(t, 'prepare')
value.run()
const builds = value.calls.filter(call => call.kind === 'docker' && call.command[0] === 'build')
assert.deepEqual(builds.map(call => call.command), [
['build', '-t', `yawan-api:${release}`, path.join(value.directory, 'api-context')],
['build', '-t', `yawan-frontend:${release}`, path.join(value.source, 'frontend')],
])
const firstBuild = value.calls.indexOf(builds[0])
const backups = value.calls.filter(call => call.kind === 'docker' && call.command.includes('pg_dump'))
assert.equal(backups.length, 2)
assert.ok(backups.every(call => value.calls.indexOf(call) < firstBuild))
assert.ok(value.calls.find(call => call.kind === 'tunnel-backup'))
assert.ok(value.calls.find(call => call.kind === 'retain-assets'))
const manifest = JSON.parse(fs.readFileSync(path.join(value.directory, 'manifest.json')))
assert.equal(manifest.stage, 'prepared')
assert.equal(manifest.api_image_id, 'sha256:built-image')
assert.equal(manifest.frontend_image_id, 'sha256:built-image')
assert.equal(manifest.retained_assets.retained, 7)
})
test('refresh-api uses its immutable revision snapshot and preserves idle and image identity checks', async t => {
const value = publisher(t, 'refreshAPI')
await value.run()
const buildIndex = value.calls.findIndex(call => call.kind === 'docker' && call.command[0] === 'build')
const build = value.calls[buildIndex].command
const revision = build[2].match(/-api-(\d{17})$/)[1]
assert.deepEqual(build, ['build', '-t', `yawan-api:${release}-api-${revision}`, path.join(value.directory, `api-context-${revision}`)])
const idleIndexes = value.calls.flatMap((call, index) => call.kind === 'idle' ? [index] : [])
const stopIndex = value.calls.findIndex(call => call.kind === 'docker' && call.command[0] === 'stop')
assert.equal(idleIndexes.length, 2)
assert.ok(idleIndexes[0] < buildIndex && buildIndex < idleIndexes[1] && idleIndexes[1] < stopIndex)
const start = value.calls.find(call => call.kind === 'docker' && call.command[0] === 'run').command
assert.equal(start.at(-1), 'sha256:built-image')
assert.ok(start.includes('WEKNORA_REVIEWED_API_IMAGE=sha256:built-image'))
const manifest = JSON.parse(fs.readFileSync(path.join(value.directory, 'manifest.json')))
assert.equal(manifest.api_image_id, 'sha256:built-image')
assert.deepEqual(manifest.previous_apis, [{ image: 'yawan-api:previous', image_id: 'sha256:previous',
container: `yawan-native-acceptance-before-${revision}` }])
})
test('an API build failure retains the staged context and never stops the running API', async t => {
const value = publisher(t, 'refreshAPI', true)
await assert.rejects(value.run(), /stubbed API build failure/)
assert.equal(value.calls.some(call => call.kind === 'docker' && ['stop', 'rename', 'run'].includes(call.command[0])), false)
const build = value.calls.find(call => call.kind === 'docker' && call.command[0] === 'build').command
assert.equal(fs.readFileSync(path.join(build.at(-1), 'WeKnora-native-acceptance-next'), 'utf8'), 'reviewed-api')
const manifest = JSON.parse(fs.readFileSync(path.join(value.directory, 'manifest.json')))
assert.equal(manifest.api_image_id, 'sha256:previous')
assert.equal(manifest.stage, 'origin-ready')
})
+447
View File
@@ -0,0 +1,447 @@
const fs = require('node:fs')
const path = require('node:path')
const { execFileSync } = require('node:child_process')
const { parseEnvironment } = require('./workbench_acceptance_runtime.cjs')
const { preserveFrontendAssets } = require('./preserve_frontend_assets.cjs')
const { stagePublicAPIContext } = require('./stage_public_api_context.cjs')
const root = path.resolve(__dirname, '../..')
const database = 'yawan_native_acceptance_20260915'
const apiName = 'yawan-native-acceptance'
const network = 'weknora_WeKnora-network'
const publicVerificationReports = ['public-release-smoke', 'public-navigation', 'public-access-control', 'public-resource-isolation', 'public-resource-files', 'public-tenant-isolation', 'public-indexed-retrieval-isolation']
const fixtureReconciliationCounts = ['active_users', 'active_memberships', 'roles', 'assignments', 'grants',
'unrevoked_tokens', 'active_agents', 'active_skills', 'active_digital_humans', 'workbench_resources',
'active_sessions', 'synthetic_messages', 'file_resources', 'file_bindings', 'file_access_grants',
'active_knowledge_bases', 'active_documents', 'active_chunks', 'tags', 'tag_relations', 'publications',
'secondary_tenant', 'secondary_memberships', 'memory_item_embeddings', 'memory_items', 'memory_tombstones',
'memory_topic_stats', 'memory_doc_affinity', 'memory_subjects']
const docker = (...args) => execFileSync('docker', ['--context', 'colima-yawan', ...args], {
encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'],
}).trim()
const writeJSON = (file, value) => fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n', { mode: 0o600 })
// Include every native queue, plus queues discovered in Redis. Read all states
// in one read-only Lua snapshot: an active task can otherwise move to retry
// between separate commands and appear idle. Archived/completed history does
// not block publication; waiting aggregation work does.
const nativeQueues = ['default', 'chat_attachment', 'postprocess', 'summary', 'multimodal',
'graph', 'question', 'sync', 'low', 'wiki', 'memory']
const queueStates = ['active', 'pending', 'scheduled', 'retry', 'aggregating', 'aggregation_sets']
const queueSnapshotScript = `
local names = {}
for _, name in ipairs(ARGV) do names[name] = true end
for _, name in ipairs(redis.call('SMEMBERS', KEYS[1])) do names[name] = true end
local result = {}
for name, _ in pairs(names) do
local prefix = 'asynq:{' .. name .. '}:'
local aggregating = 0
for _, group in ipairs(redis.call('SMEMBERS', prefix .. 'groups')) do
aggregating = aggregating + redis.call('ZCARD', prefix .. 'g:' .. group)
end
result[name] = {
active = redis.call('LLEN', prefix .. 'active'),
pending = redis.call('LLEN', prefix .. 'pending'),
scheduled = redis.call('ZCARD', prefix .. 'scheduled'),
retry = redis.call('ZCARD', prefix .. 'retry'),
aggregating = aggregating,
aggregation_sets = redis.call('ZCARD', prefix .. 'aggregation_sets')
}
end
return cjson.encode(result)
`
function idleCount(value) {
if (!Number.isSafeInteger(value) || value < 0) throw new Error('Cannot verify background task counts')
return value
}
function assertIdleSnapshot(snapshot) {
if (!snapshot || idleCount(snapshot.workbench) !== 0) throw new Error('Background jobs, uploads or reserved balances are still active')
if (idleCount(snapshot.pending_ops) !== 0) throw new Error('Wiki or other persisted background operations are still pending')
const queues = snapshot.queues
if (!queues || typeof queues !== 'object' || Array.isArray(queues)
|| nativeQueues.some(name => !Object.hasOwn(queues, name))) throw new Error('Cannot verify native background queues')
for (const state of Object.values(queues)) {
if (!state || typeof state !== 'object' || queueStates.some(name => idleCount(state[name]) !== 0)) {
throw new Error('Native background queues still have active or waiting tasks')
}
}
}
function idle(run = execFileSync) {
const invoke = (args, env = process.env) => run('docker', ['--context', 'colima-yawan', ...args], {
encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, timeout: 15000,
env, stdio: ['ignore', 'pipe', 'pipe'],
}).trim()
let snapshot
try {
snapshot = JSON.parse(invoke(['exec', 'WeKnora-postgres', 'psql', '-X', '-qAt',
'-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', database, '-c',
"SELECT json_build_object('workbench', (SELECT count(*) FROM workbench_jobs WHERE status IN ('queued','running')) + (SELECT count(*) FROM digital_human_jobs WHERE status IN ('queued','running')) + (SELECT count(*) FROM workbench_knowledge_version_jobs WHERE status IN ('queued','running')) + (SELECT count(*) FROM knowledge_classification_jobs WHERE status='running') + (SELECT count(*) FROM resumable_uploads WHERE state NOT IN ('completed','cancelled','failed','expired')) + (SELECT count(*) FROM workbench_balance_holds WHERE status='reserved'), 'pending_ops', (SELECT count(*) FROM task_pending_ops));"]))
const environment = JSON.parse(invoke(['inspect', apiName, '--format', '{{json .Config.Env}}']))
if (!Array.isArray(environment) || environment.some(value => typeof value !== 'string')) throw new Error()
const runtime = parseEnvironment(environment.join('\n'))
// Use the running API's Redis host, ACL identity and database, not the old
// deployment or the redis-cli defaults. Secrets are passed only in env.
const address = /^(?:\[([^\]]+)\]|([^:]+)):(\d+)$/.exec(runtime.REDIS_ADDR || '')
if (!address || !/^\d+$/.test(runtime.REDIS_DB || '') || runtime.DB_NAME !== database) throw new Error()
const redisArgs = ['redis-cli', '--raw', '--no-auth-warning', '-h', address[1] || address[2],
'-p', address[3], '-n', runtime.REDIS_DB]
if (runtime.REDIS_USERNAME) redisArgs.push('--user', runtime.REDIS_USERNAME)
if (/^true$/i.test((runtime.REDIS_USE_TLS || '').trim())) {
redisArgs.push('--tls')
if (runtime.REDIS_TLS_SERVER_NAME) redisArgs.push('--sni', runtime.REDIS_TLS_SERVER_NAME)
if (/^true$/i.test((runtime.REDIS_TLS_INSECURE_SKIP_VERIFY || '').trim())) redisArgs.push('--insecure')
}
redisArgs.push('EVAL_RO', queueSnapshotScript, '1', 'asynq:queues', ...nativeQueues)
snapshot.queues = JSON.parse(invoke(['exec', '-e', 'REDISCLI_AUTH', 'WeKnora-redis', ...redisArgs],
{ ...process.env, REDISCLI_AUTH: runtime.REDIS_PASSWORD || '' }))
} catch {
// Docker, Redis and JSON errors may embed credentials or server output.
// An unavailable or unparseable queue is never considered empty.
throw new Error('Cannot verify background task idleness; publication stopped')
}
assertIdleSnapshot(snapshot)
return snapshot
}
function prepare(release) {
if (!/^public-\d{8}-\d{6}$/.test(release || '')) throw new Error('Expected public-YYYYMMDD-HHMMSS release identifier')
const directory = path.join(root, 'releases', release)
if (fs.existsSync(directory)) throw new Error('Release already exists; do not overwrite immutable artifacts')
fs.mkdirSync(directory, { recursive: true, mode: 0o700 })
const env = parseEnvironment(fs.readFileSync(path.join(root, 'acceptance/runtime.env'), 'utf8'))
if (env.DB_NAME !== database) throw new Error('Unexpected source database')
Object.assign(env, {
GIN_MODE: 'release', FRONTEND_BASE_URL: 'https://agent.pducv.cn',
DISABLE_REGISTRATION: 'true', WEKNORA_AUTH_REGISTRATION_MODE: 'invite_only',
WEKNORA_TENANT_ENABLE_RBAC: 'true', WEKNORA_TENANT_ENABLE_CROSS_TENANT_ACCESS: 'false',
WEKNORA_TENANT_SELF_SERVICE_CREATION_ENABLED: 'false',
WEKNORA_TRUSTED_PROXIES: '127.0.0.1,::1,172.19.0.0/16',
})
fs.writeFileSync(path.join(directory, 'runtime.env'), Object.entries(env).map(([key, value]) => `${key}=${value}`).join('\n') + '\n', { mode: 0o600 })
fs.copyFileSync(path.join(root, 'acceptance/runtime.env'), path.join(directory, 'previous-runtime.env'))
fs.copyFileSync('/Users/macclaw/.cloudflared/config.yml', path.join(directory, 'previous-tunnel.yml'))
for (const db of [database, 'WeKnora']) {
const fd = fs.openSync(path.join(directory, `${db}.dump`), 'wx', 0o600)
try { execFileSync('docker', ['--context', 'colima-yawan', 'exec', 'WeKnora-postgres', 'pg_dump', '-U', 'postgres', '-Fc', db], { stdio: ['ignore', fd, 'pipe'] }) }
finally { fs.closeSync(fd) }
}
const manifest = { release, prepared_at: new Date().toISOString(), database, api_image: `yawan-api:${release}`, frontend_image: `yawan-frontend:${release}`, previous_api: `yawan-native-before-${release}`, public_origin: 'http://127.0.0.1:18088', url: 'https://agent.pducv.cn', stage: 'prepared' }
writeJSON(path.join(directory, 'manifest.json'), manifest)
manifest.retained_assets = preserveFrontendAssets(root, release)
fs.writeFileSync(path.join(root, 'src/frontend/dist/release.json'), JSON.stringify({ release, built_at: manifest.prepared_at }))
const apiContext = stagePublicAPIContext(root, release)
docker('build', '-t', manifest.api_image, apiContext)
docker('build', '-t', manifest.frontend_image, path.join(root, 'src/frontend'))
for (const [key, image] of [['api_image_id', manifest.api_image], ['frontend_image_id', manifest.frontend_image]]) {
manifest[key] = docker('image', 'inspect', image, '--format', '{{.Id}}')
}
writeJSON(path.join(directory, 'manifest.json'), manifest)
console.log(JSON.stringify({ release, stage: 'prepared', backups: 2, immutable_images: true }))
}
function isActivatedRelease(runtime, manifest) {
return !!manifest.release && !!manifest.api_image_id
&& runtime.image === manifest.api_image_id && runtime.release === manifest.release
}
async function activate(release) {
if (!/^public-\d{8}-\d{6}$/.test(release || '')) throw new Error('Invalid release identifier')
const directory = path.join(root, 'releases', release)
const manifest = JSON.parse(fs.readFileSync(path.join(directory, 'manifest.json'), 'utf8'))
if (manifest.stage !== 'prepared') throw new Error('Release has already been activated')
idle()
const frontendName = 'yawan-public-frontend'
const previousFrontend = `${frontendName}-before-${release}`
const hasFrontend = docker('ps', '-a', '--filter', `name=^/${frontendName}$`, '--format', '{{.Names}}') === frontendName
const runtime = JSON.parse(docker('inspect', apiName, '--format', '{"image":{{json .Image}},"release":{{json (index .Config.Labels "yawan.release")}}}'))
const apiAlreadyStarted = isActivatedRelease(runtime, manifest)
if (apiAlreadyStarted && docker('ps', '-a', '--filter', `name=^/${manifest.previous_api}$`, '--format', '{{.Names}}') !== manifest.previous_api) {
throw new Error('Cannot resume activation without the previous API rollback container')
}
let frontendRenamed = false
let frontendCreated = false
if (!apiAlreadyStarted) {
docker('stop', '--time', '40', apiName)
docker('rename', apiName, manifest.previous_api)
}
try {
if (!apiAlreadyStarted) docker('run', '-d', '--name', apiName, '--restart', 'unless-stopped', '--network', network,
'--env-file', path.join(directory, 'runtime.env'), '-p', '127.0.0.1:18080:8080',
'-e', `WEKNORA_REVIEWED_RELEASE=${release}`, '-e', `WEKNORA_REVIEWED_API_IMAGE=${manifest.api_image_id}`,
'--label', `yawan.release=${release}`,
'-v', `${path.join(root, 'acceptance/files')}:/data/files`,
'-v', `${path.join(root, 'acceptance/resumable-uploads')}:/data/uploads`, manifest.api_image_id)
let healthy = false
for (let attempt = 0; attempt < 45; attempt++) {
try { healthy = (await fetch('http://127.0.0.1:18080/health', { signal: AbortSignal.timeout(1500) })).ok } catch {}
if (healthy) break
await new Promise(resolve => setTimeout(resolve, 1000))
}
if (!healthy) throw new Error('New API did not become healthy')
if (hasFrontend) {
docker('stop', '--time', '20', frontendName)
docker('rename', frontendName, previousFrontend)
frontendRenamed = true
}
docker('run', '-d', '--name', 'yawan-public-frontend', '--restart', 'unless-stopped', '--network', network,
'-p', '127.0.0.1:18088:80', '-e', `APP_HOST=${apiName}`, '-e', 'APP_PORT=8080',
'-e', 'DEFAULT_LOCALE=zh-CN', '-e', 'MAX_FILE_SIZE_MB=50', '--label', `yawan.release=${release}`, manifest.frontend_image_id)
frontendCreated = true
docker('exec', 'yawan-public-frontend', 'nginx', '-t')
const frontendHealth = await fetch('http://127.0.0.1:18088/api/v1/access/me', { signal: AbortSignal.timeout(10000) })
if (frontendHealth.status !== 401) throw new Error('Frontend proxy health failed')
} catch (error) {
if (frontendCreated || frontendRenamed) {
try { docker('rm', '-f', frontendName) } catch {}
}
try { docker('rm', '-f', apiName) } catch {}
docker('rename', manifest.previous_api, apiName)
docker('start', apiName)
if (frontendRenamed) {
docker('rename', previousFrontend, frontendName)
docker('start', frontendName)
} else if (hasFrontend) {
docker('restart', frontendName)
}
throw error
}
if (frontendRenamed) manifest.previous_frontend = previousFrontend
manifest.stage = 'origin-ready'
manifest.activated_at = new Date().toISOString()
writeJSON(path.join(directory, 'manifest.json'), manifest)
writeJSON(path.join(root, 'public-release.json'), manifest)
console.log(JSON.stringify({ release, stage: manifest.stage, origin: manifest.public_origin, tunnel_changed: false }))
}
async function refreshFrontend(release) {
if (!/^public-\d{8}-\d{6}$/.test(release || '')) throw new Error('Invalid release identifier')
const directory = path.join(root, 'releases', release)
const manifest = JSON.parse(fs.readFileSync(path.join(directory, 'manifest.json'), 'utf8'))
if (manifest.stage !== 'origin-ready') throw new Error('Only refresh an unpublished origin')
const builtAt = new Date().toISOString()
const revision = builtAt.replace(/[^0-9]/g, '')
const image = `yawan-frontend:${release}-ui-${revision}`
manifest.retained_assets = preserveFrontendAssets(root, `${release}-ui-${revision}`)
fs.writeFileSync(path.join(root, 'src/frontend/dist/release.json'), JSON.stringify({ release, built_at: builtAt, frontend_revision: revision }))
docker('build', '-t', image, path.join(root, 'src/frontend'))
const imageID = docker('image', 'inspect', image, '--format', '{{.Id}}')
const previous = `yawan-public-frontend-before-${revision}`
docker('stop', '--time', '20', 'yawan-public-frontend')
docker('rename', 'yawan-public-frontend', previous)
try {
docker('run', '-d', '--name', 'yawan-public-frontend', '--restart', 'unless-stopped', '--network', network,
'-p', '127.0.0.1:18088:80', '-e', `APP_HOST=${apiName}`, '-e', 'APP_PORT=8080',
'-e', 'DEFAULT_LOCALE=zh-CN', '-e', 'MAX_FILE_SIZE_MB=50', '--label', `yawan.release=${release}`, imageID)
docker('exec', 'yawan-public-frontend', 'nginx', '-t')
if (!(await fetch('http://127.0.0.1:18088/login', { signal: AbortSignal.timeout(10000) })).ok) throw new Error('Frontend health failed')
} catch (error) {
try { docker('rm', '-f', 'yawan-public-frontend') } catch {}
docker('rename', previous, 'yawan-public-frontend')
docker('start', 'yawan-public-frontend')
throw error
}
manifest.previous_frontends = [...(manifest.previous_frontends || []), { image: manifest.frontend_image, image_id: manifest.frontend_image_id, container: previous }]
manifest.frontend_image = image
manifest.frontend_image_id = imageID
manifest.frontend_built_at = builtAt
manifest.frontend_activated_at = new Date().toISOString()
writeJSON(path.join(directory, 'manifest.json'), manifest)
writeJSON(path.join(root, 'public-release.json'), manifest)
console.log(JSON.stringify({ release, stage: manifest.stage, frontend_revision: revision, tunnel_changed: false }))
}
async function refreshAPI(release) {
if (!/^public-\d{8}-\d{6}$/.test(release || '')) throw new Error('Invalid release identifier')
const directory = path.join(root, 'releases', release)
const manifest = JSON.parse(fs.readFileSync(path.join(directory, 'manifest.json'), 'utf8'))
if (manifest.stage !== 'origin-ready') throw new Error('Only refresh an unverified origin')
idle()
const revision = new Date().toISOString().replace(/[^0-9]/g, '')
const image = `yawan-api:${release}-api-${revision}`
const apiContext = stagePublicAPIContext(root, release, revision)
docker('build', '-t', image, apiContext)
const imageID = docker('image', 'inspect', image, '--format', '{{.Id}}')
idle()
const previous = `${apiName}-before-${revision}`
docker('stop', '--time', '40', apiName)
docker('rename', apiName, previous)
try {
docker('run', '-d', '--name', apiName, '--restart', 'unless-stopped', '--network', network,
'--env-file', path.join(directory, 'runtime.env'), '-p', '127.0.0.1:18080:8080',
'-e', `WEKNORA_REVIEWED_RELEASE=${release}`, '-e', `WEKNORA_REVIEWED_API_IMAGE=${imageID}`,
'--label', `yawan.release=${release}`,
'-v', `${path.join(root, 'acceptance/files')}:/data/files`,
'-v', `${path.join(root, 'acceptance/resumable-uploads')}:/data/uploads`, imageID)
let healthy = false
for (let attempt = 0; attempt < 45; attempt++) {
try { healthy = (await fetch('http://127.0.0.1:18080/health', { signal: AbortSignal.timeout(1500) })).ok } catch {}
if (healthy) break
await new Promise(resolve => setTimeout(resolve, 1000))
}
if (!healthy) throw new Error('Refreshed API health failed')
docker('exec', 'yawan-public-frontend', 'nginx', '-s', 'reload')
} catch (error) {
try { docker('rm', '-f', apiName) } catch {}
docker('rename', previous, apiName)
docker('start', apiName)
docker('exec', 'yawan-public-frontend', 'nginx', '-s', 'reload')
throw error
}
manifest.previous_apis = [...(manifest.previous_apis || []), {image: manifest.api_image, image_id: manifest.api_image_id, container: previous}]
manifest.api_image = image
manifest.api_image_id = imageID
manifest.api_activated_at = new Date().toISOString()
writeJSON(path.join(directory, 'manifest.json'), manifest)
writeJSON(path.join(root, 'public-release.json'), manifest)
console.log(JSON.stringify({release, stage: manifest.stage, api_revision: revision}))
}
function publicationTimestamp(value, field, now) {
const parsed = typeof value === 'string' ? Date.parse(value) : NaN
if (!Number.isFinite(parsed) || new Date(parsed).toISOString() !== value || parsed > now) {
throw new Error(`Invalid publication timestamp: ${field}`)
}
return parsed
}
function assertSuccessfulVerification(result, report) {
if (!result || result.passed !== true || !Array.isArray(result.checks) || !result.checks.length
|| result.checks.some(check => check?.passed !== true)
|| !Array.isArray(result.failures) || result.failures.length) {
throw new Error(`Public verification did not pass: ${report}`)
}
if (Object.hasOwn(result, 'cleanup') && (!Array.isArray(result.cleanup) || !result.cleanup.length
|| result.cleanup.some(item => !item || typeof item !== 'object' || Array.isArray(item)
|| typeof item.name !== 'string' || !item.name.trim() || item.passed !== true
|| (Object.hasOwn(item, 'error') && item.error !== undefined && item.error !== null && item.error !== '')))) {
throw new Error(`Public verification cleanup did not pass: ${report}`)
}
}
function assertPublicVerification(manifest, result, report, now = Date.now()) {
const activated = publicationTimestamp(manifest.activated_at, 'activated_at', now)
const changes = [activated]
for (const field of ['api_activated_at', 'frontend_built_at', 'frontend_activated_at']) {
if (Object.hasOwn(manifest, field)) {
const changed = publicationTimestamp(manifest[field], field, now)
if (changed < activated) throw new Error(`Publication timestamp predates activation: ${field}`)
changes.push(changed)
}
}
if (manifest.previous_apis?.length && !Object.hasOwn(manifest, 'api_activated_at')) {
throw new Error('Missing API activation timestamp')
}
const frontendRefreshed = Object.hasOwn(manifest, 'frontend_built_at')
|| Object.hasOwn(manifest, 'frontend_activated_at') || manifest.previous_frontends?.length
if (frontendRefreshed) {
const built = publicationTimestamp(manifest.frontend_built_at, 'frontend_built_at', now)
const enabled = publicationTimestamp(manifest.frontend_activated_at, 'frontend_activated_at', now)
if (enabled < built) throw new Error('Frontend activation predates its build')
}
assertSuccessfulVerification(result, report)
if (report === 'public-indexed-retrieval-isolation') {
const knowledgeIDs = result.fixtures?.kbs
if (!Array.isArray(knowledgeIDs) || !knowledgeIDs.length
|| knowledgeIDs.some(id => typeof id !== 'string' || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id))
|| new Set(knowledgeIDs).size !== knowledgeIDs.length) {
throw new Error(`Invalid indexed knowledge fixtures: ${report}`)
}
// The final SQL reconciliation relies on these entries to attest Milvus cleanup.
const knowledgeCleanup = result.cleanup?.filter(item => item.name.startsWith('knowledge '))
if (!knowledgeCleanup || knowledgeCleanup.length !== knowledgeIDs.length
|| knowledgeIDs.some(id => knowledgeCleanup.filter(item => item.name === `knowledge ${id}`).length !== 1)) {
throw new Error(`Incomplete indexed knowledge cleanup: ${report}`)
}
}
const started = publicationTimestamp(result.started_at, `${report}.started_at`, now)
const completedField = Object.hasOwn(result, 'checked_at') ? 'checked_at' : 'time'
const completed = publicationTimestamp(result[completedField], `${report}.${completedField}`, now)
if (completed < started) throw new Error(`Public verification finished before it started: ${report}`)
if (started < Math.max(...changes)) throw new Error(`Public verification started before this runtime: ${report}`)
if (new URL(result.origin || result.base).origin !== manifest.url) throw new Error(`Wrong verification origin: ${report}`)
if (['public-navigation', 'public-access-control'].includes(report)) {
let apiOrigin
try { apiOrigin = new URL(result.api_origin).origin } catch {}
if (apiOrigin !== manifest.url) throw new Error(`Wrong verification API origin: ${report}`)
}
}
function assertFixtureReconciliation(manifest, result, reports, now = Date.now()) {
const name = 'public-fixture-reconciliation'
assertSuccessfulVerification(result, name)
if (result.release !== manifest.release || result.origin !== manifest.url || result.read_only !== true) {
throw new Error('Fixture reconciliation identity does not match this release')
}
const checked = publicationTimestamp(result.checked_at, `${name}.checked_at`, now)
if (!result.reports || Array.isArray(result.reports)
|| Object.keys(result.reports).length !== publicVerificationReports.length) {
throw new Error('Fixture reconciliation must cover all seven source reports')
}
for (const sourceName of publicVerificationReports) {
const source = reports?.[sourceName]
assertPublicVerification(manifest, source, sourceName, now)
const completed = Object.hasOwn(source, 'checked_at') ? source.checked_at : source.time
const reconciled = result.reports[sourceName]
if (!reconciled || reconciled.started_at !== source.started_at || reconciled.completed_at !== completed) {
throw new Error(`Fixture reconciliation does not match current report: ${sourceName}`)
}
if (checked < Date.parse(completed)) throw new Error(`Fixture reconciliation predates source report: ${sourceName}`)
}
const expected = [...fixtureReconciliationCounts, 'file_fixture_directory_removed']
const byName = new Map(result.checks.map(check => [check.name, check]))
if (result.checks.length !== expected.length || byName.size !== expected.length
|| expected.some(check => !byName.has(check))) {
throw new Error('Fixture reconciliation is missing required residual checks')
}
for (const check of fixtureReconciliationCounts) {
if (byName.get(check).count !== 0) throw new Error(`Fixture reconciliation has residual rows: ${check}`)
}
const prefix = reports['public-resource-files'].prefix
if (!/^file-isolation-[0-9]+$/.test(prefix || '')
|| byName.get('file_fixture_directory_removed').path !== path.join(root, 'acceptance', 'files', '10059', 'exports', prefix)) {
throw new Error('Fixture reconciliation checked the wrong attachment directory')
}
}
async function markPublished(release) {
if (!/^public-\d{8}-\d{6}$/.test(release || '')) throw new Error('Invalid release identifier')
const directory = path.join(root, 'releases', release)
const manifest = JSON.parse(fs.readFileSync(path.join(directory, 'manifest.json'), 'utf8'))
if (manifest.release !== release) throw new Error('Manifest release does not match the requested release')
if (manifest.stage !== 'origin-ready') throw new Error('Expected verified origin')
const response = await fetch(`${manifest.url}/release.json`, { signal: AbortSignal.timeout(30000) })
if (!response.ok || (await response.json()).release !== release) throw new Error('Public origin does not serve this release')
const reports = Object.fromEntries(publicVerificationReports.map(name => [name,
JSON.parse(fs.readFileSync(path.join(root, 'acceptance', `${name}.json`), 'utf8'))]))
const reconciliation = JSON.parse(fs.readFileSync(path.join(root, 'acceptance', 'public-fixture-reconciliation.json'), 'utf8'))
assertFixtureReconciliation(manifest, reconciliation, reports)
manifest.stage = 'published'
manifest.published_at = new Date().toISOString()
manifest.public_verification_reports = publicVerificationReports
manifest.fixture_reconciliation_checked_at = reconciliation.checked_at
writeJSON(path.join(directory, 'manifest.json'), manifest)
writeJSON(path.join(root, 'public-release.json'), manifest)
console.log(JSON.stringify({ release, stage: manifest.stage, url: manifest.url }))
}
module.exports = { idle, assertIdleSnapshot, nativeQueues, queueStates, queueSnapshotScript, isActivatedRelease, assertPublicVerification,
assertFixtureReconciliation, publicVerificationReports, fixtureReconciliationCounts }
if (require.main === module) {
const [action, release] = process.argv.slice(2)
Promise.resolve().then(() => {
if (action === 'prepare') return prepare(release)
if (action === 'activate') return activate(release)
if (action === 'refresh-frontend') return refreshFrontend(release)
if (action === 'refresh-api') return refreshAPI(release)
if (action === 'mark-published') return markPublished(release)
throw new Error('Use prepare, activate, refresh-api, refresh-frontend or mark-published, followed by release identifier')
}).catch(() => {
// Docker diagnostics may contain environment secrets; keep console output bounded.
console.error('Public release command failed; inspect the private release directory and container health. This command does not change the tunnel.')
process.exitCode = 1
})
}
+420
View File
@@ -0,0 +1,420 @@
const assert = require('node:assert/strict')
const test = require('node:test')
const fs = require('node:fs')
const path = require('node:path')
const vm = require('node:vm')
const { idle, assertIdleSnapshot, nativeQueues, queueStates, isActivatedRelease, assertPublicVerification,
assertFixtureReconciliation, publicVerificationReports, fixtureReconciliationCounts } = require('./publish_public_runtime.cjs')
test('a frontend-only release with an unchanged API image is a new activation', () => {
const manifest = { release: 'public-20260920-165100', api_image_id: 'sha256:same-image' }
assert.equal(isActivatedRelease({ image: manifest.api_image_id, release: 'public-20260920-160500' }, manifest), false)
assert.equal(isActivatedRelease({ image: manifest.api_image_id, release: manifest.release }, manifest), true)
assert.equal(isActivatedRelease({ image: 'sha256:other-image', release: manifest.release }, manifest), false)
assert.equal(isActivatedRelease({ image: manifest.api_image_id, release: null }, manifest), false)
})
function emptySnapshot() {
return { workbench: 0, pending_ops: 0,
queues: Object.fromEntries(nativeQueues.map(name => [name,
Object.fromEntries(queueStates.map(state => [state, 0]))])) }
}
test('permits idle runtime while retaining archived and completed task history', () => {
const snapshot = emptySnapshot()
Object.assign(snapshot.queues.graph, { archived: 10, completed: 16 })
assert.doesNotThrow(() => assertIdleSnapshot(snapshot))
})
test('blocks persisted Wiki operations even when every Redis queue is empty', () => {
const snapshot = emptySnapshot()
snapshot.pending_ops = 1
assert.throws(() => assertIdleSnapshot(snapshot), /persisted background/)
})
test('blocks workbench jobs, uploads or reserved balances', () => {
const snapshot = emptySnapshot()
snapshot.workbench = 1
assert.throws(() => assertIdleSnapshot(snapshot), /jobs, uploads or reserved/)
})
for (const queue of nativeQueues) {
for (const state of queueStates) {
test(`blocks native ${queue} work in ${state} state`, () => {
const snapshot = emptySnapshot()
snapshot.queues[queue][state] = 1
assert.throws(() => assertIdleSnapshot(snapshot), /active or waiting/)
})
}
}
test('blocks newly discovered queues instead of ignoring future worker types', () => {
const snapshot = emptySnapshot()
snapshot.queues.future_worker = { ...snapshot.queues.graph, active: 1 }
assert.throws(() => assertIdleSnapshot(snapshot), /active or waiting/)
})
test('fails closed on missing queues and invalid count shapes', () => {
const snapshot = emptySnapshot()
delete snapshot.queues.graph
assert.throws(() => assertIdleSnapshot(snapshot), /Cannot verify/)
for (const count of [undefined, null, '', '0', -1, NaN, Infinity, 0.1, Number.MAX_SAFE_INTEGER + 1]) {
const item = emptySnapshot()
item.queues.wiki.active = count
assert.throws(() => assertIdleSnapshot(item), /Cannot verify/)
assert.throws(() => assertIdleSnapshot({ ...emptySnapshot(), pending_ops: count }), /Cannot verify/)
}
})
function fakeRuntime(overrides = {}, transformRedis = value => value) {
const runtime = {
DB_NAME: 'yawan_native_acceptance_20260915', REDIS_ADDR: 'actual-redis:6379',
REDIS_DB: '14', REDIS_USERNAME: 'worker', REDIS_PASSWORD: 'private-test-password', ...overrides,
}
const calls = []
const run = (binary, args, options) => {
calls.push({ binary, args, options })
if (args.includes('psql')) {
assert.match(args.at(-1), /count\(\*\) FROM task_pending_ops/)
return JSON.stringify({ workbench: 0, pending_ops: 0 })
}
if (args.includes('inspect')) return JSON.stringify(Object.entries(runtime).map(([key, value]) => `${key}=${value}`))
assert.ok(args.includes('redis-cli'))
return transformRedis(JSON.stringify(emptySnapshot().queues))
}
return { run, calls }
}
test('reads actual Redis identity and DB with no password in argv or parent env', () => {
const previous = process.env.REDISCLI_AUTH
const { run, calls } = fakeRuntime()
assert.doesNotThrow(() => idle(run))
const command = calls.at(-1)
assert.equal(command.binary, 'docker')
assert.ok(command.args.includes('EVAL_RO'))
assert.equal(command.args[command.args.indexOf('-h') + 1], 'actual-redis')
assert.equal(command.args[command.args.indexOf('-n') + 1], '14')
assert.equal(command.args[command.args.indexOf('--user') + 1], 'worker')
assert.equal(command.options.env.REDISCLI_AUTH, 'private-test-password')
assert.equal(process.env.REDISCLI_AUTH, previous)
assert.ok(!command.args.join(' ').includes('private-test-password'))
assert.ok(calls.every(call => call.options.timeout === 15000))
})
test('preserves runtime TLS verification and supports IPv6 Redis addresses', () => {
const { run, calls } = fakeRuntime({ REDIS_ADDR: '[::1]:6380', REDIS_USE_TLS: 'true', REDIS_TLS_SERVER_NAME: 'redis.example.test' })
idle(run)
const args = calls.at(-1).args
assert.equal(args[args.indexOf('-h') + 1], '::1')
assert.equal(args[args.indexOf('-p') + 1], '6380')
assert.ok(args.includes('--tls'))
assert.equal(args[args.indexOf('--sni') + 1], 'redis.example.test')
assert.ok(!args.includes('--insecure'))
})
test('does not publish when Redis auth, transport or response validation fails', () => {
for (const transform of [() => 'NOAUTH private-test-password', () => '{invalid', () => { throw new Error('connection failed private-test-password') }]) {
const { run } = fakeRuntime({}, transform)
assert.throws(() => idle(run), error => {
assert.match(error.message, /publication stopped/)
assert.ok(!error.message.includes('private-test-password'))
return true
})
}
for (const overrides of [{ REDIS_ADDR: '' }, { REDIS_DB: '' }, { REDIS_DB: 'broken' }, { DB_NAME: 'old_database' }]) {
assert.throws(() => idle(fakeRuntime(overrides).run), /publication stopped/)
}
})
test('minimum queue coverage follows the native Go worker topology', () => {
const source = fs.readFileSync(path.join(__dirname, '../internal/types/task.go'), 'utf8')
const names = [...source.matchAll(/^\s*Queue\w+\s*=\s*"([^"]+)"/gm)].map(match => match[1])
assert.deepEqual([...nativeQueues].sort(), names.sort())
})
const clock = Date.parse('2026-09-19T18:00:00.000Z')
function publicationFixture() {
return {
manifest: {
url: 'https://agent.pducv.cn', activated_at: '2026-09-19T14:29:38.265Z',
api_activated_at: '2026-09-19T14:51:44.349Z',
frontend_built_at: '2026-09-19T15:18:58.094Z',
frontend_activated_at: '2026-09-19T15:22:00.000Z',
},
report: {
started_at: '2026-09-19T15:23:00.000Z', checked_at: '2026-09-19T15:24:18.110Z',
origin: 'https://agent.pducv.cn', api_origin: 'https://agent.pducv.cn', checks: [{passed: true}], failures: [], passed: true,
},
}
}
const verifyPublication = (manifest, report) => assertPublicVerification(manifest, report, 'fixture-report', clock)
for (const name of ['public-navigation', 'public-access-control']) {
test(`${name} requires proof its API checks use the public origin`, () => {
const {manifest, report} = publicationFixture()
assert.doesNotThrow(() => assertPublicVerification(manifest, report, name, clock))
for (const api_origin of [undefined, '', 'invalid', 'http://127.0.0.1:18080', 'https://another.example']) {
assert.throws(() => assertPublicVerification(manifest, {...report, api_origin}, name, clock), /Wrong verification API origin/)
}
})
}
test('accepts complete verification begun after all runtime changes with either completion timestamp field', () => {
const {manifest, report} = publicationFixture()
assert.doesNotThrow(() => verifyPublication(manifest, report))
const legacyCompletion = {...report, time: report.checked_at, base: report.origin + '/api/v1'}
delete legacyCompletion.checked_at
delete legacyCompletion.origin
assert.doesNotThrow(() => verifyPublication(manifest, legacyCompletion))
})
test('rejects checks begun before initial, API or UI activation even when completed afterward', () => {
const {manifest, report} = publicationFixture()
for (const field of ['activated_at', 'api_activated_at', 'frontend_activated_at']) {
const start = new Date(Date.parse(manifest[field]) - 1).toISOString()
assert.throws(() => verifyPublication(manifest, {...report, started_at: start}), /started before this runtime/)
}
// Building a new image does not mean it has replaced the served frontend.
assert.throws(() => verifyPublication(manifest, {...report, started_at: '2026-09-19T15:20:00.000Z'}), /started before this runtime/)
})
test('initial activation needs no fabricated API or UI refresh timestamps', () => {
const {manifest, report} = publicationFixture()
const initial = {url: manifest.url, activated_at: manifest.activated_at}
assert.doesNotThrow(() => verifyPublication(initial, report))
assert.throws(() => verifyPublication({...initial, previous_apis: [{}]}, report), /Missing API/)
assert.throws(() => verifyPublication({...initial, previous_frontends: [{}]}, report), /Invalid publication timestamp/)
})
test('requires actual UI activation after every frontend rebuild', () => {
const {manifest, report} = publicationFixture()
const missingActivation = {...manifest}
delete missingActivation.frontend_activated_at
assert.throws(() => verifyPublication(missingActivation, report), /frontend_activated_at/)
const missingBuild = {...manifest}
delete missingBuild.frontend_built_at
assert.throws(() => verifyPublication(missingBuild, report), /frontend_built_at/)
assert.throws(() => verifyPublication({...manifest, frontend_activated_at: '2026-09-19T15:00:00.000Z'}, report), /activation predates its build/)
})
test('fails closed for every invalid manifest timestamp instead of comparing against NaN', () => {
const {manifest, report} = publicationFixture()
const invalid = [undefined, null, '', false, 0, 'invalid', '2026-02-30T00:00:00.000Z', '2026-09-20T00:00:00.000Z']
for (const field of ['activated_at', 'api_activated_at', 'frontend_built_at', 'frontend_activated_at']) {
for (const value of invalid) {
assert.throws(() => verifyPublication({...manifest, [field]: value}, report), /Invalid publication timestamp/)
}
}
assert.throws(() => verifyPublication({url: manifest.url}, report), /activated_at/)
assert.throws(() => verifyPublication({...manifest, api_activated_at: '2026-09-19T13:00:00.000Z'}, report), /predates activation/)
})
test('rejects missing, invalid, future and reversed report timing rather than using completion as start', () => {
const {manifest, report} = publicationFixture()
for (const field of ['started_at', 'checked_at']) {
for (const value of [undefined, null, '', false, 0, 'invalid', '2026-02-30T00:00:00.000Z', '2026-09-20T00:00:00.000Z']) {
assert.throws(() => verifyPublication(manifest, {...report, [field]: value}), /Invalid publication timestamp/)
}
}
assert.throws(() => verifyPublication(manifest, {...report, checked_at: '2026-09-19T15:22:30.000Z'}), /finished before it started/)
})
test('rejects truthy pass flags, empty checks, individual failures and a different origin', () => {
const {manifest, report} = publicationFixture()
for (const change of [{passed: 'true'}, {checks: []}, {checks: [{passed: false}]}, {checks: [{passed: 'true'}]},
{failures: ['cleanup failed']}, {failures: undefined}]) {
assert.throws(() => verifyPublication(manifest, {...report, ...change}), /did not pass/)
}
assert.throws(() => verifyPublication(manifest, {...report, origin: 'http://127.0.0.1:18088'}), /Wrong verification origin/)
})
test('rejects failed, malformed or contradictory cleanup results even when the top-level report passed', () => {
const {manifest, report} = publicationFixture()
for (const cleanup of [undefined, null, {}, false, 'passed', [], [null], [true],
[{name: 'knowledge fixture', passed: false}], [{name: 'knowledge fixture', passed: 'true'}],
[{name: '', passed: true}], [{passed: true}], [{name: 'knowledge fixture', passed: true, error: 'cleanup failed'}]]) {
assert.throws(() => verifyPublication(manifest, {...report, cleanup}), /cleanup did not pass/)
}
assert.doesNotThrow(() => verifyPublication(manifest, {...report, cleanup: [{name: 'knowledge fixture', passed: true}]}))
})
function reconciliationFixture() {
const {manifest, report} = publicationFixture()
Object.assign(manifest, {release: 'public-20260919-223000', stage: 'origin-ready',
frontend_built_at: '2026-09-19T16:50:23.481Z', frontend_activated_at: '2026-09-19T16:50:27.512Z'})
const reports = Object.fromEntries(publicVerificationReports.map((name, index) => [name, {...structuredClone(report),
started_at: `2026-09-19T17:0${index}:00.000Z`, checked_at: `2026-09-19T17:0${index}:30.000Z` }]))
reports['public-resource-files'].prefix = 'file-isolation-1758300000000'
const indexed = reports['public-indexed-retrieval-isolation']
indexed.fixtures = {kbs: ['00000000-0000-4000-8000-000000000001']}
indexed.cleanup = indexed.fixtures.kbs.map(id => ({name: `knowledge ${id}`, passed: true}))
const reconciliation = {release: manifest.release, origin: manifest.url, read_only: true,
checked_at: '2026-09-19T17:07:00.000Z', passed: true, failures: [],
reports: Object.fromEntries(publicVerificationReports.map(name => [name, {
started_at: reports[name].started_at, completed_at: reports[name].checked_at,
}])), checks: [...fixtureReconciliationCounts.map(name => ({name, count: 0, passed: true})), {
name: 'file_fixture_directory_removed', passed: true,
path: path.resolve(__dirname, '../../acceptance/files/10059/exports/file-isolation-1758300000000'),
}]}
return {manifest, reports, reconciliation}
}
const invalidIndexedCleanup = [
['missing cleanup', report => {delete report.cleanup}, /Incomplete indexed knowledge cleanup/],
['empty cleanup', report => {report.cleanup = []}, /cleanup did not pass/],
['only account cleanup', report => {report.cleanup = [{name: 'user fixture', passed: true}]}, /Incomplete indexed knowledge cleanup/],
['mismatched knowledge cleanup', report => {report.cleanup[0].name = 'knowledge 00000000-0000-4000-8000-000000000002'}, /Incomplete indexed knowledge cleanup/],
['duplicate knowledge cleanup', report => {report.cleanup.push({...report.cleanup[0]})}, /Incomplete indexed knowledge cleanup/],
['unmatched extra knowledge cleanup', report => {report.cleanup.push({name: 'knowledge 00000000-0000-4000-8000-000000000002', passed: true})}, /Incomplete indexed knowledge cleanup/],
['second knowledge fixture without cleanup', report => {report.fixtures.kbs.push('00000000-0000-4000-8000-000000000002')}, /Incomplete indexed knowledge cleanup/],
['failed knowledge cleanup', report => {report.cleanup[0].passed = false}, /cleanup did not pass/],
['missing fixtures', report => {delete report.fixtures}, /Invalid indexed knowledge fixtures/],
['missing knowledge fixtures', report => {delete report.fixtures.kbs}, /Invalid indexed knowledge fixtures/],
['empty knowledge fixtures', report => {report.fixtures.kbs = []}, /Invalid indexed knowledge fixtures/],
['invalid knowledge fixture', report => {report.fixtures.kbs = ['fixture']; report.cleanup[0].name = 'knowledge fixture'}, /Invalid indexed knowledge fixtures/],
['duplicate knowledge fixtures', report => {report.fixtures.kbs.push(report.fixtures.kbs[0])}, /Invalid indexed knowledge fixtures/],
]
for (const [label, mutate, pattern] of invalidIndexedCleanup) {
test(`blocks final publication for indexed retrieval with ${label}`, async () => {
const value = reconciliationFixture()
mutate(value.reports['public-indexed-retrieval-isolation'])
assert.throws(() => assertFixtureReconciliation(value.manifest, value.reconciliation, value.reports, clock), pattern)
const publisher = injectedPublisher(value)
await assert.rejects(publisher.publish(), pattern)
assert.deepEqual(publisher.writes, [])
})
}
test('indexed retrieval accepts every knowledge cleanup alongside account and role cleanup', () => {
const {manifest, reports, reconciliation} = reconciliationFixture()
const indexed = reports['public-indexed-retrieval-isolation']
indexed.fixtures.kbs.push('00000000-0000-4000-8000-000000000002')
indexed.cleanup.push({name: `knowledge ${indexed.fixtures.kbs[1]}`, passed: true},
{name: 'user fixture', passed: true}, {name: 'role fixture', passed: true})
assert.doesNotThrow(() => assertFixtureReconciliation(manifest, reconciliation, reports, clock))
})
test('accepts complete zero-residue reconciliation tied to all seven reports after the latest activation', () => {
const {manifest, reports, reconciliation} = reconciliationFixture()
assert.doesNotThrow(() => assertFixtureReconciliation(manifest, reconciliation, reports, clock))
const indexed = reports['public-indexed-retrieval-isolation']
indexed.time = indexed.checked_at
delete indexed.checked_at
assert.doesNotThrow(() => assertFixtureReconciliation(manifest, reconciliation, reports, clock))
})
for (const name of publicVerificationReports) {
test(`rejects reconciliation that refers to a previous execution of ${name}`, () => {
for (const field of ['started_at', 'completed_at']) {
const {manifest, reports, reconciliation} = reconciliationFixture()
reconciliation.reports[name][field] = '2026-09-19T16:55:00.000Z'
assert.throws(() => assertFixtureReconciliation(manifest, reconciliation, reports, clock), /does not match current report/)
}
})
test(`rejects ${name} started before the latest activation even with matching reconciliation metadata`, () => {
const {manifest, reports, reconciliation} = reconciliationFixture()
reports[name].started_at = '2026-09-19T16:50:27.511Z'
reconciliation.reports[name].started_at = reports[name].started_at
assert.throws(() => assertFixtureReconciliation(manifest, reconciliation, reports, clock), /started before this runtime/)
})
}
for (const [label, mutate, pattern] of [
['different release', value => {value.reconciliation.release = 'public-20260919-000000'}, /identity/],
['different origin', value => {value.reconciliation.origin = 'http://127.0.0.1:18088'}, /identity/],
['non-read-only result', value => {value.reconciliation.read_only = false}, /identity/],
['failed result', value => {value.reconciliation.passed = false}, /did not pass/],
['reported failure', value => {value.reconciliation.failures.push('residual rows')}, /did not pass/],
['future check', value => {value.reconciliation.checked_at = '2026-09-20T00:00:00.000Z'}, /Invalid publication timestamp/],
['missing check time', value => {delete value.reconciliation.checked_at}, /Invalid publication timestamp/],
['check before final source completes', value => {value.reconciliation.checked_at = '2026-09-19T17:06:29.999Z'}, /predates source report/],
['missing source metadata', value => {delete value.reconciliation.reports['public-navigation']}, /all seven/],
['renamed source metadata', value => {value.reconciliation.reports.wrong = value.reconciliation.reports['public-navigation']; delete value.reconciliation.reports['public-navigation']}, /does not match/],
['extra source metadata', value => {value.reconciliation.reports.extra = {}}, /all seven/],
['missing source report', value => {delete value.reports['public-navigation']}, /did not pass/],
['failed source report', value => {value.reports['public-navigation'].passed = false}, /did not pass/],
['failed source cleanup', value => {value.reports['public-indexed-retrieval-isolation'].cleanup[0].passed = false}, /cleanup did not pass/],
['missing residual check', value => {value.reconciliation.checks.pop()}, /missing required residual checks/],
['duplicate residual check', value => {value.reconciliation.checks[1] = value.reconciliation.checks[0]}, /missing required residual checks/],
['unknown residual check', value => {value.reconciliation.checks[0].name = 'unrelated'}, /missing required residual checks/],
['invalid numeric count', value => {value.reconciliation.checks[0].count = '0'}, /residual rows/],
['positive count with passed true', value => {value.reconciliation.checks[0].count = 1}, /residual rows/],
['missing count', value => {delete value.reconciliation.checks[0].count}, /residual rows/],
['wrong directory', value => {value.reconciliation.checks.at(-1).path = '/unrelated/path'}, /wrong attachment directory/],
['missing directory', value => {delete value.reconciliation.checks.at(-1).path}, /wrong attachment directory/],
['unsafe file prefix', value => {value.reports['public-resource-files'].prefix = '../../private'}, /wrong attachment directory/],
]) {
test(`blocks publication for ${label}`, () => {
const value = reconciliationFixture()
mutate(value)
assert.throws(() => assertFixtureReconciliation(value.manifest, value.reconciliation, value.reports, clock), pattern)
})
}
function injectedPublisher(value, missingReconciliation = false) {
const writes = [], reads = []
const module = {exports: {}}
const injectedRequire = name => {
if (name === 'node:fs') return {
readFileSync: file => {
reads.push(path.basename(file))
if (file.endsWith('/manifest.json')) return JSON.stringify(value.manifest)
if (file.endsWith('/public-fixture-reconciliation.json')) {
if (missingReconciliation) throw new Error('Missing reconciliation file')
return JSON.stringify(value.reconciliation)
}
const report = value.reports[path.basename(file, '.json')]
assert.ok(report, 'Unexpected file read')
return JSON.stringify(report)
},
writeFileSync: (file, contents) => writes.push({file, value: JSON.parse(contents)}),
}
if (name === 'node:child_process') return {execFileSync: () => assert.fail('External command forbidden')}
return require(name)
}
class FrozenDate extends Date {
constructor(...args) { super(...(args.length ? args : [clock])) }
static now() { return clock }
}
const context = {require: injectedRequire, module, __dirname, URL, AbortSignal, Date: FrozenDate,
fetch: async () => ({ok: true, json: async () => ({release: value.manifest.release})}), console: {log() {}}}
vm.runInNewContext(fs.readFileSync(path.join(__dirname, 'publish_public_runtime.cjs'), 'utf8')
+ '\n;globalThis.publish = markPublished;', context)
return {writes, reads, publish: () => context.publish(value.manifest.release)}
}
test('mark-published cannot write either manifest when reconciliation is absent or failed', async () => {
for (const absent of [true, false]) {
const value = reconciliationFixture()
value.reconciliation.passed = false
const publisher = injectedPublisher(value, absent)
await assert.rejects(publisher.publish(), /Missing reconciliation file|did not pass/)
assert.deepEqual(publisher.writes, [])
assert.ok(publisher.reads.includes('public-fixture-reconciliation.json'))
}
})
test('mark-published persists only after current reports and their final reconciliation pass', async () => {
const value = reconciliationFixture()
const publisher = injectedPublisher(value)
await publisher.publish()
assert.equal(publisher.writes.length, 2)
assert.ok(publisher.reads.includes('public-fixture-reconciliation.json'))
for (const write of publisher.writes) {
assert.equal(write.value.stage, 'published')
assert.equal(write.value.fixture_reconciliation_checked_at, value.reconciliation.checked_at)
assert.deepEqual(write.value.public_verification_reports, publicVerificationReports)
}
})
test('every required producer captures execution start before requests and writes it in its report', () => {
for (const name of ['verify_public_release', 'verify_navigation_parity', 'verify_access_control',
'verify_resource_isolation_live', 'verify_resource_files_live', 'verify_public_tenant_isolation',
'verify_indexed_retrieval_isolation']) {
const source = fs.readFileSync(path.join(__dirname, `${name}.cjs`), 'utf8')
const start = source.indexOf('const startedAt = new Date().toISOString()')
assert.ok(start >= 0 && start < source.indexOf('fetch('), `${name}: capture start before requests`)
assert.match(source, /started_at:\s*startedAt/, `${name}: persist captured start`)
}
})
+123
View File
@@ -0,0 +1,123 @@
#!/bin/bash
# 快速启动开发环境的一键脚本
# 此脚本会在一个终端中启动所有必需的服务
# 设置颜色
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m' # 无颜色
# 获取项目根目录
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
log_info() {
printf "%b\n" "${BLUE}[INFO]${NC} $1"
}
log_success() {
printf "%b\n" "${GREEN}[SUCCESS]${NC} $1"
}
log_error() {
printf "%b\n" "${RED}[ERROR]${NC} $1"
}
log_warning() {
printf "%b\n" "${YELLOW}[WARNING]${NC} $1"
}
echo ""
printf "%b\n" "${GREEN}========================================${NC}"
printf "%b\n" "${GREEN} WeKnora 快速开发环境启动${NC}"
printf "%b\n" "${GREEN}========================================${NC}"
echo ""
# 检查是否在项目根目录
cd "$PROJECT_ROOT"
# 1. 启动基础设施
log_info "步骤 1/3: 启动基础设施服务..."
./scripts/dev.sh start
if [ $? -ne 0 ]; then
log_error "基础设施启动失败"
exit 1
fi
# 等待服务就绪
log_info "等待服务启动完成..."
sleep 5
# 2. 询问是否启动后端
echo ""
log_info "步骤 2/3: 启动后端应用"
printf "%b" "${YELLOW}是否在当前终端启动后端? (y/N): ${NC}"
read -r start_backend
if [ "$start_backend" = "y" ] || [ "$start_backend" = "Y" ]; then
log_info "启动后端..."
# 在后台启动后端
nohup bash -c 'cd "'$PROJECT_ROOT'" && ./scripts/dev.sh app' > "$PROJECT_ROOT/logs/backend.log" 2>&1 &
BACKEND_PID=$!
echo $BACKEND_PID > "$PROJECT_ROOT/tmp/backend.pid"
log_success "后端已在后台启动 (PID: $BACKEND_PID)"
log_info "查看后端日志: tail -f $PROJECT_ROOT/logs/backend.log"
else
log_warning "跳过后端启动"
log_info "稍后在新终端运行: make dev-app 或 ./scripts/dev.sh app"
fi
# 3. 询问是否启动前端
echo ""
log_info "步骤 3/3: 启动前端应用"
printf "%b" "${YELLOW}是否在当前终端启动前端? (y/N): ${NC}"
read -r start_frontend
if [ "$start_frontend" = "y" ] || [ "$start_frontend" = "Y" ]; then
log_info "启动前端..."
# 在后台启动前端
nohup bash -c 'cd "'$PROJECT_ROOT'/frontend" && npm run dev' > "$PROJECT_ROOT/logs/frontend.log" 2>&1 &
FRONTEND_PID=$!
echo $FRONTEND_PID > "$PROJECT_ROOT/tmp/frontend.pid"
log_success "前端已在后台启动 (PID: $FRONTEND_PID)"
log_info "查看前端日志: tail -f $PROJECT_ROOT/logs/frontend.log"
else
log_warning "跳过前端启动"
log_info "稍后在新终端运行: make dev-frontend 或 ./scripts/dev.sh frontend"
fi
# 显示总结
echo ""
printf "%b\n" "${GREEN}========================================${NC}"
printf "%b\n" "${GREEN} 启动完成!${NC}"
printf "%b\n" "${GREEN}========================================${NC}"
echo ""
log_info "访问地址:"
echo " - 前端: http://localhost:5173"
echo " - 后端 API: http://localhost:8080"
echo " - MinIO Console: http://localhost:9001"
echo ""
log_info "管理命令:"
echo " - 查看服务状态: make dev-status"
echo " - 查看日志: make dev-logs"
echo " - 停止所有服务: make dev-stop"
echo ""
if [ -f "$PROJECT_ROOT/tmp/backend.pid" ] || [ -f "$PROJECT_ROOT/tmp/frontend.pid" ]; then
log_warning "停止后台进程:"
if [ -f "$PROJECT_ROOT/tmp/backend.pid" ]; then
echo " - 停止后端: kill \$(cat $PROJECT_ROOT/tmp/backend.pid)"
fi
if [ -f "$PROJECT_ROOT/tmp/frontend.pid" ]; then
echo " - 停止前端: kill \$(cat $PROJECT_ROOT/tmp/frontend.pid)"
fi
fi
echo ""
log_success "开发环境已就绪,开始编码吧!"
echo ""
+78
View File
@@ -0,0 +1,78 @@
const fs = require('node:fs')
const path = require('node:path')
const assert = require('node:assert/strict')
const { execFileSync } = require('node:child_process')
const root = path.resolve(__dirname, '../..')
const docker = (...args) => execFileSync('docker', ['--context', 'colima-yawan', ...args], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }).trim()
const allIDs = ['16c3e59c-4bb6-41cf-ae0a-6d0941b3d032', '20b9280e-ed73-4700-8fa5-944d6ae6dd5f', 'bb86b44a-576b-4bf8-a6bf-8f1baca3a845', 'badd084b-f229-4005-8fb3-196fbe139fe0', '5d28a7e5-e00e-44a7-a0e1-c72342cdcf68', '3ae9257d-928a-4886-8101-7cfd0ed8c27c']
const args = process.argv.slice(2)
const revisionArg = args.find(arg => arg.startsWith('--backup-revision='))
const chunksArg = args.find(arg => arg.startsWith('--chunks='))
const chunks = chunksArg ? [...new Set(chunksArg.slice('--chunks='.length).split(','))] : []
assert.ok(chunks.length <= 51, 'At most 51 explicitly reviewed chunks may be rebuilt')
for (const chunk of chunks) assert.match(chunk, /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/)
assert.ok(!chunksArg || (chunks.length > 0 && args.includes('--product-only')), 'Targeted repair requires product-only scope')
const revision = revisionArg?.slice('--backup-revision='.length) || '20260919'
assert.match(revision, /^[a-z0-9_]{1,32}$/, 'Backup revision must be an SQL-safe identifier')
assert.ok(args.every(arg => ['--apply', '--product-only'].includes(arg) || arg === revisionArg || arg === chunksArg), 'Unknown repair option')
const ids = args.includes('--product-only') ? allIDs.slice(0, 1) : allIDs
const recordsBackup = `native_graph_grounding_backup_${revision}`
const sourcesBackup = `native_graph_source_grounding_backup_${revision}`
const scope = `tenant_id=10059 AND knowledge_base_id IN (${ids.map(id => `'${id}'`).join(',')})`
const chunkValues = chunks.map(id => `'${id}'`).join(',')
const recordFilter = chunks.length ? ` AND (source_table='knowledgegraphchunkstate' AND payload->>'native_chunk_id' IN (${chunkValues}) OR source_table='knowledgegraphevidence' AND payload->>'source_chunk_id' IN (${chunks.map(id => `'native:${id}'`).join(',')}))` : ''
const sourceFilter = chunks.length ? ` AND native_chunk_id IN (${chunkValues})` : ''
const sql = query => docker('exec', 'WeKnora-postgres', 'psql', '-XAt', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-c', query)
function queueStatus() {
const inspected = JSON.parse(docker('inspect', 'yawan-native-acceptance'))[0]
const env = Object.fromEntries(inspected.Config.Env.map(value => { const i = value.indexOf('='); return [value.slice(0, i), value.slice(i + 1)] }))
const previous = process.env.REDISCLI_AUTH
process.env.REDISCLI_AUTH = env.REDIS_PASSWORD || ''
try {
const result = {}
for (const [state, command] of [['active', 'LLEN'], ['pending', 'LLEN'], ['scheduled', 'ZCARD'], ['retry', 'ZCARD']]) {
const value = docker('exec', '-e', 'REDISCLI_AUTH', 'WeKnora-redis', 'redis-cli', '-n', env.REDIS_DB || '0', command, `asynq:{graph}:${state}`)
assert.match(value, /^\d+$/)
result[state] = Number(value)
}
return result
} finally {
if (previous === undefined) delete process.env.REDISCLI_AUTH
else process.env.REDISCLI_AUTH = previous
}
}
function main() {
const queue = queueStatus()
const count = Number(sql(`SELECT count(*) FROM native_graph_records WHERE ${scope} AND payload->>'origin'='native_extraction_v1'`))
if (chunks.length) {
const matched = Number(sql(`SELECT count(*) FROM chunks WHERE ${scope} AND id IN (${chunkValues}) AND deleted_at IS NULL AND is_enabled AND chunk_type='text'`))
assert.equal(matched, chunks.length, 'Every selected source must belong to the active product knowledge base')
}
const directCount = Number(sql(`SELECT count(*) FROM native_graph_records WHERE ${scope} AND payload->>'origin'='native_extraction_v1'${recordFilter}`))
if (!process.argv.includes('--apply')) { console.log(JSON.stringify({ queue, derived_records_to_backup: count, direct_records_to_rebuild: directCount, selected_chunks: chunks })); return }
assert.equal(docker('inspect', 'yawan-native-graph-recovery', '--format', '{{.State.Running}}'), 'false', 'Stop graph enqueuer before reset')
assert.ok(Object.values(queue).every(n => n === 0), 'Wait for graph queue to drain before rebuilding')
assert.ok(count > 0 && count < 20000, 'Unexpected graph repair scope')
const backup = sql(`SELECT json_build_object('records', (SELECT json_agg(r) FROM native_graph_records r WHERE ${scope} AND payload->>'origin'='native_extraction_v1'), 'sources', (SELECT json_agg(s) FROM native_graph_sources s WHERE ${scope} AND mapping_state='native_extraction'))`)
const backupPath = path.join(root, 'acceptance', `native-graph-before-grounding-${Date.now()}.json`)
const parsed = JSON.parse(backup)
assert.equal(parsed.records.length, count)
fs.writeFileSync(backupPath, backup, { flag: 'wx', mode: 0o600 })
// A transactional database copy also keeps rollback possible if the host disk disappears.
sql(`BEGIN;
${ids.map(id => `SELECT pg_advisory_xact_lock(hashtextextended('native-graph:10059:${id}',0));`).join('\n')}
CREATE TABLE ${recordsBackup} AS SELECT * FROM native_graph_records WHERE ${scope} AND payload->>'origin'='native_extraction_v1';
CREATE TABLE ${sourcesBackup} AS SELECT * FROM native_graph_sources WHERE ${scope} AND mapping_state='native_extraction';
DO $$ BEGIN IF (SELECT count(*) FROM ${recordsBackup}) <> ${count} THEN RAISE EXCEPTION 'Graph changed after backup'; END IF; END $$;
DELETE FROM native_graph_records WHERE ${scope} AND payload->>'origin'='native_extraction_v1'${recordFilter};
DELETE FROM native_graph_sources WHERE ${scope} AND mapping_state='native_extraction'${sourceFilter};
${chunks.length ? `DELETE FROM native_graph_records r WHERE ${scope} AND r.payload->>'origin'='native_extraction_v1'
AND r.source_table IN ('knowledgeentity','knowledgerelation')
AND NOT EXISTS (SELECT 1 FROM native_graph_records e WHERE e.tenant_id=r.tenant_id AND e.knowledge_base_id=r.knowledge_base_id
AND e.source_table='knowledgegraphevidence' AND e.payload->>'subject_id'=r.legacy_id::text
AND e.payload->>'subject_type'=CASE r.source_table WHEN 'knowledgeentity' THEN 'entity' ELSE 'relation' END);` : ''}
COMMIT;`)
const remaining = Number(sql(`SELECT count(*) FROM native_graph_records WHERE ${scope} AND payload->>'origin'='native_extraction_v1'`))
console.log(JSON.stringify({ removed_derived_records: count - remaining, backed_up_sources: parsed.sources?.length || 0, selected_chunks: chunks, backup: backupPath, database_backups: [recordsBackup, sourcesBackup], source_documents_and_indexes_preserved: true }))
}
try { main() } catch (error) { console.error(error.message); process.exitCode = 1 }
+301
View File
@@ -0,0 +1,301 @@
const fs = require('node:fs')
const path = require('node:path')
const assert = require('node:assert/strict')
const { execFileSync } = require('node:child_process')
const { attemptOf, latestAttempts, planRecovery, validatePublishedRecoveryFix, validateFinalRecoveryTrigger, supportedReviewedAttempt, describeReadCoverage, durableRecoveryStatus, validateDurableRecoveryReview } = require('./wiki_recovery_policy.cjs')
const { recoveryFingerprintsSQL } = require('./wiki_recovery_budget_sql.cjs')
const { loadClosures, annotateClosed } = require('./wiki_recovery_closures.cjs')
const { validateSixthDiagnosticEvidence, validateSixthReplayEvidence, verifyReplayInputFiles, replayPins, candidateSourceTreeHash, sixthGuardPausedOperation } = require('./wiki_sixth_recovery_review.cjs')
const { seventhPins, validateSeventhCorrectionEvidence, verifySeventhBuildFiles, seventhRecoveryEnqueueSQL } = require('./wiki_seventh_recovery_review.cjs')
const fingerprints = recoveryFingerprintsSQL()
const root = path.resolve(__dirname, '../..')
const directory = path.join(root, 'acceptance/wiki-deadletter-recovery-20260919')
const tenant = 10059
const database = 'yawan_native_acceptance_20260915'
const table = 'wiki_dead_letter_recoveries'
// The service dead-letters at 6. Starting at 5 permits one new recovery
// attempt, without repeating the historical six-attempt model-call burst.
const recoveryInitialFailCount = 5
const args = process.argv.slice(2)
const apply = args.includes('--apply')
const triggerOnly = args.includes('--trigger')
const idsArg = args.find(arg => arg.startsWith('--ids='))?.slice(6)
const ids = idsArg ? idsArg.split(',').map(Number) : []
const attempt = Number(args.find(arg => arg.startsWith('--attempt='))?.slice(10) || 1)
const retryReason = args.find(arg => arg.startsWith('--reason='))?.slice(9) || ''
const reviewPath = args.find(arg => arg.startsWith('--review-file='))?.slice(14)
assert.ok(ids.every(id => Number.isSafeInteger(id) && id >= 1 && id <= 33), 'Only original dead letters 1..33 are allowed')
assert.equal(new Set(ids).size, ids.length, 'Duplicate IDs')
assert.ok(!apply || ids.length === 1, 'Apply requires --ids= with exactly one original record')
assert.ok(!args.some(arg => !['--apply', '--trigger'].includes(arg) && !['--ids=', '--attempt=', '--reason=', '--review-file='].some(prefix => arg.startsWith(prefix))), 'Unknown argument')
assert.ok(attempt === 1 || attempt === 2 || (supportedReviewedAttempt(attempt) && reviewPath), 'Further bounded recovery requires an explicit deployed-fix review file')
assert.ok(attempt === 1 || apply, 'Another attempt requires --apply')
assert.ok(!apply || reviewPath, 'Every new recovery requires --review-file= with an explicit persistent request limit and deadline')
assert.ok(!reviewPath || apply, 'Review files are only valid for a bounded manual recovery')
const literal = value => `'${String(value).replaceAll("'", "''")}'`
const docker = (args, input) => execFileSync('docker', ['--context', 'colima-yawan', ...args], {
input, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], maxBuffer: 32 * 1024 * 1024,
})
function sql(statement, readOnly = true) {
const args = ['exec', '-i']
if (readOnly) args.push('-e', 'PGOPTIONS=-c default_transaction_read_only=on')
return docker([...args, 'WeKnora-postgres', 'psql', '-X', '-q', '-U', 'postgres', '-d', database, '-At', '-v', 'ON_ERROR_STOP=1'], statement).trim()
}
const queryJSON = (statement, readOnly = true) => JSON.parse(sql(statement, readOnly))
const hasLedger = () => sql(`SELECT to_regclass('public.${table}') IS NOT NULL`) === 't'
const writePrivate = (name, value) => fs.writeFileSync(path.join(directory, name), JSON.stringify(value, null, 2) + '\n', { mode: 0o600, flag: 'wx' })
const jsonArray = expression => `CASE WHEN jsonb_typeof(${expression})='array' THEN ${expression} ELSE '[]'::jsonb END`
const hasSource = (page, knowledge) => `EXISTS (SELECT 1 FROM jsonb_array_elements_text(${jsonArray(`${page}.source_refs`)}) ref WHERE split_part(ref,'|',1)=${knowledge})`
function inventory(fingerprintIDs = ids.length ? ids : [10]) {
return queryJSON(`SELECT coalesce(json_agg(row_to_json(x)), '[]') FROM (
SELECT d.id,d.tenant_id,d.related_id AS knowledge_id,d.scope_id AS knowledge_base_id,k.name AS knowledge_base,
k.wiki_config->>'synthesis_model_id' AS model_id,
k.embedding_model_id,
CASE WHEN d.id IN (${fingerprintIDs.join(',')}) THEN ${fingerprints.source} END AS source_fingerprint,
CASE WHEN d.id IN (${fingerprintIDs.join(',')}) THEN ${fingerprints.model} END AS model_fingerprint,
n.parse_status,n.pending_subtasks_count,
(d.payload->>'op'='ingest' AND d.payload->>'knowledge_id'=d.related_id
AND d.scope='knowledge_base' AND d.fail_count=6 AND n.deleted_at IS NULL
AND k.deleted_at IS NULL AND k.indexing_strategy->>'wiki_enabled'='true'
AND n.parse_status='completed' AND n.pending_subtasks_count=0
AND EXISTS (SELECT 1 FROM models m WHERE m.id=k.wiki_config->>'synthesis_model_id' AND m.tenant_id=d.tenant_id AND m.type='KnowledgeQA' AND m.deleted_at IS NULL)) AS eligible,
(SELECT count(*) FROM chunks c WHERE c.tenant_id=d.tenant_id AND c.knowledge_id=n.id AND c.deleted_at IS NULL) AS chunks,
(SELECT coalesce(sum(length(c.content)),0) FROM chunks c WHERE c.tenant_id=d.tenant_id AND c.knowledge_id=n.id AND c.deleted_at IS NULL) AS chunk_chars,
(SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=d.tenant_id AND p.knowledge_base_id=d.scope_id AND p.deleted_at IS NULL AND ${hasSource('p', 'd.related_id')}) AS pages,
(SELECT count(*) FROM task_pending_ops o WHERE o.tenant_id=d.tenant_id AND o.scope_id=d.scope_id AND o.task_type='wiki:ingest' AND o.dedup_key=d.related_id) AS pending
FROM task_dead_letters d
JOIN knowledges n ON n.id=d.related_id AND n.tenant_id=d.tenant_id AND n.knowledge_base_id=d.scope_id
JOIN knowledge_bases k ON k.id=d.scope_id AND k.tenant_id=d.tenant_id
WHERE d.tenant_id=${tenant} AND d.task_type='wiki:ingest' AND d.id BETWEEN 1 AND 33
ORDER BY chunk_chars,d.id
) x`)
}
function status() {
if (!hasLedger()) return []
const budgetsAvailable = sql("SELECT to_regclass('public.wiki_recovery_budgets') IS NOT NULL") === 't'
const rows = queryJSON(`SELECT coalesce(json_agg(row_to_json(x)), '[]') FROM (
SELECT r.*,coalesce((to_jsonb(r)->>'attempt_number')::int,1) AS attempt_number,o.claimed_at,o.fail_count,
${budgetsAvailable ? '(SELECT row_to_json(b) FROM wiki_recovery_budgets b WHERE b.tenant_id=r.tenant_id AND b.dead_letter_id=r.dead_letter_id AND b.attempt_number=r.attempt_number AND b.pending_op_id=r.pending_op_id)' : 'NULL::json'} AS recovery_budget,
s.status AS span_status,s.error_code,
s.output->>'pages_written' AS pages_written,s.output->>'pages_dropped' AS pages_dropped,
s.output->>'pages_total' AS pages_total,s.output->>'failed_slug_writes' AS failed_slug_writes,
s.output->>'classify_batches' AS classify_batches,
s.output->>'read_coverage_version' AS read_coverage_version,
(s.output->>'source_read_complete')::boolean AS source_read_complete,
(s.output->>'source_runes')::bigint AS source_runes,
(s.output->>'source_windows')::int AS source_windows,
coalesce((SELECT jsonb_object_agg(c.item_key,c.payload::jsonb) FROM wiki_ingest_checkpoints c
WHERE c.tenant_id=r.tenant_id AND c.knowledge_base_id=r.knowledge_base_id AND c.knowledge_id=r.knowledge_id
AND c.stage='read_progress' AND c.updated_at>=r.queued_at AND c.updated_at<coalesce(r.next_queued_at,'infinity')
AND c.fingerprint=(SELECT latest.fingerprint FROM wiki_ingest_checkpoints latest
WHERE latest.tenant_id=r.tenant_id AND latest.knowledge_base_id=r.knowledge_base_id AND latest.knowledge_id=r.knowledge_id
AND latest.stage='read_progress' AND latest.updated_at>=r.queued_at AND latest.updated_at<coalesce(r.next_queued_at,'infinity')
ORDER BY latest.updated_at DESC,latest.fingerprint LIMIT 1)), '{}'::jsonb) AS read_progress,
coalesce((SELECT jsonb_object_agg(c.item_key,c.payload::jsonb) FROM wiki_ingest_checkpoints c
WHERE c.tenant_id=r.tenant_id AND c.knowledge_base_id=r.knowledge_base_id AND c.knowledge_id=r.knowledge_id
AND c.stage='page_progress' AND c.updated_at>=r.queued_at AND c.updated_at<coalesce(r.next_queued_at,'infinity')
AND c.fingerprint=(SELECT latest.fingerprint FROM wiki_ingest_checkpoints latest
WHERE latest.tenant_id=r.tenant_id AND latest.knowledge_base_id=r.knowledge_base_id AND latest.knowledge_id=r.knowledge_id
AND latest.stage='page_progress' AND latest.updated_at>=r.queued_at AND latest.updated_at<coalesce(r.next_queued_at,'infinity')
ORDER BY latest.updated_at DESC,latest.fingerprint LIMIT 1)), '{}'::jsonb) AS page_progress,
(SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=r.tenant_id AND p.knowledge_base_id=r.knowledge_base_id AND p.deleted_at IS NULL AND ${hasSource('p', 'r.knowledge_id')}) AS current_pages,
(SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=r.tenant_id AND p.knowledge_base_id=r.knowledge_base_id AND p.deleted_at IS NULL AND p.page_type='summary' AND p.status='published' AND length(trim(p.content))>0 AND ${hasSource('p', 'r.knowledge_id')}) AS published_summaries,
jsonb_array_length(${jsonArray("s.output->'pages_written_preview'")}) AS expected_pages,
(SELECT count(*) FROM jsonb_array_elements(${jsonArray("s.output->'pages_written_preview'")}) expected
WHERE NOT EXISTS (SELECT 1 FROM wiki_pages p WHERE p.tenant_id=r.tenant_id AND p.knowledge_base_id=r.knowledge_base_id
AND p.slug=expected->>'slug' AND p.deleted_at IS NULL AND p.status='published' AND length(trim(p.content))>0 AND ${hasSource('p', 'r.knowledge_id')})) AS missing_expected_pages,
(SELECT count(*) FROM knowledge_processing_spans child WHERE child.knowledge_id=r.knowledge_id AND child.started_at>=r.queued_at AND child.started_at<coalesce(r.next_queued_at,'infinity')
AND child.name LIKE 'postprocess.wiki.page[%'
AND (child.status='failed' OR child.output->>'addition_failed'='true')) AS failed_page_spans,
(SELECT count(*) FROM task_dead_letters d WHERE d.tenant_id=r.tenant_id AND d.task_type='wiki:ingest' AND d.scope_id=r.knowledge_base_id AND d.related_id=r.knowledge_id AND d.id>r.dead_letter_highwater AND d.failed_at<coalesce(r.next_queued_at,'infinity')) AS new_failures,
(SELECT count(*) FROM task_pending_ops f WHERE f.tenant_id=r.tenant_id AND f.scope_id=r.knowledge_base_id AND f.task_type='wiki:finalize'
AND (coalesce(r.deployed_fix_review->>'budget_version','')<>'1' OR (f.payload->>'recovery_dead_letter_id'=r.dead_letter_id::text AND f.payload->>'recovery_attempt'=r.attempt_number::text AND f.payload->>'recovery_pending_op_id'=r.pending_op_id::text))) AS pending_finalize,
o.id IS NOT NULL AS pending
FROM (SELECT ledger.*,lead(queued_at) OVER (PARTITION BY tenant_id,dead_letter_id ORDER BY queued_at) AS next_queued_at FROM ${table} ledger) r
LEFT JOIN task_pending_ops o ON o.id=r.pending_op_id AND o.tenant_id=r.tenant_id
LEFT JOIN LATERAL (SELECT status,error_code,output FROM knowledge_processing_spans s
WHERE s.knowledge_id=r.knowledge_id AND s.name='postprocess.wiki' AND s.started_at>=r.queued_at AND s.started_at<coalesce(r.next_queued_at,'infinity')
ORDER BY s.started_at DESC,s.id DESC LIMIT 1) s ON true
WHERE r.tenant_id=${tenant} ORDER BY r.dead_letter_id,r.queued_at
) x`).map(row => ({ ...row, full_read_status: describeReadCoverage(row), recovery_status: durableRecoveryStatus(row) || (row.pending ? (row.claimed_at ? 'running' : 'queued')
: row.new_failures > 0 || row.span_status === 'failed' ? 'failed'
: row.span_status === 'skipped' ? 'skipped'
: Number(row.pages_dropped) > 0 || Number(row.failed_slug_writes) > 0 || row.missing_expected_pages > 0 || row.failed_page_spans > 0 ? 'partial'
: row.span_status === 'done' && row.pages_dropped !== null && Number(row.pages_dropped) === 0
&& row.current_pages > 0 && row.published_summaries > 0 && row.expected_pages > 0
&& (row.deployed_fix_review?.budget_version !== 1 || (row.recovery_budget?.status === 'completed' && describeReadCoverage(row) === 'source_processed'))
&& row.expected_pages === Number(row.pages_written) && row.expected_pages === Number(row.pages_total) ? 'wiki_recovered'
: 'unverified') }))
return annotateClosed(rows, loadClosures(directory, tenant), tenant)
}
function verifyPublishedFix(review) {
const manifest = JSON.parse(fs.readFileSync(path.join(root, 'public-release.json'), 'utf8'))
const runtime = JSON.parse(docker(['inspect', 'yawan-native-acceptance', '--format', '{"running":{{json .State.Running}},"image_id":{{json .Image}},"release":{{json (index .Config.Labels "yawan.release")}}}']))
const validated = validatePublishedRecoveryFix(review, manifest, runtime)
if (review.max_attempt === 6) {
const sourceHash = candidateSourceTreeHash(path.resolve(__dirname, '..'))
if (review.candidate_diagnostic.kind === replayPins.kind) {
verifyReplayInputFiles(root)
const evidence = fs.readFileSync(path.join(root, 'acceptance/wiki-layout-identity-replay-private-20260921/evidence.json'))
validateSixthReplayEvidence(validated, evidence, sourceHash)
} else {
const evidence = fs.readFileSync(path.join(root, 'acceptance/wiki-repair-window-diagnostic-private-20260920/evidence.json'))
validateSixthDiagnosticEvidence(validated, evidence, sourceHash)
}
}
if (review.max_attempt === 7) {
const evidenceDirectory = path.join(root, seventhPins.evidence_directory)
const build = verifySeventhBuildFiles(root)
const binaryHash = docker(['exec', 'yawan-native-acceptance', 'sha256sum', '/app/WeKnora']).trim()
assert.match(binaryHash, /^[a-f0-9]{64}\s+\/app\/WeKnora$/, 'Running API binary hash unavailable')
validateSeventhCorrectionEvidence(validated, fs.readFileSync(path.join(evidenceDirectory, 'evidence.json')),
build.sourceTreeHash, binaryHash.slice(0, 64), fs.readFileSync(path.join(evidenceDirectory, 'regression-tests.jsonl')),
fs.readFileSync(path.join(evidenceDirectory, 'semantic-review.json')), build.buildManifestBytes, build.frozenBinarySHA)
}
return validated
}
function trigger(rows) {
if (!rows.length) return { triggered_scopes: 0 }
assert.ok(rows.every(row => [1, 2].includes(attemptOf(row)) || supportedReviewedAttempt(attemptOf(row))), 'Recovery trigger cannot exceed the attempt limit')
assert.ok(rows.every(row => row.deployed_fix_review?.budget_version === 1), 'Legacy recovery receipts cannot execute without a durable reviewed budget')
const finalRows = rows
if (finalRows.length) {
const sources = inventory(rows.map(row => row.dead_letter_id))
const history = status()
for (const row of finalRows) {
const review = verifyPublishedFix(row.deployed_fix_review)
validateFinalRecoveryTrigger(row, sources.find(source => source.id === row.dead_letter_id), history, review)
}
}
const scopes = [...new Set(rows.map(row => row.knowledge_base_id))].map(id => ({tenant_id: tenant, knowledge_base_id: id, language: 'zh-CN'}))
docker(['cp', path.join(root, 'acceptance/wiki-backfill-trigger'), 'yawan-native-acceptance:/tmp/wiki-backfill-trigger'])
return JSON.parse(docker(['exec', '-i', 'yawan-native-acceptance', '/tmp/wiki-backfill-trigger'], JSON.stringify(scopes)))
}
function main() {
const before = inventory()
const prior = status()
let triggered
if (apply) {
let review
if (reviewPath) {
review = verifyPublishedFix(JSON.parse(fs.readFileSync(reviewPath, 'utf8')))
}
const selected = ids.map(id => before.find(row => row.id === id))
assert.ok(selected.every(row => row?.eligible && row.chunks > 0), 'Every selected original record must remain eligible')
validateDurableRecoveryReview(selected[0], review, attempt)
const fresh = planRecovery(selected, prior, attempt, retryReason, review)
if (fresh.length) {
fs.mkdirSync(directory, { recursive: true, mode: 0o700 })
const run = Date.now()
const backup = queryJSON(`SELECT coalesce(json_agg(row_to_json(d)), '[]') FROM task_dead_letters d WHERE d.tenant_id=${tenant} AND d.task_type='wiki:ingest' AND d.id IN (${fresh.map(row => row.id).join(',')})`)
writePrivate(`original-records-${run}.json`, backup)
writePrivate(`before-${run}.json`, { tenant_id: tenant, attempt_number: attempt, retry_reason: retryReason, deployed_fix_review: review, selected: fresh, all_originals: before, prior })
// The receipt and pending operation commit together. Re-running after a lost
// response cannot enqueue the original dead letter a second time.
sql(`BEGIN;
SELECT pg_advisory_xact_lock(${tenant}, 20260919);
CREATE TABLE IF NOT EXISTS ${table} (
tenant_id bigint NOT NULL,dead_letter_id bigint NOT NULL,knowledge_id varchar NOT NULL,
knowledge_base_id varchar NOT NULL,model_id varchar NOT NULL,pending_op_id bigint NOT NULL,
queued_at timestamptz NOT NULL,baseline_pages bigint NOT NULL,baseline_chunks bigint NOT NULL,
chunk_chars bigint NOT NULL,dead_letter_highwater bigint NOT NULL,
PRIMARY KEY(tenant_id,dead_letter_id));
ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS attempt_number integer NOT NULL DEFAULT 1;
ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS retry_reason text NOT NULL DEFAULT '';
ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS deployed_fix_review jsonb;
ALTER TABLE ${table} DROP CONSTRAINT IF EXISTS ${table}_pkey;
ALTER TABLE ${table} ADD PRIMARY KEY(tenant_id,dead_letter_id,attempt_number);
LOCK TABLE task_pending_ops IN SHARE ROW EXCLUSIVE MODE;
LOCK TABLE knowledges, knowledge_bases, chunks, models IN SHARE MODE;
${attempt === 7 ? 'LOCK TABLE wiki_recovery_budgets, task_dead_letters, knowledge_processing_spans, wiki_ingest_checkpoints IN SHARE MODE;' : ''}
WITH source AS (
SELECT d.*,nextval(pg_get_serial_sequence('task_pending_ops','id')) AS recovery_op_id,
k.wiki_config->>'synthesis_model_id' AS model_id
FROM task_dead_letters d JOIN knowledges n ON n.id=d.related_id AND n.tenant_id=d.tenant_id AND n.knowledge_base_id=d.scope_id
JOIN knowledge_bases k ON k.id=d.scope_id AND k.tenant_id=d.tenant_id
WHERE d.tenant_id=${tenant} AND d.task_type='wiki:ingest' AND d.scope='knowledge_base'
AND d.id IN (${fresh.map(row => row.id).join(',')}) AND d.payload->>'op'='ingest' AND d.payload->>'knowledge_id'=d.related_id
AND n.deleted_at IS NULL AND k.deleted_at IS NULL AND n.parse_status='completed' AND n.pending_subtasks_count=0
AND k.indexing_strategy->>'wiki_enabled'='true'
${review ? `AND d.related_id=${literal(review.knowledge_id)} AND d.scope_id=${literal(review.knowledge_base_id)} AND k.wiki_config->>'synthesis_model_id'=${literal(review.model_id)}` : ''}
AND ${fingerprints.source}=${literal(review.source_fingerprint)} AND ${fingerprints.model}=${literal(review.model_fingerprint)}
AND k.embedding_model_id=${literal(review.embedding_model_id)} AND ${literal(review.deadline_at)}::timestamptz>now()
${attempt === 5 ? `AND EXISTS (SELECT 1 FROM ${table} p WHERE p.tenant_id=d.tenant_id AND p.dead_letter_id=d.id AND p.attempt_number=4 AND p.pending_op_id=4143)` : ''}
${attempt === 6 ? `AND d.id=10 AND d.tenant_id=10059
AND ${review.max_provider_requests} BETWEEN 1 AND 1000
AND ${literal(review.budget_started_at)}::timestamptz<=now()
AND ${literal(review.deadline_at)}::timestamptz>${literal(review.budget_started_at)}::timestamptz
AND ${literal(review.deadline_at)}::timestamptz<=${literal(review.budget_started_at)}::timestamptz+interval '120 minutes'
AND EXISTS (SELECT 1 FROM ${table} p JOIN wiki_recovery_budgets b
ON b.tenant_id=p.tenant_id AND b.dead_letter_id=p.dead_letter_id AND b.attempt_number=p.attempt_number AND b.pending_op_id=p.pending_op_id
WHERE p.tenant_id=d.tenant_id AND p.dead_letter_id=d.id AND p.attempt_number=5 AND p.pending_op_id=4144
AND p.knowledge_id=d.related_id AND p.knowledge_base_id=d.scope_id AND p.model_id=k.wiki_config->>'synthesis_model_id'
AND b.status='stopped_budget' AND b.knowledge_id=d.related_id AND b.knowledge_base_id=d.scope_id
AND b.model_id=k.wiki_config->>'synthesis_model_id' AND b.embedding_model_id=k.embedding_model_id
AND b.source_fingerprint=${literal(review.source_fingerprint)} AND b.model_fingerprint=${literal(review.model_fingerprint)}
AND (EXISTS (SELECT 1 FROM task_dead_letters failure WHERE failure.tenant_id=p.tenant_id
AND failure.task_type='wiki:ingest' AND failure.scope_id=p.knowledge_base_id
AND failure.related_id=p.knowledge_id AND failure.id>p.dead_letter_highwater)
OR (b.stop_reason='guard_rejected' AND b.last_phase='wiki_candidate_slug' AND b.reserved_provider_requests=4
AND jsonb_typeof(b.paused_payload::jsonb)='array' AND jsonb_array_length(b.paused_payload::jsonb)=1
AND b.paused_payload::jsonb @> ${literal(JSON.stringify([sixthGuardPausedOperation]))}::jsonb
AND (SELECT span.status='failed' AND span.error_code='EXTRACT_FAILED' FROM knowledge_processing_spans span
WHERE span.knowledge_id=p.knowledge_id AND span.name='postprocess.wiki' AND span.started_at>=p.queued_at
ORDER BY span.started_at DESC,span.id DESC LIMIT 1))))
AND NOT EXISTS (SELECT 1 FROM ${table} later WHERE later.tenant_id=d.tenant_id AND later.dead_letter_id=d.id AND later.attempt_number>5)` : ''}
${attempt === 7 ? seventhRecoveryEnqueueSQL(review, table, literal) : ''}
AND NOT EXISTS (SELECT 1 FROM task_pending_ops f WHERE f.tenant_id=d.tenant_id AND f.scope_id=d.scope_id AND f.task_type='wiki:finalize')
AND NOT EXISTS (SELECT 1 FROM ${table} r WHERE r.tenant_id=d.tenant_id AND r.dead_letter_id=d.id AND r.attempt_number=${attempt})
AND (${attempt}=1 OR EXISTS (SELECT 1 FROM ${table} r WHERE r.tenant_id=d.tenant_id AND r.dead_letter_id=d.id AND r.attempt_number=${attempt - 1}))
AND NOT EXISTS (SELECT 1 FROM ${table} r JOIN task_pending_ops o ON o.id=r.pending_op_id AND o.tenant_id=r.tenant_id WHERE r.tenant_id=${tenant})
AND NOT EXISTS (SELECT 1 FROM task_pending_ops o WHERE o.tenant_id=d.tenant_id AND o.scope_id=d.scope_id AND o.task_type='wiki:ingest' AND o.dedup_key=d.related_id)
), budget AS (
INSERT INTO wiki_recovery_budgets(tenant_id,dead_letter_id,attempt_number,pending_op_id,knowledge_base_id,knowledge_id,model_id,embedding_model_id,
source_fingerprint,model_fingerprint,reviewed_release,reviewed_api_image,max_provider_requests,reserved_provider_requests,deadline_at,status,paused_payload,verified_progress)
SELECT tenant_id,id,${attempt},recovery_op_id,scope_id,related_id,model_id,${literal(review.embedding_model_id)},
${literal(review.source_fingerprint)},${literal(review.model_fingerprint)},${literal(review.fix_release)},${literal(review.api_image_id)},
${review.max_provider_requests},0,${literal(review.deadline_at)}::timestamptz,'active','[]','{}' FROM source RETURNING pending_op_id
), inserted AS (
INSERT INTO task_pending_ops(id,tenant_id,task_type,scope,scope_id,op,dedup_key,payload,fail_count,enqueued_at)
SELECT recovery_op_id,tenant_id,task_type,scope,scope_id,'ingest',related_id,
payload||jsonb_build_object('recovery_dead_letter_id',id,'recovery_attempt',${attempt},'recovery_pending_op_id',recovery_op_id,'recovery_budget_version',1),${recoveryInitialFailCount},now() FROM source
WHERE EXISTS (SELECT 1 FROM budget b WHERE b.pending_op_id=source.recovery_op_id) RETURNING id
) INSERT INTO ${table} (tenant_id,dead_letter_id,knowledge_id,knowledge_base_id,model_id,pending_op_id,queued_at,baseline_pages,baseline_chunks,chunk_chars,dead_letter_highwater,attempt_number,retry_reason,deployed_fix_review)
SELECT d.tenant_id,d.id,d.related_id,d.scope_id,d.model_id,d.recovery_op_id,now(),
(SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=d.tenant_id AND p.knowledge_base_id=d.scope_id AND p.deleted_at IS NULL AND ${hasSource('p', 'd.related_id')}),
(SELECT count(*) FROM chunks c WHERE c.tenant_id=d.tenant_id AND c.knowledge_id=d.related_id AND c.deleted_at IS NULL),
(SELECT coalesce(sum(length(c.content)),0) FROM chunks c WHERE c.tenant_id=d.tenant_id AND c.knowledge_id=d.related_id AND c.deleted_at IS NULL),
(SELECT coalesce(max(id),0) FROM task_dead_letters),${attempt},${literal(retryReason)},${review ? literal(JSON.stringify(review)) + '::jsonb' : 'NULL'}
FROM source d JOIN inserted i ON i.id=d.recovery_op_id;
COMMIT;`, false)
assert.ok(status().some(row => row.dead_letter_id === fresh[0].id && attemptOf(row) === attempt), 'Recovery was not enqueued because state changed; inspect status before retrying')
}
triggered = trigger(status().filter(row => ids.includes(row.dead_letter_id) && attemptOf(row) === attempt && row.pending))
} else if (triggerOnly) {
triggered = trigger(latestAttempts(prior).filter(row => row.pending && (!ids.length || ids.includes(row.dead_letter_id))))
}
const history = status()
const recovered = latestAttempts(history)
const report = { checked_at: new Date().toISOString(), tenant_id: tenant, apply, triggered,
original_dead_letters: before.length, attempted: recovered.length,
remaining_not_attempted: before.filter(row => !recovered.some(recovery => recovery.dead_letter_id === row.id)).length,
scope: 'Wiki recovery only. Original records and checkpoints are preserved. One document per batch. Further attempts require their exact reviewed published fix. Only original dead letter 10 has exact fifth/sixth/seventh-attempt exceptions. The seventh requires separately reviewed fresh source identity correction tests, matching frozen source and binary, a newer published fix, at most 1000 actual provider sends and a fixed 120-minute maximum. No eighth attempt or automatic attempt increase. Terminal budget states stop scheduling. Historical page recovery does not prove full-source reading.',
candidates: before, recoveries: recovered, attempt_history: history }
fs.mkdirSync(directory, { recursive: true, mode: 0o700 })
const reportFile = path.join(directory, `status-${Date.now()}.json`)
writePrivate(path.basename(reportFile), report)
const { candidates, attempt_history, ...summary } = report
console.log(JSON.stringify({ ...summary, report_directory: directory, report_file: reportFile }, null, 2))
}
try { main() } catch (error) {
// Docker/psql exceptions may contain full SQL or sensitive runtime output.
console.error(error instanceof assert.AssertionError ? error.message : 'Wiki recovery stopped; inspect private records and container health. No automatic re-enqueue was attempted.')
process.exitCode = 1
}
+118
View File
@@ -0,0 +1,118 @@
const { test } = require('node:test')
const assert = require('node:assert/strict')
const fs = require('node:fs')
const path = require('node:path')
const vm = require('node:vm')
const policy = require('./wiki_recovery_policy.cjs')
function triggerFixture(overrides = {}) {
const review = { tenant_id: 10059, dead_letter_id: 10, knowledge_id: 'doc', knowledge_base_id: 'kb', model_id: 'model',
previous_attempt: 2, previous_error_code: 'PAGE_WRITE_INCOMPLETE', max_attempt: 3, fix_release: 'public-20260919-061500',
budget_version: 1, api_image_id: `sha256:${'a'.repeat(64)}`, source_fingerprint: 'b'.repeat(64), model_fingerprint: 'c'.repeat(64),
embedding_model_id: 'embedding', max_provider_requests: 200, deadline_at: new Date(Date.now() + 3600000).toISOString() }
const manifest = { stage: 'published', release: review.fix_release, api_image_id: `sha256:${'a'.repeat(64)}`,
published_at: '2026-09-18T22:00:00Z', ...overrides.manifest }
const runtime = { running: true, release: review.fix_release, image_id: `sha256:${'a'.repeat(64)}`, ...overrides.runtime }
const source = { id: 10, tenant_id: 10059, knowledge_id: 'doc', knowledge_base_id: 'kb', model_id: 'model',
source_fingerprint: review.source_fingerprint, model_fingerprint: review.model_fingerprint, embedding_model_id: review.embedding_model_id,
eligible: true, chunks: 1, pending: 1, ...overrides.source }
const previous = { tenant_id: 10059, dead_letter_id: 10, knowledge_id: 'doc', knowledge_base_id: 'kb', model_id: 'model',
attempt_number: 2, pending: false, new_failures: 1, span_status: 'failed', error_code: 'PAGE_WRITE_INCOMPLETE', queued_at: '2026-09-18T21:00:00Z' }
const receipt = { ...previous, attempt_number: 3, pending_op_id: 99, pending: true, claimed_at: null, new_failures: 0,
queued_at: '2026-09-18T23:00:00Z', deployed_fix_review: review,
retry_reason: 'Deployed bounded complete-page repair with persistence regression tests', ...overrides.receipt }
receipt.recovery_budget = { ...receipt, source_fingerprint: review.source_fingerprint, model_fingerprint: review.model_fingerprint,
embedding_model_id: review.embedding_model_id, max_provider_requests: review.max_provider_requests, reserved_provider_requests: 0,
deadline_at: review.deadline_at, reviewed_release: review.fix_release, reviewed_api_image: review.api_image_id, status: 'active' }
let triggerCalls = 0
let manifestReads = 0
const errors = []
const context = {
__dirname,
process: { argv: ['node', 'recover_wiki_deadletters.cjs', '--trigger', '--ids=10'] },
console: { log() {}, error(message) { errors.push(message) } },
require(name) {
if (name === 'node:fs') return {
readFileSync(file) {
assert.equal(path.basename(file), 'public-release.json')
manifestReads++
return JSON.stringify(manifest)
},
mkdirSync() {}, writeFileSync() {},
}
if (name === 'node:child_process') return { execFileSync(command, args, options = {}) {
assert.equal(command, 'docker')
if (args.includes('psql')) {
assert.ok(args.includes('PGOPTIONS=-c default_transaction_read_only=on'), 'Trigger must not enqueue another operation')
if (options.input.includes('to_regclass')) return 't'
if (options.input.includes('FROM (SELECT ledger.*')) return JSON.stringify([previous, receipt])
return JSON.stringify([source])
}
if (args.includes('inspect')) return JSON.stringify(runtime)
if (args.includes('cp')) return ''
if (args.includes('exec') && args.includes('/tmp/wiki-backfill-trigger')) {
triggerCalls++
return JSON.stringify({ triggered_scopes: 1 })
}
throw new Error('Unexpected Docker operation')
} }
if (name === './wiki_recovery_policy.cjs') return policy
return require(name)
},
}
vm.runInNewContext(fs.readFileSync(path.join(__dirname, 'recover_wiki_deadletters.cjs'), 'utf8'), context)
return { triggerCalls, manifestReads, errors, exitCode: context.process.exitCode || 0 }
}
test('--trigger rechecks the persisted final review against the published image', () => {
assert.deepEqual(triggerFixture(), { triggerCalls: 1, manifestReads: 1, errors: [], exitCode: 0 })
for (const change of [
{ manifest: { release: 'public-20260918-000000' } },
{ manifest: { stage: 'origin-ready' } },
{ runtime: { release: 'public-20260918-000000' } },
{ runtime: { image_id: `sha256:${'b'.repeat(64)}` } },
{ runtime: { running: false } },
{ receipt: { deployed_fix_review: undefined } },
]) {
const result = triggerFixture(change)
assert.equal(result.triggerCalls, 0)
assert.equal(result.exitCode, 1)
assert.equal(result.errors.length, 1)
}
})
test('--trigger rejects source or receipt changes and mismatched reviewed attempts', () => {
for (const change of [
{ source: { tenant_id: 10060 } },
{ source: { knowledge_id: 'other' } },
{ source: { knowledge_base_id: 'other' } },
{ source: { model_id: 'other' } },
{ receipt: { knowledge_base_id: 'other' } },
{ receipt: { model_id: 'other' } },
{ receipt: { attempt_number: 4 } },
]) {
const result = triggerFixture(change)
assert.equal(result.triggerCalls, 0)
assert.equal(result.exitCode, 1)
assert.equal(result.errors.length, 1)
}
})
test('page recovery and window coverage never imply complete source reading without the new map proof', () => {
assert.equal(policy.describeReadCoverage({ recovery_status: 'wiki_recovered' }), 'not_verified')
assert.equal(policy.describeReadCoverage({ read_progress: { summary: { status: 'complete', source_windows_complete: true } } }), 'in_progress')
assert.equal(policy.describeReadCoverage({ read_progress: { candidates: { status: 'paused_budget' } } }), 'paused_budget')
assert.equal(policy.describeReadCoverage({ read_coverage_version: 'wiki-full-read-v1', source_read_complete: true }), 'source_processed')
assert.equal(policy.describeReadCoverage({ read_coverage_version: 'old-prefix', source_read_complete: true }), 'not_verified')
})
test('terminal reading failures cannot be reported as an active saved prefix', () => {
const read_progress = { candidates: { status: 'reading', completed_windows: 1, total_windows: 7 } }
assert.equal(policy.describeReadCoverage({ read_progress, span_status: 'failed' }), 'failed')
assert.equal(policy.describeReadCoverage({ read_progress, recovery_budget: { status: 'stopped_budget', stop_reason: 'phase_failed' } }), 'failed')
assert.equal(policy.describeReadCoverage({ read_progress, recovery_budget: { status: 'stopped_budget', stop_reason: 'deadline' } }), 'stopped_budget')
assert.equal(policy.describeReadCoverage({ read_progress, recovery_budget: { status: 'stopped_invalid' } }), 'stopped_invalid')
assert.equal(policy.describeReadCoverage({ read_progress, recovery_budget: { status: 'active' } }), 'in_progress')
assert.equal(policy.describeReadCoverage({ read_progress, span_status: 'failed', read_coverage_version: 'wiki-full-read-v1', source_read_complete: true }), 'source_processed')
assert.equal(read_progress.candidates.completed_windows, 1)
})
@@ -0,0 +1,23 @@
const fs = require('node:fs')
const path = require('node:path')
const assert = require('node:assert/strict')
const { execFileSync } = require('node:child_process')
const root = path.resolve(__dirname, '../..')
const release = process.argv[2]
assert.match(release || '', /^public-\d{8}-\d{6}$/)
const directory = path.join(root, 'releases', release)
assert.ok(fs.statSync(path.join(directory, 'yawan_native_acceptance_20260915.dump')).size > 0, 'Backup required')
const sql = statement => execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres', 'psql', '-X', '-q', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-At', '-v', 'ON_ERROR_STOP=1'], { input: statement, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim()
const predicate = "tenant_id BETWEEN 10000 AND 10058 AND NOT is_system_admin AND deleted_at IS NULL AND email ~ '^(acceptance-.+|etl-[0-9]+|milvus-[0-9]+)@example[.]test$'"
const accounts = JSON.parse(sql(`SELECT coalesce(json_agg(json_build_object('id',id,'tenant_id',tenant_id,'email',email,'is_active',is_active)), '[]'::json) FROM users WHERE ${predicate};`))
assert.equal(accounts.length, 59, 'Only the known disposable migration fixtures may be retired')
const beforePath = path.join(directory, 'retired-fixture-accounts.json')
assert.ok(!fs.existsSync(beforePath), 'Do not overwrite previous account state')
fs.writeFileSync(beforePath, JSON.stringify({ recorded_at: new Date().toISOString(), accounts }, null, 2), { mode: 0o600, flag: 'wx' })
sql(`BEGIN;
UPDATE auth_tokens SET is_revoked=true WHERE user_id IN (SELECT id FROM users WHERE ${predicate});
UPDATE tenant_api_keys SET revoked_at=now(),updated_at=now() WHERE tenant_id IN (SELECT tenant_id FROM users WHERE ${predicate}) AND revoked_at IS NULL;
UPDATE users SET is_active=false,updated_at=now() WHERE ${predicate};
COMMIT;`)
assert.equal(sql(`SELECT count(*) FROM users WHERE ${predicate} AND is_active;`), '0')
console.log(JSON.stringify({ retired_fixture_accounts: accounts.length, business_tenant_unchanged: 10059 }))
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env node
import { spawnSync } from 'node:child_process';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
const root = resolve(dirname(fileURLToPath(import.meta.url)), '..');
const storage = resolve(root, '..');
const context = process.env.GRAPH_DOCKER_CONTEXT || 'colima-yawan';
const container = process.env.GRAPH_APP_CONTAINER || 'yawan-native-acceptance';
const inspected = spawnSync('docker', ['--context', context, 'inspect', container], { encoding: 'utf8' });
if (inspected.status !== 0) throw new Error('Cannot inspect graph worker runtime');
const config = JSON.parse(inspected.stdout)[0];
const runtime = Object.fromEntries(config.Config.Env.map(value => {
const index = value.indexOf('=');
return [value.slice(0, index), value.slice(index + 1)];
}));
const selected = Object.fromEntries(Object.entries(runtime).filter(([key]) => /^(DB_|REDIS_)/.test(key) || key === 'LEGACY_GRAPH_ENABLE'));
const args = ['--context', context, 'run', '--rm', '--network', `container:${container}`, '-v', `${root}:/workspace:ro`, '-v', `${storage}/go-cache:/go`, '-v', `${storage}/go-build:/build-cache`, '-e', 'GOMODCACHE=/go/pkg/mod', '-e', 'GOCACHE=/build-cache', '-e', 'GRAPH_NATIVE_WRITE_VERIFIED'];
if (process.env.GRAPH_BACKFILL_REVISION) args.push('-e', 'GRAPH_BACKFILL_REVISION');
if (process.env.GRAPH_LEDGER_RECONCILE_VERIFIED) args.push('-e', 'GRAPH_LEDGER_RECONCILE_VERIFIED');
for (const key of Object.keys(selected)) args.push('-e', key);
args.push('-w', '/workspace', 'golang:1.26-bookworm', 'go', 'run', 'scripts/native_graph_backfill.go', ...process.argv.slice(2));
const result = spawnSync('docker', args, { env: { ...process.env, ...selected }, stdio: 'inherit' });
process.exitCode = result.status ?? 1;
+136
View File
@@ -0,0 +1,136 @@
const fs = require('node:fs')
const path = require('node:path')
const assert = require('node:assert/strict')
const { execFileSync, spawn } = require('node:child_process')
const { recoverySchedulingGate, remainingRecoveryCandidates } = require('./wiki_recovery_scheduler_policy.cjs')
const args = process.argv.slice(2)
const options = { 'max-minutes': 180, 'max-documents': 32, 'batch-size': 1 }
let reviewDirectory
for (const arg of args) {
if (arg === '--apply' || arg === '--detach') continue
if (arg.startsWith('--review-directory=')) {
reviewDirectory = path.resolve(arg.slice('--review-directory='.length))
continue
}
const match = /^--(max-minutes|max-documents|batch-size)=(\d+)$/.exec(arg)
assert.ok(match, 'Unknown recovery runner option')
options[match[1]] = Number(match[2])
}
assert.ok(args.includes('--apply'), 'Explicit --apply required')
assert.ok(options['max-minutes'] >= 1 && options['max-minutes'] <= 180, 'Time limit must be 1..180 minutes')
assert.ok(options['max-documents'] >= 1 && options['max-documents'] <= 32, 'Document limit must be 1..32')
assert.equal(options['batch-size'], 1, 'Recovery is limited to one document per batch')
const directory = path.resolve(__dirname, '../../acceptance/wiki-deadletter-recovery-20260919')
const recoveryScript = path.join(__dirname, 'recover_wiki_deadletters.cjs')
const stateFile = path.join(directory, 'runner-state.json')
const lockFile = path.join(directory, 'runner.lock')
fs.mkdirSync(directory, { recursive: true, mode: 0o700 })
if (args.includes('--detach')) {
const log = fs.openSync(path.join(directory, 'runner-console.log'), 'a', 0o600)
const child = spawn(process.execPath, [__filename, ...args.filter(arg => arg !== '--detach')], {
detached: true, stdio: ['ignore', log, log], cwd: path.dirname(recoveryScript),
})
child.unref()
fs.closeSync(log)
console.log(JSON.stringify({ launched_pid: child.pid, state_file: stateFile }))
} else {
main().catch(() => {
console.error('Recovery runner stopped. Inspect runner-state.json and private status reports.')
process.exitCode = 1
})
}
function summarize(report) {
const counts = {}
const readCoverage = {}
for (const row of report.recoveries) counts[row.recovery_status] = (counts[row.recovery_status] || 0) + 1
for (const row of report.recoveries) readCoverage[row.full_read_status || 'not_verified'] = (readCoverage[row.full_read_status || 'not_verified'] || 0) + 1
const { historical_full_read_unverified_ids, source_reading_complete } = recoverySchedulingGate(report)
return { attempted: report.attempted, remaining_not_attempted: report.remaining_not_attempted, counts,
full_read_counts: readCoverage,
historical_full_read_unverified_ids, source_reading_complete,
current_reading: report.recoveries.filter(row => row.pending).map(row => ({ dead_letter_id: row.dead_letter_id, read_progress: row.read_progress || {}, page_progress: row.page_progress || {} })),
page_associations: report.recoveries.reduce((sum, row) => sum + row.current_pages, 0),
new_failure_records: report.recoveries.reduce((sum, row) => sum + row.new_failures, 0),
report_file: report.report_file }
}
async function main() {
// An existing lock requires operator inspection, even if its process died.
// Never start a second scheduler on the basis of a possibly reused PID.
const lock = fs.openSync(lockFile, 'wx', 0o600)
fs.writeFileSync(lock, JSON.stringify({ pid: process.pid, started_at: new Date().toISOString() }))
fs.closeSync(lock)
const started = Date.now()
let queued = 0
let lastReport
let stopping = false
process.on('SIGTERM', () => { stopping = true })
process.on('SIGINT', () => { stopping = true })
const save = (state, extra = {}) => {
const value = { state, pid: process.pid, started_at: new Date(started).toISOString(),
updated_at: new Date().toISOString(), options, newly_enqueued: queued,
...(lastReport ? summarize(lastReport) : {}), ...extra }
const temporary = `${stateFile}.${process.pid}.tmp`
fs.writeFileSync(temporary, JSON.stringify(value, null, 2) + '\n', { mode: 0o600 })
fs.renameSync(temporary, stateFile)
fs.appendFileSync(path.join(directory, 'runner-events.jsonl'), JSON.stringify(value) + '\n', { mode: 0o600 })
console.log(JSON.stringify(value))
}
const inspect = (commandArgs = []) => {
const consoleReport = JSON.parse(execFileSync(process.execPath, [recoveryScript, ...commandArgs], {
encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 120000, maxBuffer: 4 * 1024 * 1024,
}))
assert.equal(path.dirname(consoleReport.report_file), directory, 'Unexpected report directory')
lastReport = { ...JSON.parse(fs.readFileSync(consoleReport.report_file, 'utf8')), report_file: consoleReport.report_file }
return lastReport
}
try {
save('starting')
while (true) {
const report = inspect()
const { state, ...coverage } = recoverySchedulingGate(report)
if (state === 'stopped_for_review') {
save(state, coverage)
break
}
if (stopping || Date.now() - started >= options['max-minutes'] * 60000) {
save(stopping ? 'stopped_by_signal' : 'time_limit_reached')
break
}
if (state === 'waiting_for_current_batch') {
save(state, coverage)
await new Promise(resolve => setTimeout(resolve, 45000))
continue
}
if (report.remaining_not_attempted === 0) {
save(state, coverage)
break
}
if (queued >= options['max-documents']) {
save('document_limit_reached')
break
}
const candidates = remainingRecoveryCandidates(report)
const ids = candidates.slice(0, Math.min(options['batch-size'], options['max-documents'] - queued)).map(row => row.id)
assert.ok(ids.length > 0, 'No eligible remaining originals')
const reviewFile = reviewDirectory && path.join(reviewDirectory, `dead-letter-${ids[0]}-attempt-1.json`)
if (!reviewFile || !fs.existsSync(reviewFile)) {
save('stopped_for_review', { batch_ids: ids, reason: 'A per-document reviewed request limit and fixed deadline are required before enqueueing.' })
break
}
save('enqueueing_batch', { batch_ids: ids })
inspect(['--apply', `--ids=${ids.join(',')}`, `--review-file=${reviewFile}`])
queued += ids.length
save('batch_enqueued', { batch_ids: ids })
}
} catch (error) {
save('stopped_on_error', { reason: error instanceof assert.AssertionError ? error.message : 'Recovery command failed; no automatic retry was scheduled.' })
throw error
} finally {
fs.unlinkSync(lockFile)
}
}
@@ -0,0 +1,43 @@
'use strict'
// Offline CLI. It consumes reviewed fixture files and never calls a provider.
// A real deployment adapter must reserve/attest its diagnostic lane before
// using the pure continuation module; this command is only for acceptance.
const fs = require('node:fs')
const path = require('node:path')
const { runDiagnosticContinuation } = require('./wiki_leaf_diagnostic_continuation.cjs')
function argument(name) {
const prefix = `--${name}=`
const value = process.argv.slice(2).find(arg => arg.startsWith(prefix))
return value?.slice(prefix.length)
}
const allowed = new Set(['--manifest=', '--source=', '--outcome='])
const unknown = process.argv.slice(2).filter(arg => ![...allowed].some(prefix => arg.startsWith(prefix)))
if (unknown.length || !argument('manifest') || !argument('source') || !argument('outcome')) {
console.error('usage: node scripts/run_wiki_leaf_diagnostic_continuation.cjs --manifest=FILE --source=FILE --outcome=FILE')
process.exitCode = 2
} else {
try {
const read = file => fs.readFileSync(path.resolve(file), 'utf8')
const manifest = JSON.parse(read(argument('manifest')))
const source = read(argument('source'))
const outcome = JSON.parse(read(argument('outcome')))
const capture = []
const result = runDiagnosticContinuation(manifest, source, () => outcome, artifact => {
// Keep the CLI output replay-safe. The real private capture sink owns
// the raw response; this fixture command emits hashes and byte counts.
capture.push({ status: artifact.status, idempotency_key: artifact.idempotency_key,
response_sha256: artifact.response_sha256, response_bytes: artifact.response_bytes })
})
const windows = result.windows.map(window => ({ key: window.key, status: window.status,
start_rune: window.start_rune, end_rune: window.end_rune, text_sha256: window.text_sha256 }))
process.stdout.write(JSON.stringify({ receipt: result.receipt, windows, capture }, null, 2) + '\n')
} catch (error) {
process.stderr.write(JSON.stringify({ passed: false, code: error.code || 'continuation_failed', error: error.message }) + '\n')
process.exitCode = 1
}
}
+36
View File
@@ -0,0 +1,36 @@
const fs = require('node:fs')
const path = require('node:path')
function copyEntry(source, destination) {
const stat = fs.lstatSync(source)
if (stat.isFile()) {
fs.copyFileSync(source, destination, fs.constants.COPYFILE_EXCL)
} else if (stat.isDirectory()) {
fs.mkdirSync(destination, { mode: stat.mode & 0o777 })
for (const name of fs.readdirSync(source)) copyEntry(path.join(source, name), path.join(destination, name))
} else {
throw new Error('API build context requires regular files and directories; symlinks are not supported')
}
}
// Keep the reviewed inputs outside the source tree; never reuse a partial context.
function stagePublicAPIContext(root, release, revision) {
if (!/^public-\d{8}-\d{6}$/.test(release || '')) throw new Error('Invalid API context release')
if (revision !== undefined && !/^\d{17}$/.test(revision)) throw new Error('Invalid API context revision')
const source = path.join(root, 'src')
const inputs = [
['scripts/Dockerfile.public-api', 'Dockerfile', 'file'],
['WeKnora-native-acceptance-next', 'WeKnora-native-acceptance-next', 'file'],
['migrations', 'migrations', 'directory'],
]
for (const [name, , type] of inputs) {
const stat = fs.lstatSync(path.join(source, name))
if (!(type === 'file' ? stat.isFile() : stat.isDirectory())) throw new Error(`Invalid API context input: ${name}`)
}
const directory = path.join(root, 'releases', release, revision === undefined ? 'api-context' : `api-context-${revision}`)
fs.mkdirSync(directory, { mode: 0o700 })
for (const [name, target] of inputs) copyEntry(path.join(source, name), path.join(directory, target))
return directory
}
module.exports = { stagePublicAPIContext }
+108
View File
@@ -0,0 +1,108 @@
const assert = require('node:assert/strict')
const test = require('node:test')
const fs = require('node:fs')
const os = require('node:os')
const path = require('node:path')
const { stagePublicAPIContext } = require('./stage_public_api_context.cjs')
const release = 'public-20260921-040700'
const revision = '20260921040700123'
function fixture(t) {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'public-api-context-'))
t.after(() => fs.rmSync(root, { recursive: true, force: true }))
const source = path.join(root, 'src')
fs.mkdirSync(path.join(source, 'scripts'), { recursive: true })
fs.mkdirSync(path.join(source, 'migrations/versioned'), { recursive: true })
fs.mkdirSync(path.join(source, 'migrations/empty'))
fs.mkdirSync(path.join(root, 'releases', release), { recursive: true })
const dockerfile = fs.readFileSync(path.join(__dirname, 'Dockerfile.public-api'))
const binary = Buffer.from([0x7f, 0x45, 0x4c, 0x46, 0x00, 0xff, 0x0a])
const migration = Buffer.from('BEGIN;\nSELECT 1;\nCOMMIT;\n')
fs.writeFileSync(path.join(source, 'scripts/Dockerfile.public-api'), dockerfile)
fs.writeFileSync(path.join(source, 'WeKnora-native-acceptance-next'), binary, { mode: 0o755 })
fs.writeFileSync(path.join(source, 'migrations/versioned/001.sql'), migration)
return { root, source, dockerfile, binary, migration }
}
function entries(directory) {
return fs.readdirSync(directory).sort().flatMap(name => {
const file = path.join(directory, name)
return fs.statSync(file).isDirectory() ? [name + '/', ...entries(file).map(child => name + '/' + child)] : [name]
})
}
test('stages only Dockerfile, executable bytes and the complete migration tree', t => {
const value = fixture(t)
fs.writeFileSync(path.join(value.source, '.env'), 'PRIVATE_TEST_SECRET=do-not-copy\n')
fs.writeFileSync(path.join(value.root, 'releases', release, 'runtime.env'), 'PRIVATE_RELEASE_SECRET=do-not-copy\n')
fs.mkdirSync(path.join(value.source, 'unrelated-cache'))
const unrelated = fs.openSync(path.join(value.source, 'unrelated-cache/large.bin'), 'wx')
try { fs.ftruncateSync(unrelated, 512 * 1024 * 1024) } finally { fs.closeSync(unrelated) }
const context = stagePublicAPIContext(value.root, release)
assert.equal(context, path.join(value.root, 'releases', release, 'api-context'))
assert.deepEqual(entries(context), ['Dockerfile', 'WeKnora-native-acceptance-next', 'migrations/',
'migrations/empty/', 'migrations/versioned/', 'migrations/versioned/001.sql'])
assert.deepEqual(fs.readFileSync(path.join(context, 'Dockerfile')), value.dockerfile)
assert.deepEqual(fs.readFileSync(path.join(context, 'WeKnora-native-acceptance-next')), value.binary)
assert.deepEqual(fs.readFileSync(path.join(context, 'migrations/versioned/001.sql')), value.migration)
assert.equal(fs.statSync(path.join(context, 'WeKnora-native-acceptance-next')).mode & 0o777, 0o755)
assert.equal(fs.statSync(context).mode & 0o777, 0o700)
})
test('contexts are immutable snapshots and refresh revisions use independent directories', t => {
const value = fixture(t)
const context = stagePublicAPIContext(value.root, release)
const changed = Buffer.from('new reviewed binary\n')
fs.writeFileSync(path.join(value.source, 'WeKnora-native-acceptance-next'), changed)
assert.throws(() => stagePublicAPIContext(value.root, release), { code: 'EEXIST' })
assert.deepEqual(fs.readFileSync(path.join(context, 'WeKnora-native-acceptance-next')), value.binary)
const refreshed = stagePublicAPIContext(value.root, release, revision)
assert.equal(refreshed, path.join(value.root, 'releases', release, `api-context-${revision}`))
assert.deepEqual(fs.readFileSync(path.join(refreshed, 'WeKnora-native-acceptance-next')), changed)
assert.throws(() => stagePublicAPIContext(value.root, release, revision), { code: 'EEXIST' })
})
test('never overwrites or cleans a previous partial context', t => {
const value = fixture(t)
const context = path.join(value.root, 'releases', release, 'api-context')
fs.mkdirSync(context)
fs.writeFileSync(path.join(context, 'partial-evidence'), 'retained')
assert.throws(() => stagePublicAPIContext(value.root, release), { code: 'EEXIST' })
assert.deepEqual(entries(context), ['partial-evidence'])
})
test('rejects invalid identifiers before creating a context', t => {
const value = fixture(t)
for (const candidate of ['', '../outside', 'public-20260921-040700/../../outside']) {
assert.throws(() => stagePublicAPIContext(value.root, candidate), /Invalid API context release/)
}
for (const candidate of ['', '../outside', '20260921', '20260921040700123/../../outside']) {
assert.throws(() => stagePublicAPIContext(value.root, release, candidate), /Invalid API context revision/)
}
assert.deepEqual(fs.readdirSync(path.join(value.root, 'releases', release)), [])
})
test('rejects symlinked binary and migration roots', t => {
for (const name of ['WeKnora-native-acceptance-next', 'migrations']) {
const value = fixture(t)
const input = path.join(value.source, name)
const target = path.join(value.root, 'outside')
fs.renameSync(input, target)
fs.symlinkSync(target, input)
assert.throws(() => stagePublicAPIContext(value.root, release), /Invalid API context input/)
assert.deepEqual(fs.readdirSync(path.join(value.root, 'releases', release)), [])
}
})
test('rejects nested migration symlinks without copying their external contents', t => {
const value = fixture(t)
const secret = path.join(value.root, 'private.env')
fs.writeFileSync(secret, 'PRIVATE_TEST_SECRET=do-not-copy\n')
fs.symlinkSync(secret, path.join(value.source, 'migrations/private.env'))
assert.throws(() => stagePublicAPIContext(value.root, release), /symlinks are not supported/)
const context = path.join(value.root, 'releases', release, 'api-context')
assert.equal(fs.existsSync(path.join(context, 'migrations/private.env')), false)
assert.throws(() => stagePublicAPIContext(value.root, release), { code: 'EEXIST' })
assert.equal(fs.readFileSync(secret, 'utf8'), 'PRIVATE_TEST_SECRET=do-not-copy\n')
})
+771
View File
@@ -0,0 +1,771 @@
#!/bin/bash
# 该脚本用于按需启动/停止Ollama和docker-compose服务
# 设置颜色
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m' # 无颜色
# 获取项目根目录(脚本所在目录的上一级)
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
# 版本信息
VERSION="1.0.1" # 版本更新
SCRIPT_NAME=$(basename "$0")
# 显示帮助信息
show_help() {
printf "%b\n" "${GREEN}WeKnora 启动脚本 v${VERSION}${NC}"
printf "%b\n" "${GREEN}用法:${NC} $0 [选项]"
echo "选项:"
echo " -h, --help 显示帮助信息"
echo " -o, --ollama 启动Ollama服务"
echo " -d, --docker 启动Docker容器服务"
echo " -a, --all 启动所有服务(默认)"
echo " -s, --stop 停止所有服务"
echo " -c, --check 检查环境并诊断问题"
echo " -r, --restart 重新构建并重启指定容器"
echo " -l, --list 列出所有正在运行的容器"
echo " -p, --pull 拉取最新的Docker镜像"
echo " --no-pull 启动时不拉取镜像(默认会拉取)"
echo " -v, --version 显示版本信息"
exit 0
}
# 显示版本信息
show_version() {
printf "%b\n" "${GREEN}WeKnora 启动脚本 v${VERSION}${NC}"
exit 0
}
# 日志函数
log_info() {
printf "%b\n" "${BLUE}[INFO]${NC} $1"
}
log_warning() {
printf "%b\n" "${YELLOW}[WARNING]${NC} $1"
}
log_error() {
printf "%b\n" "${RED}[ERROR]${NC} $1"
}
log_success() {
printf "%b\n" "${GREEN}[SUCCESS]${NC} $1"
}
# 选择可用的 Docker Compose 命令(优先 docker compose,其次 docker-compose)
DOCKER_COMPOSE_BIN=""
DOCKER_COMPOSE_SUBCMD=""
detect_compose_cmd() {
# 优先使用 Docker Compose 插件
if docker compose version &> /dev/null; then
DOCKER_COMPOSE_BIN="docker"
DOCKER_COMPOSE_SUBCMD="compose"
return 0
fi
# 回退到 docker-compose (v1)
if command -v docker-compose &> /dev/null; then
if docker-compose version &> /dev/null; then
DOCKER_COMPOSE_BIN="docker-compose"
DOCKER_COMPOSE_SUBCMD=""
return 0
fi
fi
# 都不可用
return 1
}
# 检查并创建.env文件
check_env_file() {
log_info "检查环境变量配置..."
if [ ! -f "$PROJECT_ROOT/.env" ]; then
log_warning ".env 文件不存在,将从模板创建"
if [ -f "$PROJECT_ROOT/.env.example" ]; then
cp "$PROJECT_ROOT/.env.example" "$PROJECT_ROOT/.env"
log_success "已从 .env.example 创建 .env 文件"
else
log_error "未找到 .env.example 模板文件,无法创建 .env 文件"
return 1
fi
else
log_info ".env 文件已存在"
fi
# 检查必要的环境变量是否已设置
source "$PROJECT_ROOT/.env"
local missing_vars=()
# 检查基础变量
if [ -z "$DB_DRIVER" ]; then missing_vars+=("DB_DRIVER"); fi
if [ -z "$STORAGE_TYPE" ]; then missing_vars+=("STORAGE_TYPE"); fi
return 0
}
# 安装Ollama(根据平台不同采用不同方法)
install_ollama() {
# 检查是否为远程服务
get_ollama_base_url
if [ $IS_REMOTE -eq 1 ]; then
log_info "检测到远程Ollama服务配置,无需在本地安装Ollama"
return 0
fi
log_info "本地Ollama未安装,正在安装..."
OS=$(uname)
if [ "$OS" = "Darwin" ]; then
# Mac安装方式
log_info "检测到Mac系统,使用brew安装Ollama..."
if ! command -v brew &> /dev/null; then
# 通过安装包安装
log_info "Homebrew未安装,使用直接下载方式..."
curl -fsSL https://ollama.com/download/Ollama-darwin.zip -o ollama.zip
unzip ollama.zip
mv ollama /usr/local/bin
rm ollama.zip
else
brew install ollama
fi
else
# Linux安装方式
log_info "检测到Linux系统,使用安装脚本..."
curl -fsSL https://ollama.com/install.sh | sh
fi
if [ $? -eq 0 ]; then
log_success "本地Ollama安装完成"
return 0
else
log_error "本地Ollama安装失败"
return 1
fi
}
# 获取Ollama基础URL,检查是否为远程服务
get_ollama_base_url() {
check_env_file
# 从环境变量获取Ollama基础URL
OLLAMA_URL=${OLLAMA_BASE_URL:-"http://host.docker.internal:11434"}
# 提取主机部分
OLLAMA_HOST=$(echo "$OLLAMA_URL" | sed -E 's|^https?://||' | sed -E 's|:[0-9]+$||' | sed -E 's|/.*$||')
# 提取端口部分
OLLAMA_PORT=$(echo "$OLLAMA_URL" | grep -oE ':[0-9]+' | grep -oE '[0-9]+' || echo "11434")
# 检查是否为localhost或127.0.0.1
IS_REMOTE=0
if [ "$OLLAMA_HOST" = "localhost" ] || [ "$OLLAMA_HOST" = "127.0.0.1" ] || [ "$OLLAMA_HOST" = "host.docker.internal" ]; then
IS_REMOTE=0 # 本地服务
else
IS_REMOTE=1 # 远程服务
fi
}
# 启动Ollama服务
start_ollama() {
log_info "正在检查Ollama服务..."
# 提取主机和端口
get_ollama_base_url
log_info "Ollama服务地址: $OLLAMA_URL"
if [ $IS_REMOTE -eq 1 ]; then
log_info "检测到远程Ollama服务,将直接使用远程服务,不进行本地安装和启动"
# 检查远程服务是否可用
if curl -s "$OLLAMA_URL/api/tags" &> /dev/null; then
log_success "远程Ollama服务可访问"
return 0
else
log_warning "远程Ollama服务不可访问,请确认服务地址正确且已启动"
return 1
fi
fi
# 以下为本地服务的处理
# 检查Ollama是否已安装
if ! command -v ollama &> /dev/null; then
install_ollama
if [ $? -ne 0 ]; then
return 1
fi
fi
# 检查Ollama服务是否已运行
if curl -s "http://localhost:$OLLAMA_PORT/api/tags" &> /dev/null; then
log_success "本地Ollama服务已经在运行,端口:$OLLAMA_PORT"
else
log_info "启动本地Ollama服务..."
# 注意:官方推荐使用 systemctl 或 launchctl 管理服务,直接后台运行仅用于临时场景
systemctl restart ollama || (ollama serve > /dev/null 2>&1 < /dev/null &)
# 等待服务启动
MAX_RETRIES=30
COUNT=0
while [ $COUNT -lt $MAX_RETRIES ]; do
if curl -s "http://localhost:$OLLAMA_PORT/api/tags" &> /dev/null; then
log_success "本地Ollama服务已成功启动,端口:$OLLAMA_PORT"
break
fi
echo -ne "等待Ollama服务启动... ($COUNT/$MAX_RETRIES)\r"
sleep 1
COUNT=$((COUNT + 1))
done
echo "" # 换行
if [ $COUNT -eq $MAX_RETRIES ]; then
log_error "本地Ollama服务启动失败"
return 1
fi
fi
log_success "本地Ollama服务地址: http://localhost:$OLLAMA_PORT"
return 0
}
# 停止Ollama服务
stop_ollama() {
log_info "正在停止Ollama服务..."
# 检查是否为远程服务
get_ollama_base_url
if [ $IS_REMOTE -eq 1 ]; then
log_info "检测到远程Ollama服务,无需在本地停止"
return 0
fi
# 检查Ollama是否已安装
if ! command -v ollama &> /dev/null; then
log_info "本地Ollama未安装,无需停止"
return 0
fi
# 查找并终止Ollama进程
if pgrep -x "ollama" > /dev/null; then
# 优先使用systemctl
if command -v systemctl &> /dev/null; then
sudo systemctl stop ollama
else
pkill -f "ollama serve"
fi
log_success "本地Ollama服务已停止"
else
log_info "本地Ollama服务未运行"
fi
return 0
}
# 检查Docker是否已安装
check_docker() {
log_info "检查Docker环境..."
if ! command -v docker &> /dev/null; then
log_error "未安装Docker,请先安装Docker"
return 1
fi
# 检查并选择可用的 Docker Compose 命令
if detect_compose_cmd; then
if [ "$DOCKER_COMPOSE_BIN" = "docker" ]; then
log_info "已检测到 Docker Compose 插件 (docker compose)"
else
log_info "已检测到 docker-compose (v1)"
fi
else
log_error "未检测到 Docker Compose(既没有 docker compose 也没有 docker-compose)。请安装其中之一。"
return 1
fi
# 检查Docker服务运行状态
if ! docker info &> /dev/null; then
log_error "Docker服务未运行,请启动Docker服务"
return 1
fi
log_success "Docker环境检查通过"
return 0
}
check_platform() {
# 检测当前系统平台
log_info "检测系统平台信息..."
if [ "$(uname -m)" = "x86_64" ]; then
export PLATFORM="linux/amd64"
elif [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then
export PLATFORM="linux/arm64"
else
log_warning "未识别的平台类型:$(uname -m),将使用默认平台 linux/amd64"
export PLATFORM="linux/amd64"
fi
log_info "当前平台:$PLATFORM"
}
# 预拉取沙箱镜像(Agent Skills 执行所需,仅拉取不启动)
ensure_sandbox_image() {
local sandbox_image="wechatopenai/weknora-sandbox:${WEKNORA_VERSION:-latest}"
# 检查本地是否已存在沙箱镜像
if docker image inspect "$sandbox_image" &> /dev/null; then
log_success "沙箱镜像已就绪: $sandbox_image"
return 0
fi
log_info "沙箱镜像 ($sandbox_image) 未检测到,正在后台拉取..."
log_info "Agent Skills 功能依赖此镜像,首次执行前需要拉取完成"
# 后台拉取,不阻塞主流程
(
if PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD --profile sandbox pull sandbox 2>/dev/null; then
log_success "沙箱镜像拉取完成: $sandbox_image"
else
log_warning "沙箱镜像拉取失败,Agent Skills 功能可能不可用"
log_warning "可稍后手动拉取: $DOCKER_COMPOSE_BIN $DOCKER_COMPOSE_SUBCMD --profile sandbox pull sandbox"
fi
) &
return 0
}
# 启动Docker容器
start_docker() {
log_info "正在启动Docker容器..."
# 检查Docker环境
check_docker
if [ $? -ne 0 ]; then
return 1
fi
# 检查.env文件
check_env_file
# 读取.env文件
source "$PROJECT_ROOT/.env"
storage_type=${STORAGE_TYPE:-local}
check_platform
# 进入项目根目录再执行docker-compose命令
cd "$PROJECT_ROOT"
# 启动基本服务
log_info "启动核心服务容器..."
# 统一通过已检测到的 Compose 命令启动
if [ "$NO_PULL" = true ]; then
# 不拉取镜像,使用本地镜像
log_info "跳过镜像拉取,使用本地镜像..."
PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD up --build -d
else
# 拉取最新镜像
log_info "拉取最新镜像..."
PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD up --pull always -d
fi
if [ $? -ne 0 ]; then
log_error "Docker容器启动失败"
return 1
fi
log_success "所有Docker容器已成功启动"
# 显示容器状态
log_info "当前容器状态:"
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD ps
# 预拉取Sandbox镜像(Agent Skills 执行所需,仅拉取不启动)
ensure_sandbox_image
return 0
}
# 停止Docker容器
stop_docker() {
log_info "正在停止Docker容器..."
# 检查Docker环境
check_docker
if [ $? -ne 0 ]; then
# 即使检查失败也尝试停止,以防万一
log_warning "Docker环境检查失败,仍将尝试停止容器..."
fi
# 进入项目根目录再执行docker-compose命令
cd "$PROJECT_ROOT"
# 停止所有容器
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD down --remove-orphans
if [ $? -ne 0 ]; then
log_error "Docker容器停止失败"
return 1
fi
log_success "所有Docker容器已停止"
return 0
}
# 列出所有正在运行的容器
list_containers() {
log_info "列出所有正在运行的容器..."
# 检查Docker环境
check_docker
if [ $? -ne 0 ]; then
return 1
fi
# 进入项目根目录再执行docker-compose命令
cd "$PROJECT_ROOT"
# 列出所有容器
printf "%b\n" "${BLUE}当前正在运行的容器:${NC}"
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD ps --services | sort
return 0
}
# 拉取最新的Docker镜像
pull_images() {
log_info "正在拉取最新的Docker镜像..."
# 检查Docker环境
check_docker
if [ $? -ne 0 ]; then
return 1
fi
# 检查.env文件
check_env_file
# 读取.env文件
source "$PROJECT_ROOT/.env"
storage_type=${STORAGE_TYPE:-local}
check_platform
# 进入项目根目录再执行docker-compose命令
cd "$PROJECT_ROOT"
# 拉取所有镜像
log_info "拉取所有服务的最新镜像..."
PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD pull
if [ $? -ne 0 ]; then
log_error "镜像拉取失败"
return 1
fi
# 拉取 sandbox 镜像(sandbox 在 profile 中,需要单独拉取)
log_info "拉取沙箱镜像..."
PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD --profile sandbox pull sandbox 2>/dev/null || \
log_warning "沙箱镜像拉取失败(非必需,跳过)"
log_success "所有镜像已成功拉取到最新版本"
# 显示拉取的镜像信息
log_info "已拉取的镜像:"
docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.CreatedAt}}\t{{.Size}}" | head -10
return 0
}
# 重启指定容器
restart_container() {
local container_name="$1"
if [ -z "$container_name" ]; then
log_error "未指定容器名称"
echo "可用的容器有:"
list_containers
return 1
fi
log_info "正在重新构建并重启容器: $container_name"
# 检查Docker环境
check_docker
if [ $? -ne 0 ]; then
return 1
fi
check_platform
# 进入项目根目录再执行docker-compose命令
cd "$PROJECT_ROOT"
# 检查容器是否存在
if ! "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD ps --services | grep -q "^$container_name$"; then
log_error "容器 '$container_name' 不存在或未运行"
echo "可用的容器有:"
list_containers
return 1
fi
# 构建并重启容器
log_info "正在重新构建容器 '$container_name'..."
PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD build "$container_name"
if [ $? -ne 0 ]; then
log_error "容器 '$container_name' 构建失败"
return 1
fi
log_info "正在重启容器 '$container_name'..."
PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD up -d --no-deps "$container_name"
if [ $? -ne 0 ]; then
log_error "容器 '$container_name' 重启失败"
return 1
fi
log_success "容器 '$container_name' 已成功重新构建并重启"
return 0
}
# 检查系统环境
check_environment() {
log_info "开始环境检查..."
# 检查操作系统
OS=$(uname)
log_info "操作系统: $OS"
# 检查Docker
check_docker
# 检查.env文件
check_env_file
get_ollama_base_url
if [ $IS_REMOTE -eq 1 ]; then
log_info "检测到远程Ollama服务配置"
if curl -s "$OLLAMA_URL/api/tags" &> /dev/null; then
version=$(curl -s "$OLLAMA_URL/api/tags" | grep -o '"version":"[^"]*"' | cut -d'"' -f4)
log_success "远程Ollama服务可访问,版本: $version"
else
log_warning "远程Ollama服务不可访问,请确认服务地址正确且已启动"
fi
else
if command -v ollama &> /dev/null; then
log_success "本地Ollama已安装"
if curl -s "http://localhost:$OLLAMA_PORT/api/tags" &> /dev/null; then
version=$(curl -s "http://localhost:$OLLAMA_PORT/api/tags" | grep -o '"version":"[^"]*"' | cut -d'"' -f4)
log_success "本地Ollama服务正在运行,版本: $version"
else
log_warning "本地Ollama已安装但服务未运行"
fi
else
log_warning "本地Ollama未安装"
fi
fi
# 检查沙箱镜像
log_info "检查沙箱镜像..."
local sandbox_image="wechatopenai/weknora-sandbox:${WEKNORA_VERSION:-latest}"
if docker image inspect "$sandbox_image" &> /dev/null; then
log_success "沙箱镜像已就绪: $sandbox_image"
else
log_warning "沙箱镜像未找到: $sandbox_image (Agent Skills 功能需要此镜像)"
log_info "可通过以下命令拉取: $0 -p 或 docker pull $sandbox_image"
fi
# 检查磁盘空间
log_info "检查磁盘空间..."
df -h | grep -E "(Filesystem|/$)"
# 检查内存
log_info "检查内存使用情况..."
if [ "$OS" = "Darwin" ]; then
vm_stat | perl -ne '/page size of (\d+)/ and $size=$1; /Pages free:\s*(\d+)/ and print "Free Memory: ", $1 * $size / 1048576, " MB\n"'
else
free -h | grep -E "(total|Mem:)"
fi
# 检查CPU
log_info "CPU信息:"
if [ "$OS" = "Darwin" ]; then
sysctl -n machdep.cpu.brand_string
echo "CPU核心数: $(sysctl -n hw.ncpu)"
else
grep "model name" /proc/cpuinfo | head -1
echo "CPU核心数: $(nproc)"
fi
# 检查容器状态
log_info "检查容器状态..."
if docker info &> /dev/null; then
docker ps -a
else
log_warning "无法获取容器状态,Docker可能未运行"
fi
log_success "环境检查完成"
return 0
}
# 解析命令行参数
START_OLLAMA=false
START_DOCKER=false
STOP_SERVICES=false
CHECK_ENVIRONMENT=false
LIST_CONTAINERS=false
RESTART_CONTAINER=false
PULL_IMAGES=false
NO_PULL=false
CONTAINER_NAME=""
# 没有参数时默认启动所有服务
if [ $# -eq 0 ]; then
START_OLLAMA=true
START_DOCKER=true
fi
while [ "$1" != "" ]; do
case $1 in
-h | --help ) show_help
;;
-o | --ollama ) START_OLLAMA=true
;;
-d | --docker ) START_DOCKER=true
;;
-a | --all ) START_OLLAMA=true
START_DOCKER=true
;;
-s | --stop ) STOP_SERVICES=true
;;
-c | --check ) CHECK_ENVIRONMENT=true
;;
-l | --list ) LIST_CONTAINERS=true
;;
-p | --pull ) PULL_IMAGES=true
;;
--no-pull ) NO_PULL=true
START_OLLAMA=true
START_DOCKER=true
;;
-r | --restart ) RESTART_CONTAINER=true
CONTAINER_NAME="$2"
shift
;;
-v | --version ) show_version
;;
* ) log_error "未知选项: $1"
show_help
;;
esac
shift
done
# 执行环境检查
if [ "$CHECK_ENVIRONMENT" = true ]; then
check_environment
exit $?
fi
# 列出所有容器
if [ "$LIST_CONTAINERS" = true ]; then
list_containers
exit $?
fi
# 拉取最新镜像
if [ "$PULL_IMAGES" = true ]; then
pull_images
exit $?
fi
# 重启指定容器
if [ "$RESTART_CONTAINER" = true ]; then
restart_container "$CONTAINER_NAME"
exit $?
fi
# 执行服务操作
if [ "$STOP_SERVICES" = true ]; then
# 停止服务
stop_ollama
OLLAMA_RESULT=$?
stop_docker
DOCKER_RESULT=$?
# 显示总结
echo ""
log_info "=== 停止结果 ==="
if [ $OLLAMA_RESULT -eq 0 ]; then
log_success "✓ Ollama服务已停止"
else
log_error "✗ Ollama服务停止失败"
fi
if [ $DOCKER_RESULT -eq 0 ]; then
log_success "✓ Docker容器已停止"
else
log_error "✗ Docker容器停止失败"
fi
log_success "服务停止完成。"
else
# 启动服务
OLLAMA_RESULT=1
DOCKER_RESULT=1
if [ "$START_OLLAMA" = true ]; then
start_ollama
OLLAMA_RESULT=$?
fi
if [ "$START_DOCKER" = true ]; then
start_docker
DOCKER_RESULT=$?
fi
# 显示总结
echo ""
log_info "=== 启动结果 ==="
if [ "$START_OLLAMA" = true ]; then
if [ $OLLAMA_RESULT -eq 0 ]; then
log_success "✓ Ollama服务已启动"
else
log_error "✗ Ollama服务启动失败"
fi
fi
if [ "$START_DOCKER" = true ]; then
if [ $DOCKER_RESULT -eq 0 ]; then
log_success "✓ Docker容器已启动"
else
log_error "✗ Docker容器启动失败"
fi
fi
if [ "$START_OLLAMA" = true ] && [ "$START_DOCKER" = true ]; then
if [ $OLLAMA_RESULT -eq 0 ] && [ $DOCKER_RESULT -eq 0 ]; then
log_success "所有服务启动完成,可通过以下地址访问:"
printf "%b\n" "${GREEN} - 前端界面: http://localhost:${FRONTEND_PORT:-80}${NC}"
printf "%b\n" "${GREEN} - API接口: http://localhost:${APP_PORT:-8080}${NC}"
echo ""
log_info "正在持续输出容器日志(按 Ctrl+C 退出日志,容器不会停止)..."
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD logs app docreader postgres --since=10s -f
else
log_error "部分服务启动失败,请检查日志并修复问题"
fi
elif [ "$START_OLLAMA" = true ] && [ $OLLAMA_RESULT -eq 0 ]; then
log_success "Ollama服务启动完成,可通过以下地址访问:"
printf "%b\n" "${GREEN} - Ollama API: http://localhost:$OLLAMA_PORT${NC}"
elif [ "$START_DOCKER" = true ] && [ $DOCKER_RESULT -eq 0 ]; then
log_success "Docker容器启动完成,可通过以下地址访问:"
printf "%b\n" "${GREEN} - 前端界面: http://localhost:${FRONTEND_PORT:-80}${NC}"
printf "%b\n" "${GREEN} - API接口: http://localhost:${APP_PORT:-8080}${NC}"
echo ""
log_info "正在持续输出容器日志(按 Ctrl+C 退出日志,容器不会停止)..."
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD logs app docreader postgres --since=10s -f
fi
fi
exit 0
+178
View File
@@ -0,0 +1,178 @@
#!/usr/bin/env bash
set -euo pipefail
#
# 本地测试 Homebrew Formula
#
# 流程:打包 → 创建本地 tap → 写入 Formula → brew install → 验证
#
# 用法:
# ./scripts/test-homebrew.sh # 完整测试(含前端构建)
# SKIP_FRONTEND=1 ./scripts/test-homebrew.sh # 跳过前端构建
# SKIP_BUILD=1 ./scripts/test-homebrew.sh # 跳过构建(使用已有 tarball)
#
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
cd "${ROOT_DIR}"
TAP_NAME="weknora/test"
FORMULA_NAME="weknora-lite-test"
VERSION="test"
GOOS=$(go env GOOS)
GOARCH=$(go env GOARCH)
ARCHIVE="WeKnora-lite_${VERSION}_${GOOS}_${GOARCH}"
TARBALL="${ROOT_DIR}/dist/${ARCHIVE}.tar.gz"
# ── Step 1: Build package ──
if [ "${SKIP_BUILD:-}" != "1" ]; then
echo "=== Step 1: Build local package ==="
SKIP_FRONTEND="${SKIP_FRONTEND:-0}" ./scripts/package-lite.sh "${VERSION}"
else
echo "=== Step 1: Skipped (SKIP_BUILD=1) ==="
fi
if [ ! -f "${TARBALL}" ]; then
echo "Error: tarball not found at ${TARBALL}"
exit 1
fi
SHA=$(shasum -a 256 "${TARBALL}" | awk '{print $1}')
echo ""
echo " Tarball: ${TARBALL}"
echo " SHA256: ${SHA}"
# ── Step 2: Create local tap ──
echo ""
echo "=== Step 2: Set up local tap ==="
# Create tap if it doesn't exist
TAP_DIR="$(brew --repository)/Library/Taps/weknora/homebrew-test"
if [ ! -d "${TAP_DIR}" ]; then
mkdir -p "${TAP_DIR}/Formula"
(cd "${TAP_DIR}" && git init -q && git commit --allow-empty -m "init" -q)
echo " Created tap: ${TAP_NAME}"
else
echo " Tap already exists: ${TAP_NAME}"
fi
# ── Step 3: Write Formula into tap ──
echo ""
echo "=== Step 3: Generate Formula ==="
cat > "${TAP_DIR}/Formula/${FORMULA_NAME}.rb" << RUBY
class WeknoraLiteTest < Formula
desc "WeKnora Lite (local test)"
homepage "https://github.com/Tencent/WeKnora"
version "${VERSION}"
license "Apache-2.0"
url "file://${TARBALL}"
sha256 "${SHA}"
def install
libexec.install "WeKnora-lite"
pkgshare.install "web" if File.directory?("web")
pkgshare.install "config" if File.directory?("config")
pkgshare.install ".env.lite.example"
if File.directory?("migrations")
pkgshare.install "migrations"
end
(bin/"weknora-lite-test").write <<~SH
#!/bin/bash
CONFIG_DIR="\\\${WEKNORA_CONFIG_DIR:-\\\${XDG_CONFIG_HOME:-\\\$HOME/.config}/weknora-test}"
DATA_DIR="\\\${WEKNORA_DATA_DIR:-\\\${XDG_DATA_HOME:-\\\$HOME/.local/share}/weknora-test}"
mkdir -p "\\\$DATA_DIR/files" "\\\$CONFIG_DIR/config" 2>/dev/null
if [ ! -f "\\\$CONFIG_DIR/config/config.yaml" ]; then
cp -r "#{pkgshare}/config/" "\\\$CONFIG_DIR/config/"
fi
if [ ! -d "\\\$CONFIG_DIR/migrations" ] && [ -d "#{pkgshare}/migrations" ]; then
ln -sf "#{pkgshare}/migrations" "\\\$CONFIG_DIR/migrations"
fi
if [ ! -f "\\\$CONFIG_DIR/.env.lite" ]; then
cp "#{pkgshare}/.env.lite.example" "\\\$CONFIG_DIR/.env.lite"
sed -i'' -e "s|DB_PATH=.*|DB_PATH=\\\$DATA_DIR/weknora.db|" "\\\$CONFIG_DIR/.env.lite"
sed -i'' -e "s|LOCAL_STORAGE_BASE_DIR=.*|LOCAL_STORAGE_BASE_DIR=\\\$DATA_DIR/files|" "\\\$CONFIG_DIR/.env.lite"
echo ""
echo "已创建配置文件: \\\$CONFIG_DIR/.env.lite"
echo ""
fi
set -a
source "\\\$CONFIG_DIR/.env.lite"
set +a
export DB_PATH="\\\${DB_PATH:-\\\$DATA_DIR/weknora.db}"
export LOCAL_STORAGE_BASE_DIR="\\\${LOCAL_STORAGE_BASE_DIR:-\\\$DATA_DIR/files}"
export WEKNORA_WEB_DIR="\\\${WEKNORA_WEB_DIR:-#{pkgshare}/web}"
cd "\\\$CONFIG_DIR"
exec "#{libexec}/WeKnora-lite" "\\\$@"
SH
end
def post_install
(var/"weknora-test").mkpath
(var/"log").mkpath
end
service do
run [bin/"weknora-lite-test"]
keep_alive true
working_dir var/"weknora-test"
log_path var/"log/weknora-lite-test.log"
error_log_path var/"log/weknora-lite-test.log"
end
test do
assert_predicate bin/"weknora-lite-test", :executable?
end
end
RUBY
(cd "${TAP_DIR}" && git add -A && git commit -m "update formula" -q --allow-empty)
echo " Formula written to: ${TAP_DIR}/Formula/${FORMULA_NAME}.rb"
# ── Step 4: Install ──
echo ""
echo "=== Step 4: Install ==="
brew reinstall "${TAP_NAME}/${FORMULA_NAME}" 2>&1 || brew install "${TAP_NAME}/${FORMULA_NAME}"
# ── Step 5: Verify ──
echo ""
echo "=== Step 5: Verify ==="
echo ""
echo " which:"
which weknora-lite-test || true
echo ""
echo " Installed files:"
brew list "${TAP_NAME}/${FORMULA_NAME}"
echo ""
echo " Test paths (isolated from production):"
echo " Config: ~/.config/weknora-test/.env.lite"
echo " Data: ~/.local/share/weknora-test/"
echo ""
echo "=== Done ==="
echo ""
echo "前台运行:"
echo " weknora-lite-test"
echo ""
echo "后台服务:"
echo " brew services start ${TAP_NAME}/${FORMULA_NAME}"
echo " brew services info ${TAP_NAME}/${FORMULA_NAME}"
echo " brew services stop ${TAP_NAME}/${FORMULA_NAME}"
echo ""
echo "日志:"
echo " $(brew --prefix)/var/log/weknora-lite-test.log"
echo ""
echo "卸载测试:"
echo " brew services stop ${FORMULA_NAME} 2>/dev/null"
echo " brew uninstall ${FORMULA_NAME}"
echo " brew untap ${TAP_NAME}"
echo " rm -rf ~/.config/weknora-test ~/.local/share/weknora-test"
+129
View File
@@ -0,0 +1,129 @@
import copy
import sqlite3
import unittest
from unittest.mock import patch
from weknora_identity_etl import ADMIN_TARGET_ID
from weknora_operations_etl import (KINDS, notification_owner, operations_plan, operations_sql,
reconcile, redact, validate_plan)
USER = "63ce184e-5ab6-5240-8269-14026d501a68"
AGENT = "da4d5f8f-e1c4-574e-b34a-8a324615b603"
KB = "237fa6a0-1e49-58c8-8185-4191d14a9094"
NOW = "2026-09-15T12:00:00"
def fixture():
rows = {kind: [] for kind in (*KINDS, "campaign", "segment")}
rows["pointaccount"] = [{"id": 1, "user_id": 1, "balance_points": 110, "updated_at": NOW}]
rows["pointtransaction"] = [{"id": 1, "user_id": 1, "delta_points": 100, "kind": "credit", "reason": "grant",
"operator_user_id": 1, "metadata": {}, "created_at": NOW},
{"id": 2, "user_id": 1, "delta_points": -10, "kind": "debit", "reason": "generation",
"operator_user_id": 1, "metadata": {"usage_event_id": 1}, "created_at": NOW}]
rows["usageevent"] = [{"id": 1, "user_id": 1, "agent_id": 2, "provider_id": 1, "model": "test",
"unit": "token", "quantity": 1, "currency": "USD", "amount_minor": 5000,
"llm_call_id": 1, "campaign_id": 1, "run_id": 1, "segment_id": 1,
"input_quantity": 1, "output_quantity": 0, "service_type": "llm",
"status": "measured", "metadata": {}, "created_at": NOW}]
rows["campaign"] = [{"id": 1, "owner_user_id": 1}, {"id": 2, "owner_user_id": 8}]
rows["segment"] = [{"id": 1, "campaign_id": 1}, {"id": 2, "campaign_id": 1}, {"id": 42, "campaign_id": 2}]
return rows
def plan(rows=None):
rows = rows or fixture()
db = sqlite3.connect(":memory:")
db.execute("PRAGMA query_only=ON")
try:
with patch("weknora_operations_etl._read", side_effect=lambda _db, kind: copy.deepcopy(rows[kind])):
result = operations_plan(db, {"1": ADMIN_TARGET_ID, "8": USER}, {"2": AGENT}, {"3": KB}, {}, {})
finally:
db.close()
result["source_snapshot_sha256"] = "a" * 64
return result
class OperationsTests(unittest.TestCase):
def test_mismatch_preserves_authoritative_balance_without_credit(self):
result = plan()
self.assertEqual(result["reconciliation"][0]["difference"], 20)
native = [i for i in result["items"] if i["target_table"] == "workbench_balances"]
self.assertEqual(native[0]["native_snapshot"]["balance"], 110)
self.assertEqual(native[0]["history_payload"]["balance_reconciliation"]["difference"], 20)
self.assertEqual(len([i for i in result["items"] if i["target_table"] == "workbench_transactions"]), 2)
validate_plan(result)
def test_missing_account_does_not_create_credit_or_balance(self):
rows = fixture()
rows["pointaccount"] = []
result = plan(rows)
self.assertEqual(result["reconciliation"][0]["reason"], "missing_source_account")
self.assertFalse(any(i["target_table"] == "workbench_balances" for i in result["items"]))
self.assertEqual(sum(i["target_table"] == "workbench_transactions" for i in result["items"]), 2)
def test_points_use_actual_debits_not_currency_or_token_estimates(self):
usage = next(i for i in plan()["items"] if i["source_kind"] == "usageevent")
self.assertEqual(usage["native_snapshot"]["points"], 10)
self.assertEqual(usage["native_snapshot"]["metadata"]["legacy_amount_minor"], 5000)
def test_storage_quantity_is_not_tokens(self):
rows = fixture()
rows["usageevent"][0].update(unit="byte", input_quantity=999, output_quantity=999)
usage = next(i["native_snapshot"] for i in plan(rows)["items"] if i["source_kind"] == "usageevent")
self.assertEqual((usage["input_tokens"], usage["output_tokens"]), (0, 0))
def test_usage_ownerless_is_admin_archive_only(self):
rows = fixture()
rows["usageevent"][0]["user_id"] = 0
usage = next(i for i in plan(rows)["items"] if i["source_kind"] == "usageevent")
self.assertEqual(usage["target_table"], "")
self.assertEqual(usage["owner_id"], "")
def test_ambiguous_notifications_not_assigned_by_numeric_reference(self):
ids = {"1": ADMIN_TARGET_ID, "8": USER}
owner, reason = notification_owner({"type": "failure", "ref_id": 2}, {1: 1, 2: 8}, {2: 1}, ids)
self.assertEqual(owner, "")
self.assertTrue(reason)
owner, reason = notification_owner({"type": "failure", "ref_id": 42}, {1: 1, 2: 8}, {42: 2}, ids)
self.assertEqual(owner, USER)
def test_alerts_are_never_broadcast(self):
owner, _ = notification_owner({"type": "alert", "ref_id": 0}, {}, {}, {"1": ADMIN_TARGET_ID})
self.assertEqual(owner, "")
def test_redacted_native_log_preserves_original_in_private_plan(self):
rows = fixture()
secret = "sk-123456789abcdefghijklmnop"
rows["systemlog"] = [{"id": 1, "level": "WARNING", "source": "provider", "message": "api_key=" + secret,
"trace_id": "", "status": 500, "created_at": NOW}]
item = next(i for i in plan(rows)["items"] if i["source_kind"] == "systemlog")
self.assertIn(secret, item["source_snapshot"]["message"])
self.assertNotIn(secret, item["native_snapshot"]["message"])
self.assertEqual(item["native_snapshot"]["level"], "warning")
self.assertEqual(redact({"api_key": secret})["api_key"], "[REDACTED]")
def test_native_balance_tampering_rejected(self):
result = plan()
result["items"][0]["native_snapshot"]["balance"] += 20
with self.assertRaisesRegex(ValueError, "balance"):
validate_plan(result)
def test_native_owner_tampering_rejected(self):
result = plan()
result["items"][0]["native_snapshot"]["user_id"] = USER
with self.assertRaisesRegex(ValueError, "owner"):
validate_plan(result)
def test_duplicate_accounts_rejected(self):
with self.assertRaisesRegex(ValueError, "Duplicate"):
reconcile([{"user_id": 1, "balance_points": 0}] * 2, [])
def test_sql_requires_snapshot_provenance(self):
result = plan()
result.pop("source_snapshot_sha256")
with self.assertRaisesRegex(ValueError, "fingerprint"):
operations_sql(result)
if __name__ == "__main__":
unittest.main()
+150
View File
@@ -0,0 +1,150 @@
import unittest
from weknora_access_etl import (ACCESS_TABLES, ADMIN_TARGET_ID, digest, make_plan, migration_sql,
role_permissions, stable, TARGET_DATABASE, TARGET_TENANT)
from apply_weknora_access import rollback_sql
class AccessMigrationTests(unittest.TestCase):
def test_editor_covers_grouped_operations_only(self):
permissions, _ = role_permissions({"permissions": [
"read", "feature:tasks", "tasks:read", "tasks:write", "tasks:review", "tasks:deliver",
"feature:agents", "agents:read", "agents:write", "memories:read", "memories:write", "memories:review",
"feature:flows", "flows:read", "feature:knowledge", "kb:read", "kb:write",
"feature:digital-humans", "digital_humans:read", "logs:read", "usage:read"]})
expected = [module + "." + action for module in
("content", "flows", "agents", "knowledge", "digital-humans") for action in ("read", "write")]
self.assertEqual(permissions, sorted(expected + ["content.review", "content.deliver"]))
def test_generic_read_does_not_grant_any_module(self):
self.assertEqual(role_permissions({"permissions": ["read"]})[0], [])
def test_legacy_reviewer_cannot_gain_export_or_agent_management(self):
permissions, notes = role_permissions({"permissions": ["audit:read", "read", "review"]})
self.assertEqual(permissions, ["content.read", "content.review"])
self.assertNotIn("content_read_withheld_because_native_read_also_exposes_export", notes)
def test_content_delivery_does_not_grant_generation(self):
self.assertEqual(role_permissions({"permissions": ["tasks:review", "tasks:deliver"]})[0],
["content.deliver", "content.read", "content.review"])
self.assertEqual(role_permissions({"permissions": ["tasks:review", "tasks:write"]})[0],
["content.read", "content.review", "content.write"])
self.assertEqual(role_permissions({"permissions": ["tasks:read", "tasks:deliver"]})[0], [])
def test_agent_write_does_not_infer_memory_review(self):
permissions, _ = role_permissions({"permissions": ["agents:read", "agents:write", "memories:write"]})
self.assertIn("agents.read", permissions)
self.assertNotIn("agents.write", permissions)
def test_memory_review_does_not_infer_agent_write(self):
self.assertEqual(role_permissions({"permissions": ["feature:memories", "memories:review"]})[0], [])
def test_old_config_management_does_not_grant_all_settings_or_access(self):
permissions, _ = role_permissions({"permissions": ["config:write", "admin:manage", "users:manage"]})
self.assertNotIn("settings.write", permissions)
self.assertNotIn("access.write", permissions)
def test_source_admin_does_not_create_global_admin(self):
self.assertEqual(role_permissions({"permissions": ["*"]})[0], [])
def test_stable_role_identity(self):
self.assertEqual(stable("editor"), stable("editor"))
self.assertNotEqual(stable("editor"), stable("viewer"))
def plan(self):
value = {"target_database": TARGET_DATABASE, "tenant_id": TARGET_TENANT,
"grants": [], "roles": [], "assignments": [], "publications": [], "source_role_definitions": []}
value["plan_sha256"] = digest(value)
return value
def test_sql_defaults_to_rollback_and_does_not_update_or_delete_native_rows(self):
sql = migration_sql(self.plan())
self.assertTrue(sql.rstrip().endswith("ROLLBACK;"))
self.assertNotIn("UPDATE users", sql)
self.assertNotIn("UPDATE tenant_members", sql)
self.assertNotIn("DELETE FROM", sql)
self.assertIn("m.status='suspended'", sql)
self.assertIn("Native resource authorization already exists", sql)
self.assertIn("change->'legacy_source_spec'=saved", sql)
self.assertIn("change->'mapped_permissions'=spec->'permissions'", sql)
def test_tampered_or_wrong_target_plan_is_rejected(self):
plan = self.plan()
plan["tenant_id"] = 2
with self.assertRaises(ValueError):
migration_sql(plan)
plan = self.plan()
plan["roles"] = [{"permissions": ["access.write"]}]
with self.assertRaises(ValueError):
migration_sql(plan)
def test_explicit_grants_are_not_inferred(self):
plan = self.plan()
plan["grants"] = [{"kind": "agents"}]
plan["plan_sha256"] = digest({k: v for k, v in plan.items() if k != "plan_sha256"})
with self.assertRaises(ValueError):
migration_sql(plan)
def source_and_target(self):
role = {"role": "viewer", "organization_id": 1, "label": "Viewer", "permissions": ["read"]}
source = {"roles": [role], "users": [{"id": 1, "enabled": 1},
{"id": 3, "enabled": 1, "organization_id": 1, "role": "viewer"}],
"grants": {}, "memory_count": 0,
"resources": {key: [] for key in ("agent", "agentskill", "canvas", "knowledgebase", "digitalhuman")}}
identity = {"source_user_id": 3, "target_user_id": "user3", "user_tenant": TARGET_TENANT,
"is_active": False, "is_system_admin": False, "can_access_all_tenants": False, "user_deleted": None,
"source_snapshot": {"role": "viewer"}, "memberships": [
{"tenant_id": TARGET_TENANT, "role": "contributor", "status": "suspended", "deleted_at": None}]}
target = {"database": TARGET_DATABASE, "organization": {"source_role_definitions": [role]},
"identities": [{"source_user_id": 1, "target_user_id": ADMIN_TARGET_ID}, identity],
"resources": [], "asset_mappings": [], "flow_mappings": [], "access_tables": list(ACCESS_TABLES)}
return source, target
def build(self, source, target):
return make_plan(source, target, {"tenant_id": TARGET_TENANT}, "snapshot")
def test_explicit_deny_all_assignment_is_created_for_viewer(self):
plan = self.build(*self.source_and_target())
self.assertEqual(len(plan["assignments"]), 1)
self.assertEqual(plan["roles"][0]["permissions"], [])
self.assertEqual(plan["assignments"][0]["role_id"], stable("viewer"))
def test_active_migrated_account_is_rejected(self):
source, target = self.source_and_target()
target["identities"][1]["is_active"] = True
with self.assertRaisesRegex(ValueError, "inactive and suspended"):
self.build(source, target)
def test_cross_tenant_membership_is_rejected(self):
source, target = self.source_and_target()
target["identities"][1]["memberships"].append(
{"tenant_id": 9, "role": "viewer", "status": "active", "deleted_at": None})
with self.assertRaisesRegex(ValueError, "inactive and suspended"):
self.build(source, target)
def test_missing_enabled_identity_is_rejected(self):
source, target = self.source_and_target()
target["identities"].pop()
with self.assertRaisesRegex(ValueError, "lacks a durable identity"):
self.build(source, target)
def test_source_permission_drift_is_rejected(self):
source, target = self.source_and_target()
target["organization"]["source_role_definitions"] = []
with self.assertRaisesRegex(ValueError, "role definitions differ"):
self.build(source, target)
def test_rollback_is_scoped_guarded_and_rehearsal_by_default(self):
from weknora_access_etl import LEDGER
state = {table: [] for table in (*ACCESS_TABLES, LEDGER)}
sql = rollback_sql(self.plan(), state, state)
self.assertTrue(sql.rstrip().endswith("ROLLBACK;"))
self.assertIn("Native edits or missing rows", sql)
self.assertIn("New references to migrated access rows", sql)
self.assertIn("Account activation changed", sql)
self.assertNotIn("DELETE FROM users", sql)
self.assertNotIn("DELETE FROM tenant_members", sql)
if __name__ == "__main__":
unittest.main()
+318
View File
@@ -0,0 +1,318 @@
"""Read-only asset plans, lossless text and native SQL migration contracts."""
import copy
import json
import sqlite3
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
from weknora_asset_etl import (AGENT_COLUMNS, SKILL_COLUMNS, TEMPLATE_COLUMNS, ADMIN_TARGET_ID,
LEDGER, BINDING_LEDGER, asset_id, asset_plan, asset_sql, safe_asset_summary)
from weknora_model_etl import model_plan, model_id
USER_ID = "ab043a7d-d10e-58fe-8417-3fd2cbab5f77"
KB_ID = "7e6ac485-f766-4d58-ae38-2f0456a022ef"
class AssetTests(unittest.TestCase):
def setUp(self):
self.db = sqlite3.connect(":memory:")
self.addCleanup(self.db.close)
integer_columns = {"id", "model_provider_id", "model_catalog_id", "prompt_template_id", "rules_version",
"builtin", "organization_id", "owner_user_id", "digital_human_id", "enabled", "version"}
for table, columns in (("agent", AGENT_COLUMNS), ("agentskill", SKILL_COLUMNS), ("agentprompttemplate", TEMPLATE_COLUMNS)):
self.db.execute("CREATE TABLE " + table + " (" + ",".join(
column + (" INTEGER" if column in integer_columns else " TEXT") for column in columns) + ")")
self.db.executescript("""
CREATE TABLE agentskillbinding (id INTEGER,agent_id INTEGER,skill_id INTEGER);
CREATE TABLE agentgrant (id INTEGER,agent_id INTEGER,user_id INTEGER,permission TEXT,enabled INTEGER,created_at TEXT);
CREATE TABLE agentskillgrant (id INTEGER,skill_id INTEGER,user_id INTEGER,permission TEXT,enabled INTEGER,created_at TEXT);
CREATE TABLE modelprovider (id INTEGER,name TEXT,base_url TEXT,api_key TEXT,models TEXT,
default_model TEXT,builtin INTEGER,enabled INTEGER,organization_id INTEGER);
CREATE TABLE modelcatalog (id INTEGER,provider_id INTEGER,model_name TEXT,display_name TEXT,
capabilities TEXT,capability_status TEXT,enabled INTEGER,builtin INTEGER,rate_version TEXT,
input_rate REAL,output_rate REAL);
CREATE TABLE appsetting (key TEXT,value TEXT);
INSERT INTO modelprovider VALUES (3,'deepseek','https://api.deepseek.com/v1','DO-NOT-READ-SECRET',
'["deepseek-flash","deepseek-v4-pro"]','deepseek-flash',0,1,1);
INSERT INTO modelcatalog VALUES (6,3,'deepseek-v4-pro','Pro','[]','untested',1,0,'r1',1,2);
INSERT INTO modelcatalog VALUES (7,3,'deepseek-flash','Flash','[]','untested',1,0,'r1',1,2);
INSERT INTO appsetting VALUES ('default_model_provider_id','3');
""")
self.agent(1, owner_user_id=0, builtin=1, visibility="global", kb_ids="[3]")
self.agent(19, owner_user_id=3, status="disabled", visibility="private")
self.agent(22, owner_user_id=3, visibility="private")
self.skill(1, owner_user_id=0, builtin=1, visibility="global")
self.skill(5, owner_user_id=3, visibility="private")
self.skill(9, owner_user_id=1, visibility="private")
self.db.executemany("INSERT INTO agentskillbinding VALUES (?,?,?)",
[(101, 1, 1), (102, 1, 5), (103, 1, 9), (104, 19, 1), (105, 22, 1), (106, 22, 5)])
template = {key: "" for key in TEMPLATE_COLUMNS}
template.update(id=1, name="Source template", desc="Preserve template", version=1, enabled=1, builtin=1,
organization_id=1, owner_user_id=0, visibility="global", constraints='["constraint"]',
examples='[]', output_schema='{}', role_prompt="original-template-body",
task_template="{{runtime_input}}", created_at="2026-09-15", updated_at="2026-09-15")
self.insert("agentprompttemplate", TEMPLATE_COLUMNS, template)
self.identities, self.kbs = {1: ADMIN_TARGET_ID, 3: USER_ID}, {3: KB_ID}
def insert(self, table, columns, values):
self.db.execute("INSERT INTO " + table + " (" + ",".join(columns) + ") VALUES (" +
",".join("?" for _ in columns) + ")", [values[k] for k in columns])
def agent(self, source_id, **overrides):
values = {key: None for key in AGENT_COLUMNS}
values.update(id=source_id, name=f"Agent {source_id}", role="production", desc="original description",
content_tags='["source-tag"]', kb_ids='[]', fusion_agent_ids='[]', model_provider_id=None,
model_name='deepseek-chat', model_catalog_id=None, prompt_template_id=1,
allowed_tools='["knowledge_search","read_agent_memory"]', rules=" Original rules\n\n",
rules_version=2, status="active", builtin=0, organization_id=1, owner_user_id=1,
visibility="private", created_at="2026-09-15 00:00:00",
runtime_snapshot='{"primary":{"rules":"frozen-original-rules"}}', persona_json='{"voice":"plain"}',
digital_human_id=None, kb_category_ids='{}')
values.update(overrides)
self.insert("agent", AGENT_COLUMNS, values)
def skill(self, source_id, **overrides):
values = dict(id=source_id, name=f"Skill {source_id}", desc="source description",
instructions=" Original skill body\n\n", enabled=1, builtin=0, version=2,
organization_id=1, owner_user_id=1, visibility="private", created_at="2026-09-15 00:00:00")
values.update(overrides)
self.insert("agentskill", SKILL_COLUMNS, values)
def plan(self):
plan = asset_plan(self.db, self.identities, self.kbs, model_plan(self.db))
plan["source_snapshot_sha256"] = "a" * 64
return plan
def test_preserves_original_text_and_unsupported_configuration(self):
plan = self.plan()
self.assertEqual(plan["agents"][0]["target"]["config"]["system_prompt"], " Original rules\n\n")
self.assertEqual(plan["skills"][0]["target"]["instructions"], " Original skill body\n\n")
self.assertEqual(plan["agents"][0]["source"]["runtime_snapshot"]["primary"]["rules"], "frozen-original-rules")
self.assertEqual(plan["prompt_templates"][0]["role_prompt"], "original-template-body")
self.assertEqual(plan["agents"][0]["source"]["content_tags"], ["source-tag"])
self.assertEqual(plan["agents"][0]["target"]["config"]["content_tags"], ["source-tag"])
self.assertEqual(plan["agents"][0]["target"]["config"]["role"], "production")
self.assertNotIn("role", plan["unsupported_fields"])
self.assertNotIn("content_tags", plan["unsupported_fields"])
self.assertEqual(plan["agents"][0]["target"]["config"]["fusion_agent_ids"], [])
self.assertNotIn("fusion_agent_ids", plan["unsupported_fields"])
def test_rejects_invalid_fusion_configuration(self):
for value in ('[1]', '[2,2]', '[true]', '["2"]', '[3]', '[0]'):
with self.subTest(value=value):
self.db.execute("UPDATE agent SET fusion_agent_ids=? WHERE id=1", (value,))
with self.assertRaises(ValueError):
self.plan()
self.db.execute("UPDATE agent SET fusion_agent_ids='[]' WHERE id=1")
def test_maps_direct_fusions_without_adding_their_knowledge_or_recursion(self):
self.db.execute("UPDATE agent SET fusion_agent_ids='[22,19]' WHERE id=1")
self.db.execute("UPDATE agent SET fusion_agent_ids='[1]' WHERE id=22")
plan = self.plan()
config = plan["agents"][0]["target"]["config"]
self.assertEqual(config["fusion_agent_ids"], [asset_id("agent", 22), asset_id("agent", 19)])
self.assertEqual(config["knowledge_bases"], [KB_ID])
self.db.execute("UPDATE agent SET role='qc' WHERE id=19")
with self.assertRaises(ValueError):
self.plan()
def test_rejects_invalid_native_duties_and_tags(self):
for field, value in (("role", "unknown"), ("content_tags", '["same","same"]'),
("content_tags", '[" "]'), ("content_tags", '[1]')):
with self.subTest(field=field, value=value):
self.db.execute(f"UPDATE agent SET {field}=? WHERE id=1", (value,))
with self.assertRaises(ValueError):
self.plan()
self.db.execute("UPDATE agent SET role='production',content_tags='[\"source-tag\"]' WHERE id=1")
def test_replay_accepts_only_exact_pre_duty_snapshot(self):
sql = asset_sql(self.plan())
self.assertIn("NOT (mapping.native_snapshot->'config' ? 'role')", sql)
self.assertIn("NOT (mapping.native_snapshot->'config' ? 'content_tags')", sql)
self.assertIn("target #- '{config,role}' #- '{config,content_tags}'", sql)
def test_owner_zero_resolves_pinned_admin_and_private_owner_stays_separate(self):
plan = self.plan()
self.assertEqual(plan["agents"][0]["target"]["created_by"], ADMIN_TARGET_ID)
self.assertEqual(plan["skills"][0]["target"]["created_by"], ADMIN_TARGET_ID)
self.assertEqual(plan["agents"][1]["target"]["created_by"], USER_ID)
self.assertEqual(plan["agents"][1]["target"]["visibility"], "private")
self.assertFalse(plan["agents"][1]["target"]["enabled"])
self.assertFalse(plan["agents"][0]["target"]["is_builtin"])
def test_owner_checks_include_template_owners_but_exclude_unrelated_identities(self):
unrelated = "d574dd78-7ca7-41e1-ae19-e2874665a183"
template_owner = "a986bcf9-3325-4ff0-aac7-cbc1d93f5b63"
self.identities.update({2: unrelated, 8: template_owner})
self.db.execute("UPDATE agentprompttemplate SET owner_user_id=8 WHERE id=1")
plan = self.plan()
self.assertEqual(plan["identity_mapping"], {"1": ADMIN_TARGET_ID, "3": USER_ID, "8": template_owner})
self.assertEqual(self.identities[2], unrelated)
self.assertNotIn(unrelated, asset_sql(plan))
def test_private_cross_owner_binding_is_preserved_but_not_effective(self):
plan = self.plan()
self.assertEqual(len(plan["bindings"]), 6)
blocked = next(b for b in plan["bindings"] if b["source"]["id"] == 102)
self.assertFalse(blocked["effective"])
self.assertIn("blocked_private_owner_mismatch", blocked["review"])
self.assertNotIn(asset_id("skill", 5), plan["agents"][0]["target"]["config"]["instruction_skill_ids"])
self.assertIn(asset_id("skill", 9), plan["agents"][0]["target"]["config"]["instruction_skill_ids"])
self.assertIn(asset_id("skill", 5), plan["agents"][2]["target"]["config"]["instruction_skill_ids"])
def test_global_skill_remains_bound_to_private_other_owner_agent(self):
plan = self.plan()
selected = plan["agents"][2]["target"]["config"]["instruction_skill_ids"]
self.assertIn(asset_id("skill", 1), selected)
binding = next(b for b in plan["bindings"] if b["source"]["id"] == 105)
self.assertTrue(binding["effective"])
self.assertEqual(binding["review"], [])
def test_native_skill_is_non_executable_and_disabled_flag_is_preserved(self):
self.db.execute("UPDATE agentskill SET enabled=0 WHERE id=1")
plan = self.plan()
self.assertFalse(plan["skills"][0]["target"]["enabled"])
self.assertTrue(plan["skills"][0]["target"]["instruction_only"])
self.assertEqual(plan["skills"][0]["target"]["bundle_ref"], "")
for agent in plan["agents"]:
self.assertEqual(agent["target"]["config"]["allowed_tools"], [])
self.assertEqual(agent["target"]["config"]["skills_selection_mode"], "none")
def test_native_model_uses_reviewed_legacy_fallback(self):
plan = self.plan()
for item in plan["agents"]:
self.assertEqual(item["target"]["config"]["model_id"], model_id(3, "deepseek-flash"))
self.assertTrue(item["model_resolution"]["model_fallback"])
def test_selected_kb_is_mapped_and_no_kb_becomes_none(self):
plan = self.plan()
self.assertEqual(plan["agents"][0]["target"]["config"]["knowledge_bases"], [KB_ID])
self.assertEqual(plan["agents"][0]["target"]["config"]["kb_selection_mode"], "selected")
self.assertEqual(plan["agents"][1]["target"]["config"]["kb_selection_mode"], "none")
def test_readonly_and_no_credentials_read(self):
def authorize(action, table, column, database, trigger):
if action == sqlite3.SQLITE_READ and table == "modelprovider" and column == "api_key":
return sqlite3.SQLITE_DENY
return sqlite3.SQLITE_OK
self.db.set_authorizer(authorize)
self.db.execute("PRAGMA query_only=ON")
plan = self.plan()
self.assertNotIn("DO-NOT-READ-SECRET", json.dumps(plan))
def test_bad_identity_or_organization_rejected(self):
for identity in ({3: USER_ID}, {1: USER_ID, 3: ADMIN_TARGET_ID}, {1: ADMIN_TARGET_ID}):
with self.subTest(identity=identity), self.assertRaises(ValueError):
asset_plan(self.db, identity, self.kbs, model_plan(self.db))
self.db.execute("UPDATE agent SET organization_id=2 WHERE id=19")
with self.assertRaisesRegex(ValueError, "organization"):
self.plan()
def test_unknown_visibility_rejected(self):
self.db.execute("UPDATE agentskill SET visibility='shared' WHERE id=5")
with self.assertRaisesRegex(ValueError, "visibility"):
self.plan()
def test_explicit_grants_require_review(self):
self.db.execute("INSERT INTO agentskillgrant VALUES (1,5,1,'use',1,'2026-09-15')")
with self.assertRaisesRegex(ValueError, "explicit grants"):
self.plan()
def test_missing_and_duplicate_binding_rejected(self):
self.db.execute("INSERT INTO agentskillbinding VALUES (200,1,1)")
with self.assertRaisesRegex(ValueError, "duplicated"):
self.plan()
self.db.execute("DELETE FROM agentskillbinding WHERE id=200")
self.db.execute("INSERT INTO agentskillbinding VALUES (200,777,1)")
with self.assertRaisesRegex(ValueError, "missing asset"):
self.plan()
def test_unmapped_kb_or_category_scope_never_widens(self):
self.db.execute("UPDATE agent SET kb_ids='[7]' WHERE id=1")
with self.assertRaisesRegex(ValueError, "unmapped"):
self.plan()
self.db.execute("UPDATE agent SET kb_ids='[3]',kb_category_ids='{\"3\":[4]}' WHERE id=1")
with self.assertRaisesRegex(ValueError, "category restrictions"):
self.plan()
def test_missing_legacy_template_rejected(self):
self.db.execute("DELETE FROM agentprompttemplate")
with self.assertRaisesRegex(ValueError, "missing prompt template"):
self.plan()
def test_plan_is_deterministic_and_source_ids_are_namespaced(self):
self.assertEqual(self.plan(), self.plan())
self.assertNotEqual(asset_id("agent", 1), asset_id("skill", 1))
self.assertNotEqual(asset_id("agent", 1), asset_id("agent", 19))
def test_sql_requires_fingerprint(self):
for digest in (None, "", "A" * 64, "a" * 63):
plan = self.plan()
plan["source_snapshot_sha256"] = digest
with self.subTest(digest=digest), self.assertRaises(ValueError):
asset_sql(plan)
def test_tampered_targets_or_access_fail_before_sql(self):
mutations = [
lambda p: p.update(target_tenant_id=42),
lambda p: p.update(target_database="production"),
lambda p: p["agents"][1]["target"].update(enabled=True),
lambda p: p["agents"][1]["target"].update(visibility="global"),
lambda p: p["skills"][1]["target"].update(instruction_only=False),
lambda p: p["agents"][0]["target"]["config"].update(allowed_tools=["execute"]),
lambda p: p["agents"][0]["target"]["config"].update(system_prompt="changed"),
lambda p: p["bindings"][1].update(effective=True),
lambda p: p["bindings"][1].update(review=[]),
]
for mutate in mutations:
plan = self.plan()
mutate(plan)
with self.subTest(mutate=mutate), self.assertRaises(ValueError):
asset_sql(plan)
def test_safe_summary_contains_no_prompt_bodies(self):
summary = safe_asset_summary(self.plan())
self.assertEqual(summary["counts"]["bindings"], 6)
self.assertEqual(summary["effective_bindings"], 5)
self.assertEqual(summary["blocked_bindings"], 1)
self.assertEqual(summary["disabled_agent_ids"], [19])
self.assertNotIn("Original", json.dumps(summary))
self.assertNotIn("frozen", json.dumps(summary))
self.assertNotIn("template-body", json.dumps(summary))
def test_sql_is_guarded_insert_only_and_keeps_tombstones_and_user_edits(self):
sql = asset_sql(self.plan())
self.assertIn("current_database()", sql)
self.assertIn("pg_advisory_xact_lock", sql)
self.assertIn("Asset owner durable identity or membership mismatch", sql)
self.assertIn("Asset durable mapping conflict", sql)
self.assertIn("refusing recreation", sql)
self.assertIn("Native skill ID or owner/name collision", sql)
self.assertIn("Native Agent ID or owner/name collision", sql)
self.assertIn("effective boolean NOT NULL", sql)
self.assertIn("SET LOCAL log_min_error_statement = 'panic'", sql)
self.assertIn(f"CREATE TABLE IF NOT EXISTS {LEDGER}", sql)
self.assertIn(f"CREATE TABLE IF NOT EXISTS {BINDING_LEDGER}", sql)
self.assertNotIn("ON CONFLICT", sql.upper())
self.assertNotIn("UPDATE CUSTOM_AGENTS", sql.upper())
self.assertNotIn("UPDATE TENANT_SKILL_CATALOG", sql.upper())
self.assertNotIn("COMMIT", sql.upper())
self.assertNotIn("BEGIN TRANSACTION", sql.upper())
self.assertNotIn("SELECT value FROM jsonb_each_text", sql)
def test_text_is_sql_escaped_without_semantic_transformation(self):
value = " O'Brien\\line\n$assets$; SELECT 'opaque data'; "
self.db.execute("UPDATE agent SET rules=? WHERE id=1", (value,))
plan = self.plan()
self.assertEqual(plan["agents"][0]["target"]["config"]["system_prompt"], value)
sql = asset_sql(plan)
self.assertIn("O''Brien", sql)
self.assertIn("standard_conforming_strings = on", sql)
if __name__ == "__main__":
unittest.main()
+273
View File
@@ -0,0 +1,273 @@
"""Historical imports retain source facts and never start model execution."""
import copy
import base64
import json
import sqlite3
import sys
import tempfile
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
from weknora_business_etl import (ADMIN_TARGET_ID, HISTORY_TABLES, JSON_FIELDS, LEDGER, RESOURCE_KINDS,
business_id, business_plan, business_sql, safe_business_summary, postgres_plan)
USER = "ab043a7d-d10e-58fe-8417-3fd2cbab5f77"
AGENT = "72a9a19f-65f3-4996-897b-efca38654b70"
KB = "7e6ac485-f766-4d58-ae38-2f0456a022ef"
class BusinessTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.artifact = self.root / "seg_1_r1.md"
self.artifact.write_text(" Original paragraph O'Brien\n\n", encoding="utf-8")
self.db = sqlite3.connect(":memory:")
self.addCleanup(self.db.close)
shared = dict(organization_id=1, owner_user_id=3, visibility="private", created_at="2026-09-15 01:00:00")
flow = dict(id=1, name="Flow", nodes=[{"node_id": "a", "agent_id": 9, "enabled": True, "x": 20, "y": 40}],
edges=[], supervisor_agent_id=None, supervisor_mode="off", **shared)
map_row = dict(id=2, owner_user_id=3, organization_id=1, topic="Course", kb_ids=[5], kb_category_ids={},
requested_days=1, agent_id=9, version=2, confirmed_version=2,
plan={"recommended_days": 1, "rationale": "Original rationale", "insufficiencies": [],
"days": [{"day": 1, "title": "Day one", "source_ids": ["E1"], "support": "Original support", "prerequisite": "Prerequisite"}]},
evidence=[{"id": "E1", "text": "Private evidence"}], coverage={"scope": "quick"},
assessment_cancel_requested=0, maps={"1": {"day": 1, "title": "Daily framework", "children": [
{"title": "Node title", "content": "Node content", "source_ids": ["E1"], "hook": "Original hook"}]}},
status="confirmed", error="")
campaign = dict(id=3, name="Task", mindmap_text="Original map text", objective="Legacy objective", success_criteria="Legacy criteria",
canvas_snapshot={"nodes": flow["nodes"], "edges": []}, canvas_name="Flow", duration_minutes=60,
qc_mode="full", previous_campaign_id=None, task_spec=[{"title": "Original paragraph"}],
pending_confirms=["Original pending"], notes=[], status="human_review", has_failure=0,
finished_at="", **shared)
segment = dict(id=4, campaign_id=3, node_id="a", agent_id=9, agent_assignment_mode="manual", generation_history=[],
fusion_agent_ids=[], runtime_snapshot={"primary": {"rules": "Private frozen rules"}},
content_tag="Tag", title="Original paragraph", time_range="00:00-00:10", mindmap_node="Source node",
product_ref="", status="approved", round=1, artifact_path=str(self.artifact), anchor_card={},
qc_reports=[], failure={}, references=[{"chunk_id": "private-chunk", "text": "Private reference"}],
kb_category_ids=[], kb_category_recommendation={}, updated_at="2026-09-15 02:00:00")
run = dict(id=5, campaign_id=3, user_id=3, organization_id=1, status="completed", cancel_requested=0,
pause_requested=0, checkpoint={"step": "review"}, config_snapshot={"rules": "Original runtime config"},
trace_id="original-trace", runtime_version="v1", error="", created_at="2026-09-15 01:01:00",
started_at="2026-09-15 01:01:00", finished_at="2026-09-15 02:00:00")
report = dict(id=6, segment_id=4, gate="cross", checks=[{"result": "Original failed check"}], verdict="FAIL", run_id=5,
trace_id="original-trace", runtime_snapshot={}, created_at="2026-09-15 01:40:00")
event = dict(id=7, run_id=5, campaign_id=3, segment_id=4, node_id="a", trace_id="original-trace", span_id="span",
parent_span_id="", actor_agent_id=9, actor_name="Agent", tool_name="knowledge_search", event_type="error",
status="failed", payload={"error": "Original model error"}, created_at="2026-09-15 01:30:00")
session = dict(id=8, run_id=5, campaign_id=3, segment_id=4, agent_id=9, agent_name="Agent", parent_session_id=None,
phase="production", trace_id="original-trace", status="completed", turn=1, max_turns=6,
messages=[{"role": "system", "content": "Private historical prompt"}], tool_state={},
result_text="Original model result", result_meta={}, error="", created_at="2026-09-15 01:10:00",
updated_at="2026-09-15 01:20:00", finished_at="2026-09-15 01:20:00")
queue = dict(id=9, task_key="campaign-run:5", payload={"run_id": 5, "campaign_id": 3}, status="done", attempt=1)
rows = {"canvas": [flow], "trainingmap": [map_row], "campaign": [campaign], "segment": [segment],
"campaignrun": [run], "qcreport": [report], "runevent": [event], "agentsession": [session], "taskqueueitem": [queue]}
for table in (*RESOURCE_KINDS, *HISTORY_TABLES):
sample = rows.get(table, [{"id": 0}])[0]
columns = ",".join('"' + key + '"' + (" INTEGER" if type(value) is int else " TEXT") for key, value in sample.items())
self.db.execute(f'CREATE TABLE "{table}" ({columns})')
for row in rows.get(table, []):
self.insert(table, row)
self.db.commit()
self.db.execute("PRAGMA query_only=ON")
self.identities = {"1": ADMIN_TARGET_ID, "3": USER}
def insert(self, table, row):
columns = ",".join('"' + key + '"' for key in row)
values = [json.dumps(value) if key in JSON_FIELDS.get(table, ()) else value for key, value in row.items()]
self.db.execute(f'INSERT INTO "{table}" ({columns}) VALUES (' + ",".join("?" for _ in row) + ")", values)
def change(self, sql, values=()):
self.db.execute("PRAGMA query_only=OFF")
self.db.execute(sql, values)
self.db.execute("PRAGMA query_only=ON")
def plan(self):
result = business_plan(self.db, self.identities, {"9": AGENT}, {"5": KB}, self.root)
result["source_snapshot_sha256"] = "a" * 64
return result
def resource(self, plan, kind):
return next(r["target"] for r in plan["resources"] if r["source_kind"] == kind)
def test_maps_agents_owners_and_knowledge_without_inference(self):
plan = self.plan()
flow = self.resource(plan, "canvas")
self.assertEqual(flow["owner_id"], USER)
self.assertEqual(flow["data"]["nodes"][0]["agent_id"], AGENT)
self.assertEqual(flow["data"]["legacy_history"]["nodes"][0]["x"], 20)
training = self.resource(plan, "trainingmap")
self.assertEqual(training["data"]["knowledge_base_ids"], [KB])
self.assertEqual(training["version"], 2)
task = self.resource(plan, "campaign")
self.assertEqual(task["data"]["map_id"], "")
self.assertEqual(task["data"]["flow_id"], "")
self.assertEqual(task["data"]["day"], 0)
def test_preserves_native_text_and_source_status(self):
plan = self.plan()
task = self.resource(plan, "campaign")
segment = task["data"]["segments"][0]
self.assertEqual(segment["content"], " Original paragraph O'Brien\n\n")
self.assertEqual(segment["status"], "approved")
self.assertEqual(task["status"], "human_review")
self.assertEqual(task["data"]["composed"], "")
self.assertIn('"verdict": "FAIL"', segment["review"])
self.assertIn("Original failed check", task["data"]["review"])
def test_daily_framework_preserves_node_content_without_forging_confirmation(self):
data = self.resource(self.plan(), "trainingmap")["data"]
day = data["days"][0]
self.assertEqual(day["nodes"][0]["label"], "Node title")
self.assertIn("Node content", day["framework"])
self.assertIn("Original hook", day["framework"])
self.assertFalse(day["confirmed"])
self.assertEqual(data["legacy_history"]["confirmed_version"], 2)
self.assertEqual(data["legacy_history"]["plan"]["days"][0]["support"], "Original support")
def test_every_resource_blocks_automatic_execution(self):
for resource in self.plan()["resources"]:
marker = resource["target"]["data"]["legacy_import"]
self.assertTrue(marker["read_only"])
self.assertTrue(marker["requires_review"])
self.assertEqual(marker["source_id"], resource["source_id"])
task = self.resource(self.plan(), "campaign")
self.assertTrue(task["data"]["legacy_import"]["needs_configuration"])
def test_missing_agent_and_category_scope_are_not_widened(self):
self.change("UPDATE trainingmap SET agent_id=NULL,kb_category_ids=?", (json.dumps({"5": [9]}),))
data = self.resource(self.plan(), "trainingmap")["data"]
self.assertEqual(data["agent_id"], "")
self.assertIn("knowledge_category_scope", data["legacy_import"]["missing_configuration"])
self.assertTrue(data["legacy_import"]["read_only"])
def test_empty_canvas_remains_empty_and_requires_configuration(self):
self.change("UPDATE canvas SET nodes='[]'")
data = self.resource(self.plan(), "canvas")["data"]
self.assertEqual(data["nodes"], [])
self.assertIn("flow_nodes", data["legacy_import"]["missing_configuration"])
def test_unmapped_or_deleted_owner_is_not_reassigned_to_admin(self):
self.change("UPDATE canvas SET owner_user_id=2")
with self.assertRaisesRegex(ValueError, "no durable identity"):
self.plan()
def test_invalid_identity_organization_or_snapshot_rejected(self):
self.change("UPDATE canvas SET organization_id=2")
with self.assertRaisesRegex(ValueError, "organization"):
self.plan()
self.change("UPDATE canvas SET organization_id=1")
self.db.execute("PRAGMA query_only=OFF")
with self.assertRaisesRegex(ValueError, "read-only"):
self.plan()
def test_runtime_and_history_go_to_ledger_without_native_jobs_or_events(self):
plan = self.plan()
sources = {r["source_kind"]: r["source"] for r in plan["history"]}
self.assertEqual(sources["agentsession"]["messages"][0]["content"], "Private historical prompt")
self.assertEqual(sources["runevent"]["payload"]["error"], "Original model error")
self.assertEqual(sources["taskqueueitem"]["status"], "done")
sql = business_sql(plan).upper()
for table in ("WORKBENCH_JOBS", "WORKBENCH_EVENTS", "WORKBENCH_REVISIONS", "KNOWLEDGE_USAGE", "POINTACCOUNT"):
self.assertNotIn("INSERT INTO " + table, sql)
def test_conflicting_old_qc_run_is_preserved_but_not_displayed_as_current_qc(self):
self.change("UPDATE qcreport SET run_id=999")
plan = self.plan()
report = next(i for i in plan["history"] if i["source_kind"] == "qcreport")
self.assertEqual(report["source"]["run_id"], 999)
self.assertIn("source_run_campaign_mismatch_ledger_only", report["relation_review"])
self.assertEqual(self.resource(plan, "campaign")["data"]["segments"][0]["review"], "")
self.assertEqual(self.resource(plan, "campaign")["data"]["review"], "")
def test_unrelated_legacy_sessions_are_not_copied(self):
self.change("UPDATE agentsession SET campaign_id=999")
plan = self.plan()
self.assertFalse(any(i["source_kind"] == "agentsession" for i in plan["history"]))
self.assertEqual(plan["excluded_history_counts"]["agentsession"], 1)
def test_all_historical_artifact_versions_are_preserved(self):
older = self.root / "seg_1_r0.md"
older.write_text("Previous version", encoding="utf-8")
self.change("UPDATE segment SET generation_history=?", (json.dumps([{"round": 0, "artifact_path": str(older)}]),))
plan = self.plan()
self.assertEqual(len(plan["artifacts"]), 2)
self.assertEqual(plan["artifacts"][str(older)]["body"], "Previous version")
def test_missing_artifact_is_explicit_and_does_not_fake_content(self):
self.artifact.unlink()
plan = self.plan()
self.assertEqual(safe_business_summary(plan)["artifacts_missing"], 1)
task = self.resource(plan, "campaign")["data"]
self.assertEqual(task["segments"][0]["content"], "")
self.assertIn("segment_artifact", task["legacy_import"]["missing_configuration"])
def test_artifact_cannot_escape_allowed_directory(self):
self.change("UPDATE segment SET artifact_path='/etc/passwd'")
with self.assertRaisesRegex(ValueError, "outside"):
self.plan()
def test_report_omits_private_bodies_and_paths(self):
summary = json.dumps(safe_business_summary(self.plan()))
for forbidden in ("Original", "Private", "O'Brien", self.tmp.name, "historical prompt", "Node content"):
self.assertNotIn(forbidden, summary)
def test_tampering_projection_or_artifact_fails(self):
mutations = [lambda p: p.update(target_database="production"), lambda p: p.update(target_tenant_id=1),
lambda p: p["resources"][0]["target"].update(owner_id=ADMIN_TARGET_ID),
lambda p: p["resources"][0]["target"]["data"]["legacy_import"].update(read_only=False),
lambda p: p["artifacts"][str(self.artifact)].update(body="Changed body"),
lambda p: p["history"][0].update(target_id=AGENT)]
for mutate in mutations:
plan = self.plan()
mutate(plan)
with self.subTest(mutate=mutate), self.assertRaises(ValueError):
business_sql(plan)
def test_bad_fingerprint_fails(self):
for digest in (None, "", "X" * 64, "a" * 63):
plan = self.plan()
plan["source_snapshot_sha256"] = digest
with self.subTest(digest=digest), self.assertRaises(ValueError):
business_sql(plan)
def test_null_in_archived_evidence_is_losslessly_encoded_for_postgres(self):
original = "source\x00evidence\ud800"
self.change("UPDATE trainingmap SET evidence=?", (json.dumps([{"text": original}]),))
plan = self.plan()
encoded = postgres_plan(plan)
item = next(r for r in encoded["resources"] if r["source_kind"] == "trainingmap")
source = item["source"]["evidence"][0]["text"]
self.assertEqual(base64.b64decode(source["body"]).decode("utf-8", errors="surrogatepass"), original)
self.assertEqual(item["target"]["data"]["legacy_history"]["evidence"][0]["text"], source)
self.assertEqual(self.resource(plan, "trainingmap")["data"]["legacy_history"]["evidence"][0]["text"], original)
self.assertNotIn("\\u0000", business_sql(plan))
self.assertNotIn("\\ud800", business_sql(plan))
def test_sql_is_insert_only_tenant_guarded_and_replay_preserves_edits(self):
sql = business_sql(self.plan())
for marker in ("current_database()", "yawan_native_acceptance_20260915", "pg_advisory_xact_lock(10059",
"Business owner durable identity", "Business Agent durable mapping", "Business durable mapping conflict",
"refusing recreation", "Native business ID or owner/name collision", "source_snapshot jsonb",
"artifact_snapshot jsonb", "SET LOCAL log_min_error_statement = 'panic'", LEDGER):
self.assertIn(marker, sql)
for forbidden in ("UPDATE WORKBENCH_RESOURCES", "DELETE FROM", "ON CONFLICT", "COMMIT;", "BEGIN TRANSACTION"):
self.assertNotIn(forbidden, sql.upper())
self.assertIn("O''Brien", sql)
def test_namespaced_ids_and_plan_are_deterministic(self):
self.assertEqual(self.plan(), self.plan())
self.assertNotEqual(business_id("segment", 1), business_id("canvas", 1))
self.assertNotEqual(business_id("trainingmap", 1, "1.0"), business_id("trainingmap", 1, "1.1"))
def test_duplicate_node_or_dangling_edge_fails(self):
self.change("UPDATE canvas SET edges=?", (json.dumps([{"from": "a", "to": "missing"}]),))
with self.assertRaisesRegex(ValueError, "missing node"):
self.plan()
if __name__ == "__main__":
unittest.main()
+116
View File
@@ -0,0 +1,116 @@
"""Category import regressions: tenant isolation, splits and tree ancestry."""
import copy
import unittest
from weknora_category_etl import TENANT, build_manifest, document_stem
def fixture():
source = {
"source_mutated": False, "relation_validation": {"valid": True}, "source": {"sha256": "fixture"},
"mapping": {
"knowledge_bases": [{"legacy_id": 1, "native_id": "plan-kb", "name": "Library"}],
"categories": [
{"legacy_id": 1, "native_id": "plan-root", "knowledge_base_id": "plan-kb", "parent_id": None, "name": "Root", "sort_order": 0},
{"legacy_id": 2, "native_id": "plan-child", "knowledge_base_id": "plan-kb", "parent_id": "plan-root", "name": "Child", "sort_order": 1},
],
"files": [{"legacy_id": 1, "knowledge_base_id": "plan-kb", "file_name": "Root__Child.jsonl", "category_id": "plan-child", "content_hash": "source-hash", "revision": 1}],
},
}
state = {"tenant_id": TENANT, "knowledge_bases": {"1": {"id": "real-kb", "legacy_name": "Library", "complete": True, "failed": [], "uploaded": ["Root__Child.jsonl"]}}}
target = {
"tenant": {"id": TENANT, "status": "active"},
"knowledge_bases": [{"id": "real-kb", "tenant_id": TENANT, "name": "Library"}],
"tags": [], "relations": [],
"files": [{"id": "doc", "tenant_id": TENANT, "knowledge_base_id": "real-kb", "file_name": "Root__Child.md", "file_hash": "target-hash", "file_size": 123, "parse_status": "completed", "folder_path": "", "custom_metadata": {"existing": "preserved"}}],
}
return source, state, target
class CategoryMigrationTests(unittest.TestCase):
def test_parent_tag_and_folder_tree_use_actual_kb(self):
result = build_manifest(*fixture())
self.assertEqual(result["files"][0]["folder_path"], "Root/Child")
self.assertEqual(result["files"][0]["metadata"]["existing"], "preserved")
self.assertEqual(len(result["relations"]), 2)
self.assertTrue(all(tag["knowledge_base_id"] == "real-kb" for tag in result["tags"]))
def test_split_source_maps_both_parts(self):
source, state, target = fixture()
state["knowledge_bases"]["1"]["uploaded"] = ["Root__Child.part001.md", "Root__Child.part002.md"]
file = target["files"][0]
target["files"] = [{**file, "id": f"doc-{part}", "file_name": f"Root__Child.part{part:03}.md"} for part in (1, 2)]
result = build_manifest(source, state, target)
self.assertEqual(result["summary"]["source_files"], 1)
self.assertEqual(result["summary"]["target_files"], 2)
self.assertEqual(len(result["relations"]), 4)
def test_partial_split_rejected(self):
source, state, target = fixture()
state["knowledge_bases"]["1"]["uploaded"] = ["Root__Child.part001.md", "Root__Child.part003.md"]
file = target["files"][0]
target["files"] = [{**file, "id": f"doc-{part}", "file_name": f"Root__Child.part{part:03}.md"} for part in (1, 3)]
with self.assertRaisesRegex(ValueError, "split"):
build_manifest(source, state, target)
def test_missing_split_document_rejected(self):
source, state, target = fixture()
state["knowledge_bases"]["1"]["uploaded"] = ["Root__Child.part001.md", "Root__Child.part002.md"]
target["files"][0]["file_name"] = "Root__Child.part001.md"
with self.assertRaisesRegex(ValueError, "missing"):
build_manifest(source, state, target)
def test_cross_tenant_and_wrong_mapping_rejected(self):
for item in ("tenant", "knowledge_base", "file"):
with self.subTest(item=item):
source, state, target = fixture()
if item == "tenant":
target["tenant"]["id"] += 1
elif item == "knowledge_base":
target["knowledge_bases"][0]["tenant_id"] += 1
else:
target["files"][0]["tenant_id"] += 1
with self.assertRaises(ValueError):
build_manifest(source, state, target)
def test_same_name_does_not_adopt_existing_tag(self):
source, state, target = fixture()
target["tags"] = [{"id": "someone-elses-tag", "tenant_id": TENANT, "knowledge_base_id": "real-kb", "name": "Root", "sort_order": 0, "deleted_at": None}]
with self.assertRaisesRegex(ValueError, "conflicts"):
build_manifest(source, state, target)
def test_second_plan_is_stable_and_preserves_user_metadata(self):
source, state, target = fixture()
first = build_manifest(source, state, target)
target["tags"] = [{**t, "deleted_at": None} for t in first["tags"]]
target["files"][0]["folder_path"] = first["files"][0]["folder_path"]
target["files"][0]["custom_metadata"] = first["files"][0]["metadata"]
second = build_manifest(source, state, target)
self.assertEqual(first["tags"], second["tags"])
self.assertEqual(first["relations"], second["relations"])
self.assertEqual(first["files"][0]["metadata"], second["files"][0]["metadata"])
def test_manual_changes_and_unmapped_files_are_not_overwritten(self):
source, state, target = fixture()
for change in ("folder", "unmapped", "duplicate"):
changed = copy.deepcopy(target)
if change == "folder":
changed["files"][0]["folder_path"] = "User folder"
elif change == "unmapped":
changed["files"].append({**changed["files"][0], "id": "extra", "file_name": "Other.md"})
else:
changed["files"].append({**changed["files"][0], "id": "duplicate"})
with self.subTest(change=change), self.assertRaises(ValueError):
build_manifest(source, state, changed)
def test_path_traversal_rejected(self):
with self.assertRaises(ValueError):
document_stem("../outside.md")
source, state, target = fixture()
source["mapping"]["categories"][0]["name"] = ".."
with self.assertRaises(ValueError):
build_manifest(source, state, target)
if __name__ == "__main__":
unittest.main()
+34
View File
@@ -0,0 +1,34 @@
"""Regression checks for the read-only SQLite snapshot used by weknora_etl."""
import sqlite3
import sys
import tempfile
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
from weknora_etl import consistent_snapshot
class SnapshotTests(unittest.TestCase):
def test_backup_includes_uncheckpointed_wal(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "legacy.sqlite"
source = sqlite3.connect(path)
source.execute("PRAGMA journal_mode=WAL")
source.execute("CREATE TABLE item (id INTEGER PRIMARY KEY, value TEXT)")
source.commit()
source.execute("INSERT INTO item(value) VALUES ('only-in-wal')")
source.commit()
self.assertTrue(Path(str(path) + "-wal").exists())
with consistent_snapshot(path) as (snapshot, info):
self.assertEqual(info["snapshot_method"], "sqlite_backup")
self.assertEqual(snapshot.execute("SELECT value FROM item").fetchone()[0], "only-in-wal")
self.assertEqual(snapshot.execute("PRAGMA query_only").fetchone()[0], 1)
with self.assertRaises(sqlite3.OperationalError):
snapshot.execute("CREATE TABLE forbidden (id INTEGER)")
source.close()
if __name__ == "__main__":
unittest.main()
+210
View File
@@ -0,0 +1,210 @@
"""Read-only planner and generated-SQL contract checks for identity migration."""
import copy
import json
import sqlite3
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
from weknora_identity_etl import (ADMIN_TARGET_ID, IDENTITY_LEDGER, ORGANIZATION_LEDGER,
PREFERENCE_KEY, TARGET_TENANT, _stable_id, identity_plan,
identity_sql, identity_verification_sql)
class IdentityTests(unittest.TestCase):
def setUp(self):
self.db = sqlite3.connect(":memory:")
self.addCleanup(self.db.close)
self.db.executescript("""
CREATE TABLE organization (id INTEGER, name TEXT, slug TEXT, enabled INTEGER);
CREATE TABLE roledefinition (id INTEGER, role TEXT, label TEXT, description TEXT,
permissions TEXT, builtin INTEGER, organization_id INTEGER, updated_at TEXT);
CREATE TABLE appuser (id INTEGER, username TEXT, display_name TEXT, role TEXT,
organization_id INTEGER, enabled INTEGER, password_hash TEXT);
INSERT INTO organization VALUES (1, 'source org', 'default', 1);
INSERT INTO roledefinition VALUES (1, 'admin', 'Admin', 'All', '["*"]', 1, 1, '2026-09-15');
INSERT INTO roledefinition VALUES (2, 'editor', 'Editor', 'Edit', '["kb:read", "feature:knowledge", "tasks:write"]', 1, 1, '2026-09-15');
INSERT INTO roledefinition VALUES (3, 'reviewer', 'Reviewer', 'Review', '["audit:read", "review"]', 1, 1, '2026-09-15');
INSERT INTO roledefinition VALUES (4, 'viewer', 'Viewer', 'View', '["read"]', 1, 1, '2026-09-15');
INSERT INTO appuser VALUES (1, 'yawan_admin', 'Legacy Admin', 'admin', 1, 1, 'secret-old-hash');
INSERT INTO appuser VALUES (3, 'editor_one', 'Display One', 'editor', 1, 1, 'secret-old-hash');
INSERT INTO appuser VALUES (8, 'wuzong', 'Display Two', 'editor', 1, 1, 'secret-old-hash');
INSERT INTO appuser VALUES (5, 'deleted_test', 'Test', 'viewer', 1, 0, 'secret-old-hash');
INSERT INTO appuser VALUES (6, 'deleted_test_two', 'Test Two', 'viewer', 1, 0, 'secret-old-hash');
INSERT INTO appuser VALUES (7, 'deleted_test_three', 'Test Three', 'viewer', 1, 0, 'secret-old-hash');
""")
def plan(self):
plan = identity_plan(self.db, TARGET_TENANT)
plan["source_snapshot_sha256"] = "a" * 64
return plan
def test_three_enabled_users_and_disabled_manifest(self):
plan = self.plan()
self.assertEqual([u["source_user"]["id"] for u in plan["users"]], [1, 3, 8])
self.assertEqual(plan["skipped"][0]["reason"], "source_account_disabled")
self.assertEqual(plan["skipped"][0]["source_user"]["id"], 5)
self.assertEqual(len(plan["skipped"]), 3)
self.assertNotIn("secret-old-hash", json.dumps(plan))
admin, first, second = plan["users"]
self.assertEqual(admin["target_id"], ADMIN_TARGET_ID)
self.assertEqual(admin["action"], "map_existing_admin")
for user in (first, second):
self.assertFalse(user["is_active"])
self.assertFalse(user["is_system_admin"])
self.assertFalse(user["can_access_all_tenants"])
self.assertEqual(user["membership_status"], "suspended")
self.assertEqual(user["target_membership_role"], "contributor")
self.assertTrue(user["target_email"].endswith("@migration.invalid"))
self.assertIn("permission_review_required", user["pending_activation"])
self.assertIn("password_reset_required", user["pending_activation"])
metadata = first["preferences"][PREFERENCE_KEY]
self.assertEqual(metadata["source_user"]["display_name"], "Display One")
self.assertIn("tasks:write", metadata["source_user"]["permissions"])
def test_all_source_roles_preserved_including_roles_without_active_users(self):
roles = {role["role"]: role for role in self.plan()["source_role_definitions"]}
self.assertEqual(set(roles), {"admin", "editor", "reviewer", "viewer"})
self.assertEqual(roles["reviewer"]["permissions"], ["audit:read", "review"])
self.assertEqual(roles["reviewer"]["description"], "Review")
self.assertEqual(roles["reviewer"]["updated_at"], "2026-09-15")
def test_sql_requires_snapshot_fingerprint(self):
for digest in (None, "", "not-a-digest", "A" * 64, "a" * 63):
plan = self.plan()
if digest is None:
del plan["source_snapshot_sha256"]
else:
plan["source_snapshot_sha256"] = digest
with self.subTest(digest=digest), self.assertRaises(ValueError):
identity_sql(plan)
def test_planner_does_not_read_password_column_or_write(self):
def authorize(action, first, second, database, trigger):
if action == sqlite3.SQLITE_READ and first == "appuser" and second == "password_hash":
return sqlite3.SQLITE_DENY
return sqlite3.SQLITE_OK
self.db.set_authorizer(authorize)
self.db.execute("PRAGMA query_only=ON")
self.assertEqual(len(self.plan()["users"]), 3)
def test_ids_are_deterministic_and_tenant_scoped(self):
self.assertEqual(self.plan(), self.plan())
self.assertNotEqual(_stable_id(3, TARGET_TENANT), _stable_id(3, TARGET_TENANT + 1))
self.assertNotEqual(_stable_id(3, TARGET_TENANT), _stable_id(8, TARGET_TENANT))
def test_target_tenant_rejected(self):
for tenant in (1, TARGET_TENANT + 1, str(TARGET_TENANT), True):
with self.subTest(tenant=tenant), self.assertRaises(ValueError):
identity_plan(self.db, tenant)
def test_foreign_source_organization_rejected(self):
self.db.execute("INSERT INTO organization VALUES (2, 'other', 'other', 1)")
with self.assertRaisesRegex(ValueError, "organization"):
self.plan()
def test_cross_organization_user_rejected_even_if_disabled(self):
self.db.execute("UPDATE appuser SET organization_id=2 WHERE id=5")
with self.assertRaisesRegex(ValueError, "organization"):
self.plan()
def test_cross_organization_role_rejected(self):
self.db.execute("UPDATE roledefinition SET organization_id=2 WHERE role='editor'")
with self.assertRaisesRegex(ValueError, "organization"):
self.plan()
def test_unreviewed_enabled_account_rejected(self):
self.db.execute("INSERT INTO appuser VALUES (10, 'extra', 'Extra', 'editor', 1, 1, '')")
with self.assertRaisesRegex(ValueError, "reviewed migration batch"):
self.plan()
def test_admin_mapping_cannot_silently_change(self):
self.db.execute("UPDATE appuser SET username='other_admin' WHERE id=1")
with self.assertRaisesRegex(ValueError, "explicitly mapped"):
self.plan()
def test_unknown_enabled_role_rejected(self):
self.db.execute("UPDATE appuser SET role='viewer' WHERE id=3")
with self.assertRaisesRegex(ValueError, "Unsupported"):
self.plan()
def test_duplicate_source_names_rejected(self):
self.db.execute("UPDATE appuser SET username='EDITOR_ONE' WHERE id=8")
with self.assertRaisesRegex(ValueError, "Conflicting"):
self.plan()
def test_sql_rejects_tampered_scope_or_activation(self):
base = self.plan()
mutations = [
lambda p: p.update(target_tenant_id=1),
lambda p: p.update(source="other"),
lambda p: p.update(target_database="production"),
lambda p: p["users"][0].update(target_id="other-admin"),
lambda p: p["users"][1].update(is_active=True),
lambda p: p["users"][1].update(is_system_admin=True),
lambda p: p["users"][1].update(can_access_all_tenants=True),
lambda p: p["users"][1].update(membership_status="active"),
lambda p: p["users"][1].update(target_membership_role="owner"),
lambda p: p["users"][1].update(target_email="real@example.com"),
lambda p: p["users"][1]["source_user"].update(organization_id=2),
lambda p: p["users"][1]["preferences"][PREFERENCE_KEY].update(source="other"),
]
for mutation in mutations:
plan = copy.deepcopy(base)
mutation(plan)
with self.subTest(mutation=mutation), self.assertRaises(ValueError):
identity_sql(plan)
def test_sql_escapes_source_text_and_does_not_emit_a_password(self):
self.db.execute("UPDATE appuser SET display_name=? WHERE id=3", ("O'Brien\\note $identity$",))
sql = identity_sql(self.plan())
self.assertIn("O''Brien", sql)
self.assertIn("SET LOCAL standard_conforming_strings = on", sql)
self.assertIn("gen_random_uuid()", sql)
self.assertNotIn("secret-old-hash", sql)
def test_sql_contract_aborts_conflicts_and_preserves_exact_reruns(self):
sql = identity_sql(self.plan())
self.assertIn("current_database()", sql)
self.assertIn("lower(btrim(username))", sql)
self.assertIn("lower(btrim(email))", sql)
self.assertIn("ID/provenance collision", sql)
self.assertIn("username/email collision", sql)
self.assertIn("Orphan or deleted membership conflicts", sql)
self.assertIn("existing_user.preferences->'yawan_identity_migration' IS DISTINCT FROM", sql)
self.assertIn("existing_member.status NOT IN ('suspended', 'active')", sql)
self.assertIn("unexpected cross-tenant membership", sql)
self.assertIn("Durable identity mapping conflict", sql)
self.assertIn("IF NOT has_mapping AND (", sql)
self.assertIn("refusing to recreate a deleted account", sql)
self.assertNotIn("ON CONFLICT", sql.upper())
self.assertNotIn("UPDATE USERS", sql.upper())
self.assertNotIn("UPDATE TENANT_MEMBERS", sql.upper())
self.assertNotIn("COMMIT", sql.upper())
self.assertNotIn("BEGIN TRANSACTION", sql.upper())
self.assertEqual(sql.count("INSERT INTO users"), 1)
self.assertEqual(sql.count("INSERT INTO tenant_members"), 1)
def test_sql_stores_all_identity_mappings_without_modifying_existing_accounts(self):
sql = identity_sql(self.plan())
self.assertIn(f"CREATE TABLE IF NOT EXISTS {ORGANIZATION_LEDGER}", sql)
self.assertIn(f"CREATE TABLE IF NOT EXISTS {IDENTITY_LEDGER}", sql)
self.assertIn("source_role_definitions jsonb NOT NULL", sql)
self.assertIn("source_snapshot_sha256 varchar(64) NOT NULL", sql)
self.assertEqual(sql.count(f"INSERT INTO {IDENTITY_LEDGER}"), 1)
self.assertGreater(sql.index(f"INSERT INTO {IDENTITY_LEDGER}"), sql.index("Pinned administrator"))
self.assertNotIn("CONTINUE;", sql)
self.assertNotIn("ON CONFLICT", sql.upper())
def test_readonly_verifier_reports_mappings_and_current_state_without_password(self):
sql = identity_verification_sql(self.plan())
for field in ("database_matches", "organization_mapping_matches", "identity_mapping_matches",
"account_scope_matches", "membership_matches", "is_active", "membership_status"):
self.assertIn(field, sql)
for operation in ("INSERT INTO", "UPDATE USERS", "CREATE TABLE", "DELETE FROM", "password_hash"):
self.assertNotIn(operation, sql)
if __name__ == "__main__":
unittest.main()
+202
View File
@@ -0,0 +1,202 @@
"""Safe planner, source routing, and generated-SQL model migration contracts."""
import base64
import copy
import json
import sqlite3
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
from weknora_model_etl import (LEDGER, TARGET_TENANT, model_id, model_plan, model_sql,
model_verification_sql, read_credentials, resolve_agent_model)
def fake_encryption(value):
# This verifies the encoder boundary only; it is never a production cipher.
return "enc:v1:" + base64.urlsafe_b64encode(b"x" * (12 + len(value.encode()) + 16)).decode().rstrip("=")
class ModelTests(unittest.TestCase):
def setUp(self):
self.db = sqlite3.connect(":memory:")
self.addCleanup(self.db.close)
self.db.executescript("""
CREATE TABLE modelprovider (id INTEGER,name TEXT,base_url TEXT,api_key TEXT,models TEXT,
default_model TEXT,builtin INTEGER,enabled INTEGER,organization_id INTEGER);
CREATE TABLE modelcatalog (id INTEGER,provider_id INTEGER,model_name TEXT,display_name TEXT,
capabilities TEXT,capability_status TEXT,enabled INTEGER,builtin INTEGER,rate_version TEXT,
input_rate REAL,output_rate REAL);
CREATE TABLE appsetting (key TEXT,value TEXT);
INSERT INTO modelprovider VALUES (3,'deepseek','https://api.deepseek.com/v1','source-secret-UNIQUE',
'["deepseek-flash","deepseek-v4-pro"]','deepseek-flash',0,1,1);
INSERT INTO modelprovider VALUES (4,'images','https://images.invalid/v1','image-secret-UNIQUE',
'["gpt-image-2"]','gpt-image-2',0,1,1);
INSERT INTO modelcatalog VALUES (4,3,'deepseek-v4-flash','Disabled','[]','untested',0,0,'r1',1,2);
INSERT INTO modelcatalog VALUES (6,3,'deepseek-v4-pro','Pro','[]','untested',1,0,'r1',1,2);
INSERT INTO modelcatalog VALUES (7,3,'deepseek-flash','Flash','[]','untested',1,0,'r1',1,2);
INSERT INTO modelcatalog VALUES (8,4,'gpt-image-2','Image','[]','untested',1,0,'r1',1,2);
INSERT INTO appsetting VALUES ('default_model_provider_id','3');
""")
def plan(self):
plan = model_plan(self.db)
plan["source_snapshot_sha256"] = "a" * 64
return plan
def test_reviewed_models_only_without_native_image_generation_guess(self):
plan = self.plan()
self.assertEqual(len(plan["models"]), 2)
self.assertEqual(plan["catalog_map"], {"6": model_id(3, "deepseek-v4-pro"), "7": model_id(3, "deepseek-flash")})
self.assertEqual(plan["default_gen_model_id"], model_id(3, "deepseek-flash"))
self.assertEqual({item["reason"] for item in plan["skipped"]},
{"source_configuration_disabled", "image_generation_has_no_native_model_type"})
for item in plan["models"]:
self.assertFalse(item["is_default"])
self.assertFalse(item["is_builtin"])
self.assertEqual(item["source_catalog"]["capability_status"], "untested")
self.assertIn("no_paid_probe_performed", item["compatibility"])
self.assertNotIn("secret-UNIQUE", json.dumps(plan))
def test_planner_reads_no_credentials_and_never_writes(self):
def authorize(action, first, second, database, trigger):
if action == sqlite3.SQLITE_READ and first == "modelprovider" and second == "api_key":
return sqlite3.SQLITE_DENY
return sqlite3.SQLITE_OK
self.db.set_authorizer(authorize)
self.db.execute("PRAGMA query_only=ON")
self.assertEqual(len(self.plan()["models"]), 2)
def test_source_runtime_fallback_is_explicit_not_catalog_guess(self):
result = resolve_agent_model(self.plan(), {"model_provider_id": None, "model_name": "deepseek-chat"})
self.assertEqual(result["effective_provider_id"], 3)
self.assertEqual(result["effective_model_name"], "deepseek-flash")
self.assertEqual(result["requested_model_name"], "deepseek-chat")
self.assertTrue(result["provider_fallback"])
self.assertTrue(result["model_fallback"])
direct = resolve_agent_model(self.plan(), {"model_provider_id": 3, "model_name": "deepseek-v4-pro"})
self.assertEqual(direct["target_model_id"], model_id(3, "deepseek-v4-pro"))
self.assertFalse(direct["model_fallback"])
def test_unsupported_effective_agent_model_does_not_silently_fall_back(self):
with self.assertRaisesRegex(ValueError, "no reviewed native mapping"):
resolve_agent_model(self.plan(), {"model_provider_id": 4, "model_name": "gpt-image-2"})
def test_global_default_uses_source_builtin_and_id_order_when_setting_absent(self):
self.db.execute("DELETE FROM appsetting")
self.assertEqual(self.plan()["default_provider_id"], 3)
self.db.execute("UPDATE modelprovider SET builtin=1 WHERE id=4")
plan = self.plan()
self.assertEqual(plan["default_provider_id"], 4)
self.assertIsNone(plan["default_gen_model_id"])
with self.assertRaisesRegex(ValueError, "no reviewed native mapping"):
resolve_agent_model(plan, {"model_provider_id": None})
def test_provider_default_and_first_configured_model_match_legacy_runtime(self):
self.db.execute("UPDATE modelprovider SET default_model='' WHERE id=3")
resolved = resolve_agent_model(self.plan(), {"model_provider_id": 3, "model_name": ""})
self.assertEqual(resolved["effective_model_name"], "deepseek-flash")
def test_credentials_read_only_for_imported_models(self):
def authorize(action, first, second, database, trigger):
return sqlite3.SQLITE_OK
self.db.set_authorizer(authorize)
credentials = read_credentials(self.db, self.plan())
self.assertEqual(set(credentials), {3})
self.assertEqual(credentials[3], "source-secret-UNIQUE")
def test_source_keychain_reference_needs_resolver(self):
self.db.execute("UPDATE modelprovider SET api_key='keychain:approved:provider-3' WHERE id=3")
with self.assertRaisesRegex(ValueError, "resolver"):
read_credentials(self.db, self.plan())
credentials = read_credentials(self.db, self.plan(), lambda value, provider_id: "resolved-in-memory")
self.assertEqual(credentials, {3: "resolved-in-memory"})
def test_no_plaintext_sql_or_credential_ledger_and_no_existing_model_updates(self):
plan = self.plan()
sql = model_sql(plan, read_credentials(self.db, plan), fake_encryption)
self.assertNotIn("source-secret-UNIQUE", sql)
self.assertNotIn("image-secret-UNIQUE", sql)
self.assertNotIn("UPDATE models", sql)
self.assertNotIn("ON CONFLICT", sql)
self.assertIn("item - 'stored_parameters'", sql)
self.assertIn("current_database() <> 'yawan_native_acceptance_20260915'", sql)
self.assertIn("Model ID collision without migration provenance", sql)
self.assertIn("existing_model.deleted_at IS NOT NULL", sql)
self.assertIn("mapped.source_snapshot IS DISTINCT FROM", sql)
self.assertIn("SET LOCAL log_min_error_statement = 'panic'", sql)
self.assertIn(LEDGER, sql)
self.assertNotIn("stored_parameters", json.dumps(plan))
def test_native_cipher_required_and_callback_errors_redacted(self):
plan = self.plan()
for encrypt in (lambda x: x, lambda x: "enc:v1:invalid", lambda x: "enc:v1:has padding=="):
with self.subTest(encrypt=encrypt), self.assertRaisesRegex(ValueError, "encryption failed"):
model_sql(plan, {3: "source-secret-UNIQUE"}, encrypt)
def broken(value):
raise RuntimeError(value)
try:
model_sql(plan, {3: "source-secret-UNIQUE"}, broken)
except ValueError as error:
self.assertNotIn("source-secret", str(error))
self.assertTrue(error.__suppress_context__)
else:
self.fail("encryption failure accepted")
def test_missing_secrets_and_snapshot_are_rejected(self):
plan = self.plan()
for secret in (None, "", "keychain:reference", "enc:v1:reference", "env:reference"):
with self.subTest(secret=secret), self.assertRaises(ValueError):
model_sql(plan, {3: secret}, fake_encryption)
del plan["source_snapshot_sha256"]
with self.assertRaisesRegex(ValueError, "fingerprint"):
model_sql(plan, {3: "in-memory"}, fake_encryption)
def test_tenant_protocol_and_organization_guards(self):
for tenant in (1, str(TARGET_TENANT), True):
with self.subTest(tenant=tenant), self.assertRaises(ValueError):
model_plan(self.db, tenant)
self.db.execute("UPDATE modelprovider SET organization_id=2 WHERE id=4")
with self.assertRaisesRegex(ValueError, "organization"):
self.plan()
def test_unreviewed_endpoint_and_model_explicitly_skipped(self):
self.db.execute("UPDATE modelprovider SET base_url='https://other.invalid/v1' WHERE id=3")
plan = self.plan()
self.assertEqual(plan["models"], [])
self.assertIn("model_protocol_not_reviewed", {s["reason"] for s in plan["skipped"]})
def test_private_url_never_enters_plan(self):
self.db.execute("UPDATE modelprovider SET base_url='https://example.invalid/v1?key=private' WHERE id=3")
with self.assertRaisesRegex(ValueError, "private components"):
self.plan()
def test_ids_and_input_mapping_are_deterministic(self):
self.assertEqual(self.plan(), self.plan())
self.assertNotEqual(model_id(3, "deepseek-flash"), model_id(4, "deepseek-flash"))
self.assertNotEqual(model_id(3, "deepseek-flash"), model_id(3, "deepseek-flash", 10000))
plan = copy.deepcopy(self.plan())
plan["provider_model_map"]["3:deepseek-flash"] = "wrong"
with self.assertRaisesRegex(ValueError, "mapping"):
resolve_agent_model(plan, {})
def test_sql_rejects_scope_and_native_configuration_tampering(self):
mutations = [lambda p: p.update(target_tenant_id=10000),
lambda p: p.update(target_database="production"),
lambda p: p["models"][0].update(is_default=True),
lambda p: p["models"][0].update(is_builtin=True),
lambda p: p["models"][0]["parameters"].update(api_key="must-not-be-in-plan")]
for mutation in mutations:
plan = copy.deepcopy(self.plan())
mutation(plan)
with self.subTest(mutation=mutation), self.assertRaises(ValueError):
model_sql(plan, {3: "in-memory"}, fake_encryption)
def test_verification_emits_only_secret_presence(self):
sql = model_verification_sql()
self.assertIn("LIKE 'enc:v1:%'", sql)
self.assertNotIn("row_to_json", sql)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,42 @@
import unittest
from weknora_resource_version_etl import make_plan, transaction, TARGET_DATABASE
class ResourceVersionMigrationTests(unittest.TestCase):
def setUp(self):
self.row = {"id": 1, "resource_type": "agent", "resource_id": 14, "owner_user_id": 3,
"organization_id": 1, "status": "rejected", "payload": {"name": "Old version", "rules": "kept"},
"review_note": "copy deleted", "created_at": "2026-09-10T01:00:00Z", "reviewed_at": ""}
self.source = {"forks": [], "reviews": [self.row], "source_sha256": "a" * 64}
self.target = {"database": TARGET_DATABASE, "agents": {"14": "agent-id"}, "identities": {"3": "owner-id"}}
def test_terminal_history_preserves_payload_without_restoring_copy(self):
first = make_plan(self.source, self.target)
self.assertEqual(first, make_plan(self.source, self.target))
self.assertEqual(first["rows"][0]["proposed_snapshot"]["rules"], "kept")
sql = transaction(first)
self.assertIn("ROLLBACK", sql)
self.assertNotIn("INSERT INTO custom_agents", sql)
self.assertNotIn("INSERT INTO users", sql)
def test_active_forks_and_pending_reviews_require_real_migration(self):
self.source["forks"] = [{"id": 1}]
with self.assertRaises(ValueError):
make_plan(self.source, self.target)
self.source["forks"] = []
self.row["status"] = "pending"
with self.assertRaises(ValueError):
make_plan(self.source, self.target)
def test_missing_owner_and_foreign_scope_rejected(self):
self.target["identities"] = {}
with self.assertRaises(ValueError):
make_plan(self.source, self.target)
self.target["identities"] = {"3": "owner-id"}
self.row["organization_id"] = 2
with self.assertRaises(ValueError):
make_plan(self.source, self.target)
if __name__ == "__main__":
unittest.main()
+74
View File
@@ -0,0 +1,74 @@
#!/usr/bin/env bash
set -euo pipefail
#
# 更新 Homebrew Formula 中的版本号和 sha256
#
# 用法:
# ./scripts/update-homebrew-formula.sh v0.2.0
#
# 会自动从 GitHub Releases 下载 .sha256 文件来填充 Formula。
# 在 CI 中被 release-lite workflow 调用。
#
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
VERSION="${1:?Usage: $0 <version> (e.g. v0.2.0)}"
VERSION_BARE="${VERSION#v}"
FORMULA="${ROOT_DIR}/Formula/weknora-lite.rb"
REPO="Tencent/WeKnora"
BASE_URL="https://github.com/${REPO}/releases/download/${VERSION}"
if [ ! -f "${FORMULA}" ]; then
echo "Error: Formula not found at ${FORMULA}"
exit 1
fi
echo "Updating Formula to version ${VERSION_BARE}..."
fetch_sha256() {
local file="$1"
local url="${BASE_URL}/${file}.sha256"
local sha
sha=$(curl -sSL "${url}" | awk '{print $1}')
if [ -z "${sha}" ] || [ "${#sha}" -ne 64 ]; then
echo "Error: Failed to fetch sha256 from ${url}" >&2
exit 1
fi
echo "${sha}"
}
SHA_DARWIN_ARM64=$(fetch_sha256 "WeKnora-lite_${VERSION}_darwin_arm64.tar.gz")
SHA_DARWIN_AMD64=$(fetch_sha256 "WeKnora-lite_${VERSION}_darwin_amd64.tar.gz")
SHA_LINUX_ARM64=$(fetch_sha256 "WeKnora-lite_${VERSION}_linux_arm64.tar.gz")
SHA_LINUX_AMD64=$(fetch_sha256 "WeKnora-lite_${VERSION}_linux_amd64.tar.gz")
echo " darwin_arm64 : ${SHA_DARWIN_ARM64}"
echo " darwin_amd64 : ${SHA_DARWIN_AMD64}"
echo " linux_arm64 : ${SHA_LINUX_ARM64}"
echo " linux_amd64 : ${SHA_LINUX_AMD64}"
# Use a temp file for portable sed
TMP=$(mktemp)
cp "${FORMULA}" "${TMP}"
# Update version
sed -i.bak "s/^ version \".*\"/ version \"${VERSION_BARE}\"/" "${TMP}"
# Update sha256 values in order of appearance.
# The formula has sha256 lines in this order:
# 1. darwin arm64
# 2. darwin amd64
# 3. linux arm64
# 4. linux amd64
awk -v s1="${SHA_DARWIN_ARM64}" \
-v s2="${SHA_DARWIN_AMD64}" \
-v s3="${SHA_LINUX_ARM64}" \
-v s4="${SHA_LINUX_AMD64}" \
'BEGIN{n=0} /sha256 "/{n++; if(n==1) sub(/"[^"]*"$/,"\"" s1 "\""); else if(n==2) sub(/"[^"]*"$/,"\"" s2 "\""); else if(n==3) sub(/"[^"]*"$/,"\"" s3 "\""); else if(n==4) sub(/"[^"]*"$/,"\"" s4 "\"")} {print}' \
"${TMP}" > "${FORMULA}"
rm -f "${TMP}" "${TMP}.bak"
echo "Formula updated: ${FORMULA}"
+62
View File
@@ -0,0 +1,62 @@
const assert = require('node:assert/strict')
const fs = require('node:fs')
const path = require('node:path')
const { execFileSync } = require('node:child_process')
const root = path.resolve(__dirname, '../..')
const credentialsPath = path.join(root, 'acceptance/super-admin-login.json')
const base = 'http://127.0.0.1:18080/api/v1'
const literal = value => `'${String(value).replaceAll("'", "''")}'`
const sql = statement => execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres', 'psql', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-v', 'ON_ERROR_STOP=1', '-At'], { input: statement, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim()
async function request(route, data, token) {
const response = await fetch(base + route, {
method: 'POST', headers: { 'Content-Type': 'application/json', ...(token ? { Authorization: `Bearer ${token}` } : {}) },
body: JSON.stringify(data), signal: AbortSignal.timeout(30000),
})
const body = await response.json()
assert.ok(response.ok && body.success !== false, `${route}: HTTP ${response.status}`)
return body
}
async function main() {
const desired = JSON.parse(fs.readFileSync(0, 'utf8'))
assert.match(desired.username, /^[a-zA-Z0-9_-]{2,50}$/)
assert.ok(typeof desired.password === 'string' && [...desired.password].length >= 3)
const credentials = JSON.parse(fs.readFileSync(credentialsPath, 'utf8'))
const oldLogin = await request('/auth/login', { email: credentials.email, password: credentials.password })
assert.equal(oldLogin.user.is_system_admin, true, 'Saved identity must already be a system administrator')
assert.equal(oldLogin.user.is_active, true)
const userID = oldLogin.user.id
assert.match(userID, /^[a-f0-9-]{36}$/)
assert.equal(sql(`SELECT count(*) FROM users WHERE id<>${literal(userID)} AND LOWER(username)=LOWER(${literal(desired.username)});`), '0', 'Requested account name already exists')
const backup = path.join(root, 'acceptance', `super-admin-login-before-${Date.now()}.json`)
fs.writeFileSync(backup, JSON.stringify(credentials, null, 2) + '\n', { flag: 'wx', mode: 0o600 })
// Keep the user ID and all ownership links; only change the login name.
sql(`BEGIN;
SELECT id FROM users WHERE id=${literal(userID)} AND is_system_admin=true AND is_active=true AND deleted_at IS NULL FOR UPDATE;
UPDATE users SET username=${literal(desired.username)},updated_at=now() WHERE id=${literal(userID)} AND is_system_admin=true AND is_active=true AND deleted_at IS NULL;
INSERT INTO audit_logs (tenant_id,actor_user_id,actor_role,action,target_type,target_id,target_user_id,outcome,details,created_at)
SELECT tenant_id,id,'admin','system.user_updated','user',id,id,'success',${literal(JSON.stringify({ username_before: oldLogin.user.username, username_after: desired.username, source: 'authorized_account_update' }))}::jsonb,now() FROM users WHERE id=${literal(userID)};
COMMIT;`)
const renamed = await request('/auth/login', { username: desired.username, password: credentials.password })
assert.equal(renamed.user.id, userID)
if (credentials.password !== desired.password) {
await request('/auth/change-password', { old_password: credentials.password, new_password: desired.password }, renamed.token)
}
const updated = { ...credentials, username: desired.username, password: desired.password }
const temporary = credentialsPath + '.next'
fs.writeFileSync(temporary, JSON.stringify(updated, null, 2) + '\n', { mode: 0o600 })
fs.renameSync(temporary, credentialsPath)
const current = await request('/auth/login', { username: desired.username, password: desired.password })
assert.equal(current.user.id, userID)
assert.equal(current.user.is_system_admin, true)
if (credentials.password !== desired.password) {
const revoked = await fetch(base + '/auth/me', { headers: { Authorization: `Bearer ${oldLogin.token}` }, signal: AbortSignal.timeout(15000) })
assert.equal(revoked.status, 401, 'Previous login must be revoked after password rotation')
}
console.log(JSON.stringify({ passed: true, username: current.user.username, identity_preserved: true, system_admin: true, password_rotated: credentials.password !== desired.password }))
}
main().catch(error => { console.error(error.message); process.exitCode = 1 })
+279
View File
@@ -0,0 +1,279 @@
const startedAt = new Date().toISOString()
const assert = require('node:assert/strict')
const fs = require('node:fs')
const crypto = require('node:crypto')
const { execFileSync } = require('node:child_process')
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
const base = (process.env.VERIFY_API_ORIGIN || 'http://127.0.0.1:18080') + '/api/v1'
const origin = process.env.VERIFY_FRONTEND_ORIGIN || 'http://127.0.0.1:15173'
const reportName = process.env.VERIFY_REPORT_NAME || 'access-control-live'
assert.match(reportName, /^[a-zA-Z0-9_-]+$/)
const prefix = `access-check-${Date.now()}`
const checks = [], cleanup = [], failures = []
let admin, member, browser
const check = (name, value) => { checks.push({name, passed:!!value}); assert.ok(value, name) }
const sql = statement => execFileSync('docker', ['--context','colima-yawan','exec','WeKnora-postgres','psql','-U','postgres','-d','yawan_native_acceptance_20260915','-v','ON_ERROR_STOP=1','-Atc',statement],{encoding:'utf8',stdio:['ignore','pipe','pipe']}).trim()
const literal = value => `'${String(value).replaceAll("'", "''")}'`
function validateResponse(route, method, result, status, expected, onUnexpectedSuccess) {
const accepted=(Array.isArray(expected)?expected:[expected]).includes(status)
if(!accepted&&status>=200&&status<300&&onUnexpectedSuccess)onUnexpectedSuccess(result)
assert.ok(accepted, `${method} ${route}: HTTP ${status}; ${result.message || result.error?.message || ''}`)
return result
}
function trackCreatedResource(collection, excludedIDs=[], actions=cleanup, remove=route=>api(route,'DELETE',undefined,admin,[200,202,204,404]), executeSQL=sql) {
return result=>{
const data=result.data
const id=collection==='/knowledge-bases' ? data?.target_id || data?.knowledge_base?.id
: collection==='/workbench/resources/flows' ? data?.personal_id : data?.id
assert.ok(typeof id==='string'&&id&&!excludedIDs.includes(id),'Unexpected creation must return a new resource ID')
actions.push(()=>remove(`${collection}/${encodeURIComponent(id)}?detach=true`))
if(collection==='/workbench/resources/flows'){
assert.ok(typeof data.id==='string'&&data.id&&data.id!==id,'Unexpected fork must return its separate metadata ID')
actions.push(()=>executeSQL(`BEGIN;
DELETE FROM workbench_resource_forks WHERE tenant_id=10059 AND kind='flows' AND id=${literal(data.id)} AND personal_id=${literal(id)};
DELETE FROM workbench_access_publications WHERE tenant_id=10059 AND kind='flows' AND resource_id=${literal(id)};
COMMIT;`))
}
}
}
async function finishVerification({browser,cleanup,failures,writeReport}) {
if(browser)try{await browser.close()}catch(error){failures.push('browser close: '+error.message)}
for(const action of [...cleanup].reverse())try{await action()}catch(error){failures.push('cleanup: '+error.message)}
await writeReport()
}
async function api(route, method='GET', body, identity=admin, expected=200, onUnexpectedSuccess) {
const form = body instanceof FormData
const response = await fetch(base+route,{method,headers:{...(form?{}:{'Content-Type':'application/json'}),...(identity?{Authorization:`Bearer ${identity.token}`,'X-Tenant-ID':'10059'}:{})},body:body===undefined?undefined:form?body:JSON.stringify(body),signal:AbortSignal.timeout(30000)})
const result = response.status===204 ? {} : await response.json()
return validateResponse(route,method,result,response.status,expected,onUnexpectedSuccess)
}
async function browserContext(identity, viewport={width:1440,height:1000}) {
const context = await browser.newContext({viewport})
await context.addInitScript(a=>{
const values={weknora_user:JSON.stringify({...a.user,tenant_id:String(a.user.tenant_id)}),weknora_tenant:JSON.stringify({...a.active_tenant,id:'10059'}),weknora_token:a.token,weknora_refresh_token:a.refresh_token,weknora_memberships:JSON.stringify(a.memberships)}
for(const [key,value] of Object.entries(values))localStorage.setItem(key,value)
localStorage.setItem('weknora:new-user-guide-done:v1','1')
},identity)
return context
}
async function main() {
const credentials=JSON.parse(fs.readFileSync(`${directory}/super-admin-login.json`,'utf8'))
admin=await api('/auth/login','POST',{email:credentials.email,password:credentials.password},null)
const role=(await api('/access/roles','POST',{name:prefix,description:'Disposable permission verification',permissions:['content.read']})).data
cleanup.push(()=>api(`/access/roles/${role.id}`,'DELETE'))
check('new_role_uses_explicit_permission_version',role.permission_version===2)
await api('/access/roles','POST',{name:prefix+'-invalid',permissions:['content.write']},admin,400,trackCreatedResource('/access/roles',[role.id]))
await api(`/access/roles/${role.id}`,'PUT',{...role,version:0},admin,409)
check('role_validation_and_optimistic_version',true)
const password=crypto.randomBytes(18).toString('base64url')
const userForm={username:prefix,email:`${prefix}@example.test`,password,role:'contributor',status:'active',custom_role_id:role.id}
const user=(await api('/access/users','POST',userForm)).data
let userDeleted=false
cleanup.push(async()=>{if(!userDeleted)await api(`/access/users/${user.id}`,'DELETE')})
member=await api('/auth/login','POST',{email:userForm.email,password},null)
const initialAccess=(await api('/access/me','GET',undefined,member)).data
check('created_user_can_login',initialAccess.restricted)
check('new_role_has_exact_read_only_permissions',JSON.stringify(initialAccess.permissions)===JSON.stringify(['content.read']))
await api('/access/users','GET',undefined,member,403)
await api('/access/resources','GET',undefined,member,403)
await api('/agents','GET',undefined,member,403)
await api('/agents/draft','POST',{model_id:'unavailable',description:'This request must not invoke a model.'},member,403)
await api('/knowledge-bases','GET',undefined,member,403)
await api('/shared-agents','GET',undefined,member,403)
await api('/shared-knowledge-bases','GET',undefined,member,403)
await api('/workbench/resources/content-tasks','POST',{name:prefix+'-blocked',data:{}},member,403,trackCreatedResource('/workbench/resources/content-tasks'))
await api('/workbench/resources/content-tasks','GET',undefined,member)
await api(`/access/users/${admin.user.id}`,'DELETE',undefined,admin,400)
await api(`/access/roles/${role.id}`,'DELETE',undefined,admin,409)
check('feature_and_admin_routes_enforced',true)
const {chromium}=require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
browser=await chromium.launch({headless:true,channel:'chrome'})
const restrictedContext=await browserContext(member), restrictedPage=await restrictedContext.newPage()
const pageFailures=[]
restrictedPage.on('pageerror',e=>pageFailures.push(e.message))
restrictedPage.on('response',r=>{if(r.status()>=400 && new URL(r.url()).pathname.startsWith('/api/'))pageFailures.push(`${r.status()} ${new URL(r.url()).pathname}`)})
await restrictedPage.goto(origin+'/platform/workbench/content?tab=tasks')
await restrictedPage.waitForLoadState('networkidle')
check('restricted_menu_hides_agents_and_access',await restrictedPage.locator('[data-guide="nav-agents"], [data-guide="nav-workbench/access"]').count()===0)
check('read_only_content_has_no_create_button',await restrictedPage.getByRole('button',{name:'创建内容任务',exact:true}).count()===0)
await restrictedPage.locator('[data-guide="user-menu"]').click()
const memberItems = await restrictedPage.locator('.user-dropdown .menu-item').allTextContents()
check('ordinary_user_menu_has_only_manual_and_logout',memberItems.length===2 && memberItems.some(text=>text.trim()==='操作手册') && !memberItems.some(text=>/设置|管理/.test(text)))
await restrictedPage.locator('.dropdown-user-header').click()
check('ordinary_user_identity_does_not_mount_settings',await restrictedPage.locator('.settings-overlay').count()===0)
await restrictedPage.screenshot({path:`${directory}/${reportName}-member-menu.png`,animations:'disabled'})
for(const section of ['general','userprofile','tenant','members','models','skills','runtime-queues']) {
await restrictedPage.goto(origin+'/platform/settings?section='+section)
await restrictedPage.waitForURL('**/platform/help')
check('ordinary_user_settings_route_blocked:'+section,await restrictedPage.locator('.settings-overlay,.settings-modal').count()===0)
}
await restrictedPage.goto(origin+'/platform/workbench/access')
await restrictedPage.waitForURL('**/platform/help')
check('direct_page_access_redirects',true)
check('real_member_help_hides_admin_chapters',await restrictedPage.locator('.help-section[id^="admin-"]').count()===0)
check('restricted_page_no_failed_background_requests: '+pageFailures.join(', '),pageFailures.length===0)
await restrictedContext.close()
const resourcePermissions=['content.read','content.write','flows.read','flows.write','flows.delete','agents.read','agents.write','agents.skills','knowledge.read','knowledge.write','digital-humans.read']
const updatedRole=(await api(`/access/roles/${role.id}`,'PUT',{...role,permissions:resourcePermissions})).data
const updatedAccess=(await api('/access/me','GET',undefined,member)).data
check('updated_role_keeps_explicit_permission_version',updatedRole.permission_version===2)
check('updated_role_does_not_expand_write_permissions',JSON.stringify([...updatedAccess.permissions].sort())===JSON.stringify([...resourcePermissions].sort()))
const moduleContext=await browserContext(member),modulePage=await moduleContext.newPage(),moduleFailures=[]
modulePage.on('pageerror',e=>moduleFailures.push(e.message))
modulePage.on('response',r=>{if(r.status()>=400 && new URL(r.url()).pathname.startsWith('/api/'))moduleFailures.push(`${r.status()} ${new URL(r.url()).pathname}`)})
for(const route of ['workbench/flows','digital-humans','agents','knowledge-bases']){
await modulePage.goto(origin+'/platform/'+route)
await modulePage.waitForLoadState('networkidle')
}
check('permitted_modules_without_chat_or_sharing_load: '+moduleFailures.join(', '),moduleFailures.length===0)
await moduleContext.close()
const agents=(await api('/agents')).data
const source=agents.find(a=>a.config?.model_id)
assert.ok(source,'Configured model fixture')
const agent=(await api('/agents','POST',{name:prefix+'-agent',config:{...source.config,knowledge_bases:[],knowledge_base_ids:[],kb_selection_mode:'none',skills:[],instruction_skill_ids:[],fusion_agent_ids:[],digital_human_id:''}},admin,201)).data
cleanup.push(()=>api(`/agents/${agent.id}?detach=true`,'DELETE'))
const flow=(await api('/workbench/resources/flows','POST',{name:prefix+'-flow',data:{nodes:[{id:'a',label:'A',agent_id:agent.id}],edges:[]}},admin,201)).data
cleanup.push(()=>api(`/workbench/resources/flows/${flow.id}?detach=true`,'DELETE'))
const form=new FormData();form.set('name',prefix+'-skill');form.set('file',new Blob(['# Review\nCheck structure and citations.'],{type:'text/markdown'}),'review.md')
const skill=(await api('/skills/instructions/import','POST',form,admin,201)).data
cleanup.push(()=>api(`/skills/instructions/${skill.id}?detach=true`,'DELETE'))
const human=(await api('/digital-humans','POST',{name:prefix+'-human',model_id:source.config.model_id},admin,201)).data
cleanup.push(()=>api(`/digital-humans/${human.id}?detach=true`,'DELETE'))
const kb=(await api('/knowledge-bases','POST',{name:prefix+'-knowledge',type:'document',embedding_model_id:'a2c3ce32-6192-46c8-8072-910abfe8afed',summary_model_id:source.config.model_id,indexing_strategy:{vector_enabled:true,keyword_enabled:true,wiki_enabled:false,graph_enabled:false}},admin,[200,201])).data
cleanup.push(()=>api(`/knowledge-bases/${kb.id}`,'DELETE',undefined,admin,[200,202]))
await api(`/agents/${agent.id}`,'PUT',{name:agent.name,config:{...agent.config,kb_selection_mode:'selected',knowledge_bases:[kb.id],instruction_skill_ids:[skill.id]}})
// A metadata-only fixture verifies document authorization without embedding or model calls.
const knowledgeID=crypto.randomUUID()
sql(`INSERT INTO knowledges (id,tenant_id,knowledge_base_id,type,title,source,parse_status,enable_status,created_at,updated_at) VALUES (${literal(knowledgeID)},10059,${literal(kb.id)},'manual',${literal(prefix+'-document')},'manual','completed','enabled',now(),now())`)
cleanup.push(()=>sql(`DELETE FROM knowledges WHERE id=${literal(knowledgeID)}`))
const fixtures=[{kind:'agents',id:agent.id,route:`/agents/${agent.id}`},{kind:'flows',id:flow.id,route:`/workbench/resources/flows/${flow.id}`},{kind:'skills',id:skill.id,route:`/skills/instructions/${skill.id}`},{kind:'digital-humans',id:human.id,route:`/digital-humans/${human.id}`},{kind:'knowledge',id:kb.id,route:`/knowledge-bases/${kb.id}`}]
const resources=fixtures.map(({kind,id})=>({kind,id}))
cleanup.push(async()=>{for(const table of ['workbench_access_grants','workbench_access_publications'])sql(`DELETE FROM ${table} WHERE tenant_id=10059 AND resource_id IN (${fixtures.map(f=>literal(f.id)).join(',')})`)})
async function grants(action){await api('/access/grants','POST',{resources,user_ids:['grant','revoke'].includes(action)?[user.id]:[],action})}
await grants('unpublish')
for(const fixture of fixtures)await api(fixture.route,'GET',undefined,member,[403,404])
await grants('grant')
for(const fixture of fixtures)await api(fixture.route,'GET',undefined,member)
await api(`/agents/${human.primary_agent_id}`,'GET',undefined,member)
check('five_resource_types_and_derived_agent_grant',true)
check('granted_document_batch_visible',(await api(`/knowledge/batch?ids=${knowledgeID}`,'GET',undefined,member)).data.some(k=>k.id===knowledgeID))
await api(`/knowledge/${knowledgeID}/download`,'GET',undefined,member,403)
await api(`/knowledge-bases/${kb.id}/faq/entries/export`,'GET',undefined,member,403)
check('grants_do_not_allow_document_download_or_export',true)
await api(`/agents/${agent.id}/copy`,'POST',{},member,403,trackCreatedResource('/agents',[agent.id]))
await api(`/knowledge-bases/${kb.id}/duplicate`,'POST',{},member,403,trackCreatedResource('/knowledge-bases',[kb.id]))
check('grants_do_not_allow_resource_duplication',true)
await api(`/workbench/resources/flows/${flow.id}`,'PUT',{name:'forbidden',version:flow.version,data:flow.data},member,[403,404])
await api(`/agents/${agent.id}`,'PUT',{name:'forbidden',config:agent.config},member,403)
await api(`/knowledge-bases/${kb.id}/knowledge/manual`,'POST',{title:prefix+'-forbidden',content:'must not be created'},member,403,trackCreatedResource('/knowledge'))
check('grants_do_not_transfer_ownership',true)
await grants('revoke')
for(const fixture of fixtures)await api(fixture.route,'GET',undefined,member,[403,404])
await api(`/agents/${human.primary_agent_id}`,'GET',undefined,member,[403,404])
check('revoked_document_batch_hidden',!(await api(`/knowledge/batch?ids=${knowledgeID}`,'GET',undefined,member)).data.some(k=>k.id===knowledgeID))
await grants('publish')
for(const fixture of fixtures)await api(fixture.route,'GET',undefined,member)
await api(`/agents/${human.primary_agent_id}`,'GET',undefined,member)
check('published_document_batch_visible',(await api(`/knowledge/batch?ids=${knowledgeID}`,'GET',undefined,member)).data.some(k=>k.id===knowledgeID))
await api(`/knowledge/${knowledgeID}/download`,'GET',undefined,member,403)
await api(`/knowledge-bases/${kb.id}/faq/entries/export`,'GET',undefined,member,403)
check('global_publication_does_not_allow_document_export',true)
await api(`/agents/${agent.id}/copy`,'POST',{},member,403,trackCreatedResource('/agents',[agent.id]))
await api(`/knowledge-bases/${kb.id}/duplicate`,'POST',{},member,403,trackCreatedResource('/knowledge-bases',[kb.id]))
await api(`/workbench/resources/flows/${flow.id}`,'DELETE',undefined,member,[403,404])
await api(`/skills/instructions/${skill.id}?detach=true`,'DELETE',undefined,member,[403,404])
await api(`/agents/${agent.id}`,'PUT',{name:'forbidden',config:agent.config},member,403)
await api(`/resource-versions/flows/${flow.id}/fork`,'POST',{},member,410,trackCreatedResource('/workbench/resources/flows',[flow.id]))
check('global_resources_deny_copy_mutation_and_retired_forks',true)
const personalAgent=(await api('/agents','POST',{name:prefix+'-member-agent',config:{...source.config,knowledge_bases:[],knowledge_base_ids:[],kb_selection_mode:'none',skills:[],instruction_skill_ids:[],fusion_agent_ids:[],digital_human_id:''}},member,201)).data
cleanup.push(()=>api(`/agents/${personalAgent.id}?detach=true`,'DELETE'))
await api('/skills/instructions/mount','POST',{agent_ids:[personalAgent.id],skill_ids:[skill.id],mode:'add'},member)
check('published_skill_mounts_on_members_own_agent',(await api(`/agents/${personalAgent.id}`,'GET',undefined,member)).data.config.instruction_skill_ids.includes(skill.id))
const personalFlow=(await api('/workbench/resources/flows','POST',{name:prefix+'-member-flow',data:{nodes:[{id:'shared',label:'Shared Agent',agent_id:agent.id}],edges:[]}},member,201)).data
cleanup.push(()=>api(`/workbench/resources/flows/${personalFlow.id}?detach=true`,'DELETE'))
check('published_agent_usable_in_members_own_canvas',personalFlow.owner_id===member.user.id)
const editedFlow=(await api(`/workbench/resources/flows/${personalFlow.id}`,'PUT',{name:prefix+'-admin-edited',version:personalFlow.version,data:personalFlow.data})).data
check('administrator_can_edit_members_canvas',editedFlow.name===prefix+'-admin-edited'&&editedFlow.owner_id===member.user.id)
const editedAgent=(await api(`/agents/${personalAgent.id}`,'PUT',{name:prefix+'-admin-agent',config:personalAgent.config})).data
check('administrator_can_edit_members_agent',editedAgent.name===prefix+'-admin-agent'&&editedAgent.created_by===member.user.id)
await api('/access/grants','POST',{resources:[{kind:'agents',id:agent.id},{kind:'knowledge',id:kb.id},{kind:'skills',id:skill.id}],user_ids:[],action:'unpublish'})
for(const fixture of fixtures.filter(item=>['agents','knowledge','skills'].includes(item.kind)))await api(fixture.route,'GET',undefined,member,[403,404])
const sharedFlowTask=(await api('/workbench/resources/content-tasks','POST',{name:prefix+'-shared-flow-task',data:{flow_id:flow.id,source_text:'第一天:识别主题;第二天:练习应用。'}},member,201)).data
cleanup.push(()=>api(`/workbench/resources/content-tasks/${sharedFlowTask.id}?detach=true`,'DELETE'))
check('member_can_create_task_using_canvas_private_dependencies',sharedFlowTask.owner_id===member.user.id&&sharedFlowTask.data.flow_id===flow.id&&sharedFlowTask.data.knowledge_base_ids.includes(kb.id))
await grants('unpublish')
for(const fixture of fixtures)await api(fixture.route,'GET',undefined,member,[403,404])
await api(`/agents/${human.primary_agent_id}`,'GET',undefined,member,[403,404])
check('unpublished_document_batch_hidden',!(await api(`/knowledge/batch?ids=${knowledgeID}`,'GET',undefined,member)).data.some(k=>k.id===knowledgeID))
check('revoke_publish_unpublish_effective',true)
const adminContext=await browserContext(admin),page=await adminContext.newPage(),adminErrors=[]
page.on('pageerror',e=>adminErrors.push(e.name))
page.on('response',r=>{if(r.status()>=400&&new URL(r.url()).pathname.startsWith('/api/'))adminErrors.push(`${r.status()} ${new URL(r.url()).pathname}`)})
await page.goto(origin+'/platform/workbench/access')
await page.getByRole('heading',{name:'访问控制',exact:true}).waitFor()
await page.getByText('角色权限',{exact:true}).click()
await page.getByRole('button',{name:'创建角色',exact:true}).click()
const catalog=(await api('/access/catalog')).data
const roleDrawer=page.locator('.t-drawer--open').filter({hasText:'创建角色'})
await roleDrawer.waitFor({state:'visible'})
// Groups mount asynchronously; wait for the rendered count instead of racing it.
await page.waitForFunction(expected=>document.querySelectorAll('.t-drawer--open .permission-group-heading').length>=expected,catalog.length,{timeout:20000}).catch(()=>{})
check('role_drawer_lists_every_sidebar_module',await roleDrawer.locator('.permission-group-heading').count()===catalog.length && catalog.length>=13)
const adminGroups=catalog.filter(group=>group.admin_only)
check('catalog_contains_administrator_only_groups',adminGroups.length>0)
for(const group of adminGroups){
const heading=page.getByText(group.name,{exact:true})
const panel=roleDrawer.locator('.t-collapse-panel').filter({has:heading})
const body=panel.locator('.t-collapse-panel__body')
if(!await body.isVisible())await panel.locator('.t-collapse-panel__header').click()
await body.locator('.admin-permission-note').waitFor({state:'visible'})
check('administrator_group_has_no_grant_checkboxes:'+group.id,await body.locator('input[type="checkbox"],[role="checkbox"],.t-checkbox').count()===0)
for(const permission of group.permissions)await body.locator('.permission-options').getByText(permission.name,{exact:true}).waitFor({state:'visible'})
}
await roleDrawer.locator('.admin-permission-note').first().scrollIntoViewIfNeeded()
await page.screenshot({path:`${directory}/${reportName}-roles-desktop.png`,animations:'disabled'})
if(process.env.VERIFY_PC_ONLY!=='1'){
await page.setViewportSize({width:390,height:844})
await roleDrawer.locator('.admin-permission-note').first().scrollIntoViewIfNeeded()
await page.screenshot({path:`${directory}/${reportName}-roles-mobile.png`,animations:'disabled'})
check('role_drawer_mobile_no_overflow',await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth+2))
}
await roleDrawer.getByRole('button',{name:'取消',exact:true}).click()
await page.setViewportSize({width:1440,height:1000})
await page.getByText('资源授权',{exact:true}).click()
await page.getByText(prefix+'-agent',{exact:true}).waitFor()
await page.screenshot({path:`${directory}/${reportName}-desktop.png`,animations:'disabled'})
if(process.env.VERIFY_PC_ONLY!=='1'){
await page.setViewportSize({width:390,height:844})
await page.screenshot({path:`${directory}/${reportName}-mobile.png`,animations:'disabled'})
check('access_mobile_no_horizontal_overflow',await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth+2))
}
check('admin_access_page_no_errors',adminErrors.length===0)
await adminContext.close()
await api(`/access/users/${user.id}`,'PUT',{...userForm,password:'',role:'viewer',custom_role_id:''})
const viewerAccess=(await api('/access/me','GET',undefined,member)).data
check('native_viewer_does_not_advertise_creation',!viewerAccess.permissions.includes('content.write')&&!viewerAccess.permissions.includes('digital-humans.write')&&!viewerAccess.permissions.includes('flows.write'))
await api('/workbench/resources/content-tasks','POST',{name:prefix+'-viewer-blocked',data:{}},member,403,trackCreatedResource('/workbench/resources/content-tasks'))
await api(`/access/users/${user.id}`,'PUT',{...userForm,password:'',status:'suspended'})
await api('/access/me','GET',undefined,member,[401,403])
check('suspend_revokes_existing_session',true)
// Simulate a migrated inactive account using only the disposable fixture.
sql(`UPDATE users SET is_active=false WHERE id=${literal(user.id)}`)
await api(`/access/users/${user.id}`,'PUT',{...userForm,password:'',status:'active'},admin,400)
const nextPassword=crypto.randomBytes(18).toString('base64url'),nextEmail=`updated-${prefix}@example.test`
await api(`/access/users/${user.id}`,'PUT',{...userForm,email:nextEmail,password:nextPassword})
member=await api('/auth/login','POST',{email:nextEmail,password:nextPassword},null)
check('migrated_account_activation_requires_reset_and_allows_email_edit',true)
check('deletion_fixture_has_membership_and_assignment',sql(`SELECT (SELECT count(*) FROM tenant_members WHERE tenant_id=10059 AND user_id=${literal(user.id)} AND deleted_at IS NULL), (SELECT count(*) FROM workbench_access_assignments WHERE tenant_id=10059 AND user_id=${literal(user.id)})`)==='1|1')
await api(`/access/users/${user.id}`,'DELETE');userDeleted=true
await api('/access/me','GET',undefined,member,[401,403])
check('delete_removes_account_from_list',!(await api('/access/users')).data.some(u=>u.id===user.id))
check('delete_removes_membership_and_assignments',sql(`SELECT (SELECT count(*) FROM tenant_members WHERE user_id=${literal(user.id)} AND deleted_at IS NULL), (SELECT count(*) FROM tenant_members WHERE tenant_id=10059 AND user_id=${literal(user.id)} AND deleted_at IS NOT NULL), (SELECT count(*) FROM workbench_access_assignments WHERE user_id=${literal(user.id)})`)==='0|1|0')
}
if(require.main===module)main().catch(e=>{failures.push(e.message);process.exitCode=1}).finally(()=>finishVerification({browser,cleanup,failures,writeReport:()=>{
fs.writeFileSync(`${directory}/${reportName}.json`,JSON.stringify({started_at:startedAt,time:new Date().toISOString(),origin,api_origin:new URL(base).origin,checks,failures,passed:failures.length===0},null,2))
console.log(JSON.stringify({checks,failures,passed:failures.length===0}))
if(failures.length)process.exitCode=1
}}))
module.exports={validateResponse,trackCreatedResource,finishVerification}
+77
View File
@@ -0,0 +1,77 @@
const assert = require('node:assert/strict')
const fs = require('node:fs')
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
const origin = process.env.VERIFY_FRONTEND_ORIGIN || 'https://agent.pducv.cn'
const checks = [], failures = []
let browser
const check = (name, result) => { checks.push({name, passed: !!result}); assert.ok(result, name) }
async function main() {
const credentials = JSON.parse(fs.readFileSync(`${directory}/super-admin-login.json`, 'utf8'))
const response = await fetch(`${origin}/api/v1/auth/login`, {
method: 'POST', headers: {'Content-Type': 'application/json'},
body: JSON.stringify({email: credentials.email, password: credentials.password}),
signal: AbortSignal.timeout(30000),
})
assert.equal(response.status, 200)
const auth = await response.json()
browser = await chromium.launch({headless: true, channel: 'chrome'})
const context = await browser.newContext({viewport: {width: 1440, height: 1000}})
await context.addInitScript(a => {
const values = {
weknora_user: JSON.stringify({...a.user, tenant_id: String(a.user.tenant_id)}),
weknora_tenant: JSON.stringify({...a.active_tenant, id: '10059'}),
weknora_token: a.token, weknora_refresh_token: a.refresh_token,
weknora_memberships: JSON.stringify(a.memberships),
}
for (const [key, value] of Object.entries(values)) localStorage.setItem(key, value)
localStorage.setItem('weknora:new-user-guide-done:v1', '1')
}, auth)
const page = await context.newPage(), errors = [], writes = []
page.on('pageerror', e => errors.push(e.name))
page.on('request', request => {
if (request.method() === 'POST') writes.push(new URL(request.url()).pathname)
})
await page.goto(`${origin}/platform/agents`)
await page.getByRole('button', {name: 'AI 创建', exact: true}).waitFor()
await page.getByRole('button', {name: '手动创建', exact: true}).click()
await page.locator('.settings-modal').waitFor()
check('manual_creation_opens_editor_without_ai', !writes.includes('/api/v1/agents/draft'))
await page.screenshot({path: `${directory}/public-agent-manual-desktop.png`, animations: 'disabled'})
await page.locator('.settings-modal .close-btn').click()
await page.locator('.settings-modal').waitFor({state: 'hidden'})
await page.getByRole('button', {name: 'AI 创建', exact: true}).click()
const dialog = page.locator('.t-dialog').filter({hasText: 'AI 创建 Agent'})
await dialog.locator('textarea').fill('创建一个课程规划 Agent,面向新学员整理连续课程框架,每天衔接前一天的核心概念,并检查主题重复。只生成配置草稿。')
const generate = dialog.getByRole('button', {name: '生成配置', exact: true})
await page.setViewportSize({width: 390, height: 844})
const draftResponse = page.waitForResponse(r => new URL(r.url()).pathname === '/api/v1/agents/draft', {timeout: 120000})
await generate.click()
await dialog.locator('progress').waitFor()
check('ai_draft_has_visible_progress', await dialog.locator('progress').isVisible())
await page.screenshot({path: `${directory}/public-agent-ai-progress-mobile.png`, animations: 'disabled'})
check('ai_dialog_mobile_has_no_horizontal_overflow', await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 2))
const result = await draftResponse
check(`configured_model_generates_real_draft_http_${result.status()}`, result.status() === 200)
const draft = (await result.json()).data
check('draft_has_no_persisted_resource_id', !!draft?.name && !draft.id)
await page.locator('.settings-modal').waitFor()
const name = page.locator('[data-guide="agent-create-name"] input')
await name.waitFor()
await page.waitForFunction(expected => document.querySelector('[data-guide="agent-create-name"] input')?.value === expected, draft.name)
check('generated_draft_is_editable_before_saving', await name.inputValue() === draft.name && await name.isEditable())
await page.screenshot({path: `${directory}/public-agent-ai-draft-mobile.png`, animations: 'disabled'})
await page.setViewportSize({width: 1440, height: 1000})
await page.screenshot({path: `${directory}/public-agent-ai-draft-desktop.png`, animations: 'disabled'})
check('ai_draft_does_not_auto_create_agent', !writes.includes('/api/v1/agents'))
check('agent_creation_has_no_javascript_errors', errors.length === 0)
}
main().catch(error => {failures.push(error.message); process.exitCode = 1}).finally(async () => {
if (browser) await browser.close()
const report = {checked_at: new Date().toISOString(), origin, checks, failures, passed: failures.length === 0}
fs.writeFileSync(`${directory}/public-agent-creation.json`, JSON.stringify(report, null, 2))
console.log(JSON.stringify(report))
})
+120
View File
@@ -0,0 +1,120 @@
const assert = require('node:assert/strict')
const fs = require('node:fs')
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
const origin = process.env.VERIFY_FRONTEND_ORIGIN || 'http://127.0.0.1:15173'
const apiOrigin = process.env.VERIFY_API_ORIGIN || 'http://127.0.0.1:18080'
const reportName = process.env.VERIFY_REPORT_NAME || 'agent-creation-ui'
assert.match(reportName, /^[a-zA-Z0-9_-]+$/)
const startedAt = new Date().toISOString()
// Draft AI output is a fixture; other reads use the running local API. No resources are saved.
async function main() {
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
const credentials = JSON.parse(fs.readFileSync(`${directory}/super-admin-login.json`, 'utf8'))
const response = await fetch(`${apiOrigin}/api/v1/auth/login`, {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ email: credentials.email, password: credentials.password }),
})
assert.equal(response.status, 200)
const auth = await response.json()
const browser = await chromium.launch({ headless: true, channel: 'chrome' })
const checks = [], errors = [], failures = [], requests = [], failedRequests = [], slowRequests = []
const pendingRequests = new Map()
let finalPath = ''
const check = (name, passed) => { checks.push({ name, passed: !!passed }); assert.ok(passed, name) }
try {
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } })
await context.addInitScript(a => {
const entries = {
weknora_user: JSON.stringify({ ...a.user, tenant_id: String(a.user.tenant_id) }),
weknora_tenant: JSON.stringify({ ...a.active_tenant, id: '10059' }),
weknora_token: a.token, weknora_refresh_token: a.refresh_token,
weknora_memberships: JSON.stringify(a.memberships),
'weknora:new-user-guide-done:v1': '1',
}
for (const [key, value] of Object.entries(entries)) localStorage.setItem(key, value)
}, auth)
let releaseDraft, draftRequest
await context.route('**/api/v1/agents/draft', route => new Promise(resolve => {
draftRequest = route.request().postDataJSON()
releaseDraft = async () => {
await route.fulfill({ status: 200, json: { success: true, data: {
name: '课程框架规划助手', description: '按用户目标整理课程结构,确保前后衔接。', avatar: '',
config: { role: 'training_map', agent_type: 'custom', agent_mode: 'quick-answer', model_id: draftRequest.model_id, system_prompt: '规划课程框架并说明每一天的前置条件。', kb_selection_mode: 'none', knowledge_bases: [], mcp_selection_mode: 'none', skills_selection_mode: 'none', selected_skills: [], instruction_skill_ids: [] },
} } })
resolve()
}
}))
const page = await context.newPage()
page.on('request', request => {
requests.push({ path: new URL(request.url()).pathname, method: request.method() })
pendingRequests.set(request, Date.now())
})
page.on('requestfinished', request => {
const started = pendingRequests.get(request)
pendingRequests.delete(request)
const duration = started === undefined ? 0 : Date.now() - started
if (duration >= 3000) slowRequests.push({ path: new URL(request.url()).pathname, duration_ms: duration })
})
page.on('requestfailed', request => {
pendingRequests.delete(request)
const failure = { path: new URL(request.url()).pathname, error: request.failure()?.errorText || '' }
failedRequests.push(failure)
if (failure.error !== 'net::ERR_ABORTED') errors.push(failure)
})
page.on('pageerror', error => errors.push({ stage: checks.at(-1)?.name, error: error.stack || error.message }))
await page.goto(`${origin}/platform/agents`)
const ai = page.getByRole('button', { name: 'AI 创建', exact: true })
await ai.waitFor()
check('manual_and_ai_creation_visible', await page.getByRole('button', { name: '手动创建', exact: true }).isVisible())
await page.locator('.agent-installations').first().waitFor()
check('installation_status_visible_on_agent_cards', await page.locator('.agent-installations').count() > 0)
await page.waitForLoadState('networkidle', { timeout: 15000 })
check('hidden_agent_editor_not_requested', !requests.some(item => /\/AgentEditorModal-[^/]+\.js$/.test(item.path)))
check('hidden_draft_dialog_not_requested', !requests.some(item => /\/AgentDraftDialog-[^/]+\.js$/.test(item.path)))
check('retired_resource_versions_menu_absent', await page.locator('.sidebar').getByText('资源版本', { exact: true }).count() === 0)
await page.screenshot({ path: `${directory}/agent-creation-list-desktop.png`, animations: 'disabled' })
await ai.click()
const dialog = page.locator('.t-dialog:visible')
await dialog.getByPlaceholder('例如:面向课程学员,负责将每日课程框架整理为有前后衔接的讲稿,重点解释概念和案例。').fill('请创建负责课程规划与每日框架衔接的导图助手。')
await dialog.getByRole('button', { name: '生成配置', exact: true }).click()
await dialog.locator('progress').waitFor()
check('ai_generation_progress_visible', await dialog.locator('progress').isVisible())
check('ai_double_submit_prevented', await dialog.getByRole('button', { name: /生成配置/ }).isDisabled())
await page.screenshot({ path: `${directory}/agent-draft-progress-desktop.png`, animations: 'disabled' })
await page.setViewportSize({ width: 390, height: 844 })
check('draft_dialog_fits_mobile', await dialog.evaluate(element => { const rect = element.getBoundingClientRect(); return rect.left >= 0 && rect.right <= innerWidth + 2 }))
await page.screenshot({ path: `${directory}/agent-draft-progress-mobile.png`, animations: 'disabled' })
await releaseDraft()
const name = page.locator('[data-guide="agent-create-name"] input')
await name.waitFor()
await page.waitForFunction(() => document.querySelector('[data-guide="agent-create-name"] input')?.value === '课程框架规划助手')
check('ai_draft_loaded_into_editable_editor', await name.inputValue() === '课程框架规划助手')
check('draft_request_preserves_requirement', draftRequest.requirement === '请创建负责课程规划与每日框架衔接的导图助手。')
await page.screenshot({ path: `${directory}/agent-draft-editor-mobile.png`, animations: 'disabled' })
// Restore normal browser caching after the one mocked AI request.
await context.unroute('**/api/v1/agents/draft')
await page.goto(`${origin}/platform/workbench/resource-versions`)
await ai.waitFor()
check('retired_route_redirects_to_agents', new URL(page.url()).pathname === '/platform/agents')
await page.getByRole('button', { name: '手动创建', exact: true }).click()
await name.waitFor()
check('manual_creation_does_not_reuse_ai_draft', await name.inputValue() !== '课程框架规划助手')
check('no_business_resources_saved', !requests.some(item => ['POST', 'PUT', 'PATCH', 'DELETE'].includes(item.method) && item.path.startsWith('/api/') && !item.path.startsWith('/api/v1/auth/') && item.path !== '/api/v1/agents/draft'))
check('no_vue_runtime_errors', errors.length === 0)
} catch (error) {
failures.push(error.message)
const pages = browser.contexts().flatMap(context => context.pages())
await pages[0]?.screenshot({ path: `${directory}/agent-creation-ui-failure.png` })
} finally {
finalPath = browser.contexts().flatMap(context => context.pages()).map(page => new URL(page.url()).pathname).join(',')
const diagnostics = { path: finalPath, slow_requests: slowRequests, failed_requests: failedRequests,
pending_requests: [...pendingRequests].map(([request, started]) => ({ path: new URL(request.url()).pathname, duration_ms: Date.now() - started })) }
await browser.close()
const report = { started_at: startedAt, checked_at: new Date().toISOString(), origin, checks, errors, failures, diagnostics, ai_output_mocked: true, passed: failures.length === 0 && errors.length === 0 && checks.every(item => item.passed) }
fs.writeFileSync(`${directory}/${reportName}.json`, JSON.stringify(report, null, 2))
console.log(JSON.stringify(report))
if (!report.passed) process.exitCode = 1
}
}
main().catch(error => { console.error(error.message); process.exitCode = 1 })
+97
View File
@@ -0,0 +1,97 @@
const assert = require('node:assert/strict')
const fs = require('node:fs')
const crypto = require('node:crypto')
const { execFileSync } = require('node:child_process')
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
const base = 'http://127.0.0.1:18080/api/v1'
const prefix = `fusion-check-${Date.now()}`
const checks = [], failures = [], users = [], roles = [], agents = []
let admin, browser
const literal = value => `'${String(value).replaceAll("'", "''")}'`
const sql = statement => execFileSync('docker', ['--context', 'colima-yawan', 'exec', 'WeKnora-postgres', 'psql', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-v', 'ON_ERROR_STOP=1', '-Atc', statement], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim()
const check = (name, value) => { checks.push({ name, passed: !!value }); assert.ok(value, name) }
async function api(route, method = 'GET', body, identity = admin, expected = 200) {
const response = await fetch(base + route, { method, headers: { 'Content-Type': 'application/json', ...(identity ? { Authorization: `Bearer ${identity.token}`, 'X-Tenant-ID': '10059' } : {}) }, body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(15000) })
const result = await response.json()
assert.ok((Array.isArray(expected) ? expected : [expected]).includes(response.status), `${method} ${route}: ${response.status} ${result.message || result.error?.message || ''}`)
return result
}
async function main() {
const credentials = JSON.parse(fs.readFileSync(`${directory}/super-admin-login.json`, 'utf8'))
admin = await api('/auth/login', 'POST', { email: credentials.email, password: credentials.password }, null)
const role = (await api('/access/roles', 'POST', { name: prefix, permissions: ['agents.read', 'agents.write'] })).data
roles.push(role.id)
async function member(label) {
const password = crypto.randomBytes(18).toString('base64url'), email = `${prefix}-${label}@example.test`
const user = (await api('/access/users', 'POST', { username: `${prefix}-${label}`, email, password, role: 'contributor', status: 'active', custom_role_id: role.id })).data
users.push(user.id)
return api('/auth/login', 'POST', { email, password }, null)
}
const alice = await member('alice'), bob = await member('bob')
async function agent(label, identity, role = 'production') {
const row = (await api('/agents', 'POST', { name: `${prefix}-${label}`, config: { role, agent_mode: 'quick-answer', model_id: '404e005b-83ae-5e2e-a3b3-999f9d5cd47e', kb_selection_mode: 'none', knowledge_bases: [], fusion_agent_ids: [], system_prompt: '保持原有身份与表达语气' } }, identity, 201)).data
agents.push(row.id)
return row
}
const primary = await agent('primary', alice), helper = await agent('private-helper', bob), second = await agent('second-helper', alice), qc = await agent('qc', alice, 'qc')
check('new_agents_are_private_by_default', [primary, helper, second, qc].every(row => row.visibility === 'private'))
const update = (ids, expected = 200) => api(`/agents/${primary.id}`, 'PUT', { name: primary.name, config: { ...primary.config, fusion_agent_ids: ids } }, alice, expected)
const denied = await update([helper.id], 400)
check('private_other_user_helper_is_rejected_without_name_leak', !JSON.stringify(denied).includes(helper.name))
await update([primary.id], 400)
await update([qc.id], 400)
await update([second.id, second.id], 400)
await update(Array.from({ length: 9 }, () => crypto.randomUUID()), 400)
check('self_wrong_role_duplicate_and_over_limit_rejected', true)
async function grant(action) { await api('/access/grants', 'POST', { resources: [{ kind: 'agents', id: helper.id }], user_ids: [alice.user.id], action }) }
await grant('grant')
await update([helper.id, second.id])
check('direct_grant_can_configure_multiple_helpers', JSON.stringify((await api(`/agents/${primary.id}`, 'GET', undefined, alice)).data.config.fusion_agent_ids) === JSON.stringify([helper.id, second.id]))
await grant('revoke')
await update([helper.id], 400)
await api(`/agents/${primary.id}/copy`, 'POST', undefined, alice, 400)
check('revoked_access_blocks_update_and_copy', true)
await grant('grant')
browser = await chromium.launch({ headless: true, channel: 'chrome' })
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } })
await context.addInitScript(auth => {
for (const [key, value] of Object.entries({ weknora_user: JSON.stringify({ ...auth.user, tenant_id: String(auth.user.tenant_id) }), weknora_tenant: JSON.stringify({ ...auth.active_tenant, id: '10059' }), weknora_token: auth.token, weknora_refresh_token: auth.refresh_token, weknora_memberships: JSON.stringify(auth.memberships), 'weknora:new-user-guide-done:v1': '1' })) localStorage.setItem(key, value)
}, alice)
const page = await context.newPage(), errors = []
page.on('pageerror', error => errors.push(error.message))
await page.goto(`http://127.0.0.1:15173/platform/agents?edit=${primary.id}`)
const row = page.locator('.setting-row').filter({ has: page.getByText('融合表达 Agent', { exact: true }) })
await row.waitFor()
await row.scrollIntoViewIfNeeded()
check('editor_displays_saved_multi_selection', await row.getByText(helper.name, { exact: true }).count() > 0 && await row.getByText(second.name, { exact: true }).count() > 0)
await page.screenshot({ path: `${directory}/agent-fusion-desktop.png`, animations: 'disabled' })
await page.setViewportSize({ width: 390, height: 844 })
await row.scrollIntoViewIfNeeded()
await page.screenshot({ path: `${directory}/agent-fusion-mobile.png`, animations: 'disabled' })
const mobileLayout = await page.evaluate(() => ({ width: innerWidth, documentWidth: document.documentElement.scrollWidth,
overflow: [...document.querySelectorAll('body *')].filter(element => { const rect = element.getBoundingClientRect(); return rect.width > 0 && rect.right > innerWidth + 2 }).slice(0, 15).map(element => ({ tag: element.tagName, className: element.className, right: element.getBoundingClientRect().right })) }))
if (mobileLayout.documentWidth > mobileLayout.width + 2) console.log(JSON.stringify(mobileLayout))
check('mobile_no_document_overflow', mobileLayout.documentWidth <= mobileLayout.width + 2)
check('browser_no_runtime_errors', errors.length === 0)
await api(`/agents/${helper.id}?detach=true`, 'DELETE', undefined, bob)
const after = (await api(`/agents/${primary.id}`, 'GET', undefined, alice)).data
check('deletion_unbinds_reference_and_preserves_other_helper', JSON.stringify(after.config.fusion_agent_ids) === JSON.stringify([second.id]))
}
main().catch(error => { failures.push(error.message); process.exitCode = 1 }).finally(async () => {
await browser?.close()
if (admin) {
for (const id of agents) try { await api(`/agents/${id}?detach=true`, 'DELETE', undefined, admin, [200, 404]) } catch (error) { failures.push(`cleanup agent: ${error.message}`) }
if (agents.length) {
const ids = agents.map(literal).join(',')
sql(`DELETE FROM workbench_access_grants WHERE tenant_id=10059 AND kind='agents' AND resource_id IN (${ids}); DELETE FROM workbench_access_publications WHERE tenant_id=10059 AND kind='agents' AND resource_id IN (${ids}); DELETE FROM custom_agents WHERE tenant_id=10059 AND id IN (${ids})`)
}
for (const id of users) try { await api(`/access/users/${id}`, 'DELETE') } catch (error) { failures.push(`cleanup user: ${error.message}`) }
for (const id of roles) try { await api(`/access/roles/${id}`, 'DELETE') } catch (error) { failures.push(`cleanup role: ${error.message}`) }
check('no_live_fixture_agents_remain', sql(`SELECT count(*) FROM custom_agents WHERE tenant_id=10059 AND name LIKE ${literal(prefix + '%')}`) === '0')
}
const report = { time: new Date().toISOString(), prefix, checks, failures, passed: failures.length === 0 }
fs.writeFileSync(`${directory}/agent-fusion-live.json`, JSON.stringify(report, null, 2))
console.log(JSON.stringify(report, null, 2))
if (failures.length) process.exitCode = 1
})
+52
View File
@@ -0,0 +1,52 @@
const assert = require('node:assert/strict')
const fs = require('node:fs')
const crypto = require('node:crypto')
const { execFileSync } = require('node:child_process')
const schema = `fusion_migration_test_${crypto.randomBytes(6).toString('hex')}`
const migration = fs.readFileSync(`${__dirname}/../migrations/versioned/000208_agent_fusions.up.sql`, 'utf8')
const checks = []
const sql = query => execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres',
'psql', '-X', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-At', '-v', 'ON_ERROR_STOP=1'],
{ input: `SET search_path TO ${schema}, public;\n${query}`, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim()
const check = (name, condition) => { assert.ok(condition, name); checks.push(name) }
const value = text => `'${JSON.stringify(text).replaceAll("'", "''")}'::jsonb`
function fixture(id, fusion, role = 'production', native = { role: 'production', sentinel: 'preserved' }) {
sql(`INSERT INTO custom_agents VALUES ('agent-${id}',10059,${value(native)},NULL);
INSERT INTO yawan_legacy_asset_mappings VALUES ('yawan-workbench',1,'agent',${id},10059,'agent-${id}',${value({ role, fusion_agent_ids: fusion })},'{}');`)
}
const run = () => sql(`BEGIN; ${migration} COMMIT;`)
try {
sql(`CREATE SCHEMA ${schema};
CREATE TABLE custom_agents (id TEXT PRIMARY KEY,tenant_id BIGINT,config JSONB,deleted_at TIMESTAMPTZ);
CREATE TABLE yawan_legacy_asset_mappings (source_system TEXT,source_organization_id BIGINT,source_kind TEXT,source_id BIGINT,target_tenant_id BIGINT,target_id TEXT,source_snapshot JSONB,native_snapshot JSONB);`)
fixture(1, [3, 2]); fixture(2, [1]); fixture(3, [])
fixture(4, [2], 'production', { role: 'production', fusion_agent_ids: [], sentinel: 'explicit-empty' })
fixture(5, [2], 'production', { role: 'qc', sentinel: 'native-duty-edited' })
run()
const data = JSON.parse(sql('SELECT jsonb_object_agg(id,config) FROM custom_agents').split('\n').at(-1))
check('ordered_direct_ids_preserved_without_recursive_cycle_expansion', JSON.stringify(data['agent-1'].fusion_agent_ids) === JSON.stringify(['agent-3', 'agent-2']) && JSON.stringify(data['agent-2'].fusion_agent_ids) === JSON.stringify(['agent-1']))
check('empty_selection_and_native_duty_edits_preserved', data['agent-4'].fusion_agent_ids.length === 0 && !('fusion_agent_ids' in data['agent-5']))
check('unrelated_configuration_preserved', data['agent-1'].sentinel === 'preserved')
sql(`UPDATE custom_agents SET config=config || '{"fusion_agent_ids":[],"sentinel":"owner-edit"}' WHERE id='agent-1';`)
const before = sql('SELECT jsonb_agg(to_jsonb(a) ORDER BY id) FROM custom_agents a')
run()
check('rerun_does_not_restore_user_removed_bindings', before === sql('SELECT jsonb_agg(to_jsonb(a) ORDER BY id) FROM custom_agents a'))
const cases = [[6, [999], 'missing-target'], [7, [2, 2], 'duplicate'], [8, [8], 'self'], [9, ['2'], 'invalid-id'], [10, [2], 'primary-wrong-role']]
for (const [id, fusions, label] of cases) {
fixture(id, fusions, label === 'primary-wrong-role' ? 'qc' : 'production')
const baseline = sql('SELECT jsonb_agg(to_jsonb(a) ORDER BY id) FROM custom_agents a')
let rejected = false
try { run() } catch { rejected = true }
check(`${label}_rejected_atomically`, rejected && baseline === sql('SELECT jsonb_agg(to_jsonb(a) ORDER BY id) FROM custom_agents a'))
sql(`DELETE FROM custom_agents WHERE id='agent-${id}'; DELETE FROM yawan_legacy_asset_mappings WHERE source_id=${id};`)
}
fixture(11, [2])
sql(`UPDATE yawan_legacy_asset_mappings SET source_snapshot=jsonb_set(source_snapshot,'{role}','"qc"') WHERE source_id=2;`)
let rejected = false
try { run() } catch { rejected = true }
check('nonproduction_target_rejected', rejected)
console.log(JSON.stringify({ passed: true, checks }, null, 2))
} finally {
sql(`DROP SCHEMA IF EXISTS ${schema} CASCADE;`)
}
+144
View File
@@ -0,0 +1,144 @@
#!/usr/bin/env python3
"""Verify migrated assets through acceptance HTTP without printing bodies."""
import argparse
import json
import secrets
import urllib.error
import urllib.request
import uuid
from pathlib import Path
from apply_weknora_assets import target_snapshot
from apply_weknora_identities import sql_run
from weknora_asset_etl import LEDGER, asset_id
from weknora_category_etl import literal, save_json
from weknora_identity_etl import TARGET_DATABASE, TARGET_TENANT
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--credentials', type=Path, required=True)
parser.add_argument('--out', type=Path, required=True)
args = parser.parse_args()
if args.out.exists():
parser.error('Report exists')
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
checks, fixture_user, fixture_agent = [], None, None
completed = False
def request(path, token=None, data=None, method=None, tenant=TARGET_TENANT):
headers = {'Content-Type': 'application/json'}
if token:
headers.update(Authorization='Bearer ' + token)
if tenant:
headers['X-Tenant-ID'] = str(tenant)
req = urllib.request.Request('http://127.0.0.1:18080' + path, headers=headers,
data=json.dumps(data).encode() if data is not None else None,
method=method)
try:
with opener.open(req, timeout=30) as response:
return response.status, json.load(response)
except urllib.error.HTTPError as error:
return error.code, {}
def check(name, passed):
checks.append({'check': name, 'passed': bool(passed)})
before = target_snapshot()
if before['database'] != TARGET_DATABASE:
raise RuntimeError('Acceptance database mismatch')
credentials = json.loads(args.credentials.read_text())
status, auth = request('/api/v1/auth/login', data={k: credentials[k] for k in ('email', 'password')})
if status != 200 or not auth.get('token'):
raise RuntimeError('Acceptance administrator login failed')
token = auth['token']
expected = json.loads(sql_run(f"SELECT json_agg(json_build_object('kind',source_kind,'source_id',source_id,'id',target_id,'native',native_snapshot)) FROM {LEDGER} WHERE target_tenant_id={TARGET_TENANT} AND source_kind IN ('agent','skill');"))
agents = [a for a in expected if a['kind'] == 'agent']
skills = [a for a in expected if a['kind'] == 'skill']
try:
status, response = request('/api/v1/agents', token)
listed = {a['id']: a for a in response.get('data', [])}
check('administrator_sees_all_15_agents', status == 200 and len(agents) == 15 and all(a['id'] in listed for a in agents))
for item in agents:
status, response = request('/api/v1/agents/' + item['id'], token)
native, current = item['native'], response.get('data', {})
check('agent_' + str(item['source_id']) + '_body_model_bindings_and_owner', status == 200 and
all(current.get(k) == native[k] for k in ('name', 'created_by', 'visibility', 'enabled')) and
all(current.get('config', {}).get(k, []) == native['config'][k]
for k in ('system_prompt', 'model_id', 'knowledge_bases', 'instruction_skill_ids')))
status, response = request('/api/v1/skills/instructions', token)
listed = {a['id']: a for a in response.get('data', [])}
check('administrator_sees_all_9_skills', status == 200 and len(skills) == 9 and all(s['id'] in listed for s in skills))
for item in skills:
status, response = request('/api/v1/skills/instructions/' + item['id'], token)
current, native = response.get('data', {}), item['native']
check('skill_' + str(item['source_id']) + '_body_and_owner', status == 200 and
all(current.get(k) == native[k] for k in ('name', 'instructions', 'created_by', 'visibility', 'enabled')))
for endpoint in ('/api/v1/agents', '/api/v1/skills/instructions'):
status, _ = request(endpoint)
check('anonymous_' + endpoint.rsplit('/', 1)[-1] + '_denied', status in (401, 403))
suffix = secrets.token_hex(6)
password = secrets.token_urlsafe(18) + 'aA1!'
email = 'asset-check-' + suffix + '@migration.invalid'
status, response = request('/api/v1/system/admin/users/create', token,
{'username': 'asset-check-' + suffix, 'email': email, 'password': password})
fixture_user = response.get('user', {}).get('id')
if status != 201 or not fixture_user or str(uuid.UUID(fixture_user)) != fixture_user:
raise RuntimeError(f'Temporary acceptance user provisioning failed (HTTP {status})')
sql_run(f"INSERT INTO tenant_members(user_id,tenant_id,role,status) VALUES ({literal(fixture_user)},{TARGET_TENANT},'contributor','active');")
status, member_auth = request('/api/v1/auth/login', data={'email': email, 'password': password})
if status != 200 or not member_auth.get('token'):
raise RuntimeError('Temporary acceptance user login failed')
member_token = member_auth['token']
status, response = request('/api/v1/agents', member_token)
visible = {a['id'] for a in response.get('data', [])}
check('member_sees_13_global_agents_and_no_private_agents', status == 200 and
all((a['id'] in visible) == (a['native']['visibility'] == 'global' and a['native']['enabled']) for a in agents))
status, response = request('/api/v1/skills/instructions', member_token)
visible = {a['id'] for a in response.get('data', [])}
check('member_sees_7_global_skills_and_no_private_skills', status == 200 and
all((s['id'] in visible) == (s['native']['visibility'] == 'global') for s in skills))
for kind, number, endpoint in (('agent', 22, '/api/v1/agents/'), ('skill', 5, '/api/v1/skills/instructions/'), ('skill', 9, '/api/v1/skills/instructions/')):
status, _ = request(endpoint + asset_id(kind, number), member_token)
check('private_' + kind + '_' + str(number) + '_direct_read_denied', status in (403, 404))
# A valid no-op payload avoids modifying the original even if a guard regresses.
first = agents[0]
status, _ = request('/api/v1/agents/' + first['id'], member_token,
{'name': first['native']['name'], 'config': first['native']['config']}, 'PUT')
check('member_cannot_update_global_original_agent', status in (403, 404))
status, _ = request('/api/v1/skills/instructions/mount', member_token,
{'agent_ids': [first['id']], 'skill_ids': [asset_id('skill', 1)], 'mode': 'add'})
check('member_cannot_mount_on_another_owners_agent', status in (403, 404))
status, response = request('/api/v1/agents', member_token, {
'name': 'asset-check-' + suffix, 'config': {'agent_mode': 'quick-answer', 'kb_selection_mode': 'none',
'skills_selection_mode': 'none', 'model_id': first['native']['config']['model_id']}})
fixture_agent = response.get('data', {}).get('id')
if status != 201 or not fixture_agent or str(uuid.UUID(fixture_agent)) != fixture_agent:
raise RuntimeError('Temporary Agent creation failed')
status, response = request('/api/v1/skills/instructions/mount', member_token,
{'agent_ids': [fixture_agent], 'skill_ids': [asset_id('skill', 1)], 'mode': 'add'})
check('member_can_mount_global_skill_on_own_agent', status == 200)
status, response = request('/api/v1/agents/' + fixture_agent, member_token)
check('mounted_global_skill_persisted', status == 200 and asset_id('skill', 1) in response.get('data', {}).get('config', {}).get('instruction_skill_ids', []))
status, _ = request('/api/v1/skills/instructions/mount', member_token,
{'agent_ids': [fixture_agent], 'skill_ids': [asset_id('skill', 5)], 'mode': 'add'})
check('member_cannot_mount_private_skill', status in (403, 404))
status, _ = request('/api/v1/agents/' + first['id'], member_token, tenant=10060)
check('unrelated_tenant_header_denied', status in (403, 404))
completed = True
finally:
if fixture_user:
# Scope cleanup by the exact user returned by this run, never by name prefixes.
sql_run(f"BEGIN; DELETE FROM custom_agents WHERE created_by={literal(fixture_user)}; DELETE FROM tenant_members WHERE user_id={literal(fixture_user)} AND tenant_id={TARGET_TENANT}; DELETE FROM auth_tokens WHERE user_id={literal(fixture_user)}; UPDATE users SET is_active=false,updated_at=now() WHERE id={literal(fixture_user)}; COMMIT;")
check('existing_assets_models_and_knowledge_unchanged', before == target_snapshot())
check('all_api_checks_completed', completed)
result = {'passed': all(c['passed'] for c in checks), 'checks': checks,
'fixture_user_deactivated': bool(fixture_user), 'fixture_agent_removed': bool(fixture_agent)}
save_json(args.out, result)
print(json.dumps(result))
return 0 if result['passed'] else 1
if __name__ == '__main__':
raise SystemExit(main())
+264
View File
@@ -0,0 +1,264 @@
#!/usr/bin/env python3
"""Run asset migration regressions in acceptance PostgreSQL and roll them back.
Source rules, skill bodies, and generated SQL stay in memory. Reports contain
only check names, booleans, counts, and the fixed acceptance scope.
"""
from __future__ import annotations
import argparse
import datetime
import json
from pathlib import Path
from apply_weknora_assets import build_plan, target_snapshot
from apply_weknora_identities import sql_run
from verify_identity_etl import rejected_case
from weknora_asset_etl import (BINDING_LEDGER, LEDGER, SOURCE, TARGET_DATABASE,
TARGET_TENANT, asset_sql, safe_asset_summary)
from weknora_category_etl import literal, save_json
from weknora_etl import consistent_snapshot
from weknora_identity_etl import IDENTITY_LEDGER
def supporting_snapshot():
tables = (LEDGER, BINDING_LEDGER, IDENTITY_LEDGER, "users", "tenant_members",
"tenants", "knowledge_bases")
fields = []
for table in tables:
fields.extend((literal(table),
f"(SELECT coalesce(json_agg(fingerprint ORDER BY fingerprint),'[]') "
f"FROM (SELECT md5(to_jsonb(r)::text) AS fingerprint FROM {table} r) s)"))
return json.loads(sql_run("SELECT json_build_object(" + ",".join(fields) + ");"))
def unchanged_check(name, pairs):
conditions = []
for table, baseline in pairs:
conditions.extend((f"EXISTS (SELECT * FROM {table} EXCEPT SELECT * FROM {baseline})",
f"EXISTS (SELECT * FROM {baseline} EXCEPT SELECT * FROM {table})"))
return f"""
DO $asset_unchanged$
BEGIN
IF {' OR '.join(conditions)} THEN
RAISE EXCEPTION 'Asset regression changed rows unexpectedly';
END IF;
END $asset_unchanged$;
SELECT json_build_object('check',{literal(name)},'passed',true);
"""
def regression_sql(plan, query):
agent, skill = plan["agents"][0]["target"], plan["skills"][0]["target"]
aid, sid = literal(agent["id"]), literal(skill["id"])
other_owner = literal(next(uid for uid in plan["identity_mapping"].values()
if uid != agent["created_by"]))
other_skill_owner = literal(next(uid for uid in plan["identity_mapping"].values()
if uid != skill["created_by"]))
asset_scope = f"source_system={literal(SOURCE)} AND source_organization_id=1"
agent_ledger = f"{asset_scope} AND source_kind='agent' AND target_id={aid}"
skill_ledger = f"{asset_scope} AND source_kind='skill' AND target_id={sid}"
binding = plan["bindings"][0]
binding_where = f"{asset_scope} AND source_binding_id={binding['source']['id']}"
kb = literal(next(iter(plan["kb_mapping"].values())))
model = literal(agent["config"]["model_id"])
owner = literal(agent["created_by"])
foreign_tenant = TARGET_TENANT + 1
cases = [
("mapped_agent_wrong_tenant_rejected", f"UPDATE custom_agents SET tenant_id={foreign_tenant} WHERE id={aid};",
"Mapped native asset removed; refusing recreation"),
("mapped_skill_wrong_tenant_rejected", f"UPDATE tenant_skill_catalog SET tenant_id={foreign_tenant} WHERE id={sid};",
"Mapped native asset removed; refusing recreation"),
("mapped_agent_wrong_owner_rejected", f"UPDATE custom_agents SET created_by={other_owner} WHERE id={aid};",
"Mapped native asset removed; refusing recreation"),
("mapped_skill_wrong_owner_rejected", f"UPDATE tenant_skill_catalog SET created_by={other_skill_owner} WHERE id={sid};",
"Mapped native asset removed; refusing recreation"),
("mapped_skill_kind_change_rejected", f"UPDATE tenant_skill_catalog SET instruction_only=false WHERE id={sid};",
"Mapped native asset removed; refusing recreation"),
("asset_ledger_wrong_tenant_rejected", f"UPDATE {LEDGER} SET target_tenant_id={foreign_tenant} WHERE {agent_ledger};",
"Asset durable mapping conflict"),
("asset_ledger_wrong_id_rejected", f"UPDATE {LEDGER} SET target_id='rollback-only-id' WHERE {agent_ledger};",
"Asset durable mapping conflict"),
("asset_ledger_source_change_rejected", f"UPDATE {LEDGER} SET source_snapshot='{{}}'::jsonb WHERE {skill_ledger};",
"Asset durable mapping conflict"),
("asset_ledger_native_plan_change_rejected", f"UPDATE {LEDGER} SET native_snapshot='{{}}'::jsonb WHERE {agent_ledger};",
"Asset durable mapping conflict"),
("asset_ledger_changed_duty_rejected", f"UPDATE {LEDGER} SET native_snapshot=jsonb_set(native_snapshot,'{{config,role}}','\"invalid-role\"'::jsonb) WHERE {agent_ledger};",
"Asset durable mapping conflict"),
("hard_deleted_agent_not_recreated", f"DELETE FROM custom_agents WHERE id={aid};",
"Mapped native asset removed; refusing recreation"),
("hard_deleted_skill_not_recreated", f"DELETE FROM tenant_skill_catalog WHERE id={sid};",
"Mapped native asset removed; refusing recreation"),
("agent_id_without_ledger_rejected", f"DELETE FROM {LEDGER} WHERE {agent_ledger};",
"Native Agent ID or owner/name collision"),
("skill_id_without_ledger_rejected", f"DELETE FROM {LEDGER} WHERE {skill_ledger};",
"Native skill ID or owner/name collision"),
("agent_owner_name_without_ledger_rejected", f"DELETE FROM {LEDGER} WHERE {agent_ledger}; "
f"UPDATE custom_agents SET id='rollback-only-agent-id' WHERE id={aid};",
"Native Agent ID or owner/name collision"),
("skill_owner_name_without_ledger_rejected", f"DELETE FROM {LEDGER} WHERE {skill_ledger}; "
f"UPDATE tenant_skill_catalog SET id='rollback-only-skill-id' WHERE id={sid};",
"Native skill ID or owner/name collision"),
("inactive_target_tenant_rejected", f"UPDATE tenants SET status='suspended' WHERE id={TARGET_TENANT};",
"Asset migration requires active target tenant"),
("owner_identity_mapping_change_rejected", f"UPDATE {IDENTITY_LEDGER} SET target_user_id='rollback-only-owner' "
f"WHERE {asset_scope} AND target_user_id={owner};",
"Asset owner durable identity or membership mismatch"),
("owner_membership_deletion_rejected", f"UPDATE tenant_members SET deleted_at=now() WHERE tenant_id={TARGET_TENANT} AND user_id={owner};",
"Asset owner durable identity or membership mismatch"),
("knowledge_base_wrong_tenant_rejected", f"UPDATE knowledge_bases SET tenant_id={foreign_tenant} WHERE id={kb};",
"Asset knowledge base mapping missing or outside tenant"),
("deleted_model_rejected", f"UPDATE models SET deleted_at=now() WHERE id={model};",
"Asset model mapping missing or not usable"),
("binding_wrong_tenant_rejected", f"UPDATE {BINDING_LEDGER} SET target_tenant_id={foreign_tenant} WHERE {binding_where};",
"Legacy skill binding provenance conflict"),
("binding_agent_change_rejected", f"UPDATE {BINDING_LEDGER} SET target_agent_id='rollback-only-agent' WHERE {binding_where};",
"Legacy skill binding provenance conflict"),
("binding_source_change_rejected", f"UPDATE {BINDING_LEDGER} SET source_snapshot='{{}}'::jsonb WHERE {binding_where};",
"Legacy skill binding provenance conflict"),
("binding_access_review_change_rejected", f"UPDATE {BINDING_LEDGER} SET access_review='[\"rollback-only-review\"]'::jsonb WHERE {binding_where};",
"Legacy skill binding provenance conflict"),
("binding_effective_flag_change_rejected", f"UPDATE {BINDING_LEDGER} SET effective=NOT effective WHERE {binding_where};",
"Legacy skill binding provenance conflict"),
]
if plan["prompt_templates"]:
template_id = plan["prompt_templates"][0]["id"]
cases.append(("template_source_change_rejected",
f"UPDATE {LEDGER} SET source_snapshot='{{}}'::jsonb WHERE {asset_scope} "
f"AND source_kind='template' AND source_id={template_id};",
"Legacy prompt template provenance conflict"))
tables = ("custom_agents", "tenant_skill_catalog", LEDGER, BINDING_LEDGER, "models")
pairs = [(table, f"_asset_before_{index}") for index, table in enumerate(tables)]
sql = """BEGIN ISOLATION LEVEL SERIALIZABLE;
SET LOCAL statement_timeout='30s';
SET LOCAL lock_timeout='5s';
SET LOCAL log_statement='none';
SET LOCAL log_min_error_statement='panic';
"""
for table, baseline in pairs:
sql += f"CREATE TEMP TABLE {baseline} ON COMMIT DROP AS SELECT * FROM {table};\n"
sql += query + unchanged_check("exact_repeat_and_existing_models_unchanged", pairs)
sql += f"""
SELECT json_build_object('check','all_39_source_bindings_retained_only_30_effective','passed',
(SELECT count(*)=39 AND count(*) FILTER (WHERE effective)=30 FROM {BINDING_LEDGER}
WHERE {asset_scope} AND target_tenant_id={TARGET_TENANT})
AND (SELECT count(*)=15 FROM {LEDGER} WHERE {asset_scope} AND source_kind='agent')
AND (SELECT count(*)=9 FROM {LEDGER} WHERE {asset_scope} AND source_kind='skill')
AND NOT EXISTS (
(SELECT b.target_agent_id,b.target_skill_id FROM {BINDING_LEDGER} b
WHERE b.source_system={literal(SOURCE)} AND b.source_organization_id=1 AND b.effective
EXCEPT
SELECT a.id,s.value FROM custom_agents a JOIN {LEDGER} l
ON l.source_kind='agent' AND l.target_id=a.id AND l.target_tenant_id=a.tenant_id
CROSS JOIN LATERAL jsonb_array_elements_text(a.config->'instruction_skill_ids') s(value)
WHERE l.source_system={literal(SOURCE)} AND l.source_organization_id=1)
UNION ALL
(SELECT a.id,s.value FROM custom_agents a JOIN {LEDGER} l
ON l.source_kind='agent' AND l.target_id=a.id AND l.target_tenant_id=a.tenant_id
CROSS JOIN LATERAL jsonb_array_elements_text(a.config->'instruction_skill_ids') s(value)
WHERE l.source_system={literal(SOURCE)} AND l.source_organization_id=1
EXCEPT
SELECT b.target_agent_id,b.target_skill_id FROM {BINDING_LEDGER} b
WHERE b.source_system={literal(SOURCE)} AND b.source_organization_id=1 AND b.effective)));
SELECT json_build_object('check','cross_owner_private_skills_blocked','passed',
(SELECT count(*)=9 FROM {BINDING_LEDGER} WHERE {asset_scope} AND NOT effective)
AND NOT EXISTS (SELECT 1 FROM {BINDING_LEDGER} b
JOIN custom_agents a ON a.id=b.target_agent_id AND a.tenant_id=b.target_tenant_id
JOIN tenant_skill_catalog s ON s.id=b.target_skill_id AND s.tenant_id=b.target_tenant_id
WHERE b.source_system={literal(SOURCE)} AND b.source_organization_id=1
AND b.effective IS DISTINCT FROM (s.visibility='global' OR s.created_by=a.created_by)));
"""
for name, mutation, expected in cases:
sql += rejected_case(name, mutation, expected, query)
sql += f"""
SAVEPOINT pre_duty_replay;
UPDATE {LEDGER} SET native_snapshot=native_snapshot #- '{{config,role}}' #- '{{config,content_tags}}' #- '{{config,fusion_agent_ids}}'
WHERE {asset_scope} AND source_kind='agent';
CREATE TEMP TABLE _pre_duty_ledger ON COMMIT DROP AS SELECT * FROM {LEDGER};
"""
sql += query + unchanged_check("pre_duty_ledger_replay_preserves_native_edits_and_audit",
[("custom_agents", pairs[0][1]), (LEDGER, "_pre_duty_ledger")])
sql += "ROLLBACK TO SAVEPOINT pre_duty_replay;\n"
duty_migration = (Path(__file__).resolve().parents[1] / "migrations/versioned/000205_agent_duties.up.sql").read_text()
sql += f"""
SAVEPOINT duty_backfill;
UPDATE custom_agents SET config=config - 'role' - 'content_tags' WHERE id={aid};
{duty_migration}
SELECT json_build_object('check','missing_duties_backfilled_without_other_changes','passed',
(SELECT a.config->>'role'=l.source_snapshot->>'role'
AND a.config->'content_tags'=l.source_snapshot->'content_tags'
AND (to_jsonb(a) #- '{{config,role}}' #- '{{config,content_tags}}') =
(to_jsonb(b) #- '{{config,role}}' #- '{{config,content_tags}}')
FROM custom_agents a JOIN {LEDGER} l ON l.target_id=a.id AND l.source_kind='agent'
JOIN {pairs[0][1]} b ON b.id=a.id WHERE a.id={aid}));
UPDATE custom_agents SET config=config || '{{"role":"general","content_tags":[]}}'::jsonb WHERE id={aid};
CREATE TEMP TABLE _duty_edited_agents ON COMMIT DROP AS SELECT * FROM custom_agents;
{duty_migration}
"""
sql += unchanged_check("duty_backfill_preserves_explicit_native_edits", [("custom_agents", "_duty_edited_agents")])
sql += "ROLLBACK TO SAVEPOINT duty_backfill;\n"
sql += f"""
UPDATE custom_agents SET name='Rollback-only native Agent edit',description='local edit',
enabled=false,visibility='private',
config=jsonb_set(jsonb_set(config,'{{system_prompt}}','\"Rollback-only native prompt\"'::jsonb),
'{{instruction_skill_ids}}','[]'::jsonb) || '{{"role":"general","content_tags":[]}}'::jsonb WHERE id={aid} AND tenant_id={TARGET_TENANT};
UPDATE tenant_skill_catalog SET name='Rollback-only native skill edit',description='local edit',
instructions='Rollback-only native instructions',version='77',enabled=false,visibility='private'
WHERE id={sid} AND tenant_id={TARGET_TENANT};
CREATE TEMP TABLE _edited_agents ON COMMIT DROP AS SELECT * FROM custom_agents;
CREATE TEMP TABLE _edited_skills ON COMMIT DROP AS SELECT * FROM tenant_skill_catalog;
"""
sql += query + unchanged_check("native_edits_disabling_and_binding_removal_preserved",
[("custom_agents", "_edited_agents"), ("tenant_skill_catalog", "_edited_skills"), *pairs[2:]])
sql += f"""
UPDATE custom_agents SET deleted_at=now() WHERE id={aid} AND tenant_id={TARGET_TENANT};
UPDATE tenant_skill_catalog SET deleted_at=now() WHERE id={sid} AND tenant_id={TARGET_TENANT};
CREATE TEMP TABLE _deleted_agents ON COMMIT DROP AS SELECT * FROM custom_agents;
CREATE TEMP TABLE _deleted_skills ON COMMIT DROP AS SELECT * FROM tenant_skill_catalog;
"""
sql += query + unchanged_check("soft_deleted_assets_not_revived",
[("custom_agents", "_deleted_agents"), ("tenant_skill_catalog", "_deleted_skills"), *pairs[2:]])
sql += "ROLLBACK;\n"
return sql, len(cases) + 8
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--sqlite", type=Path, required=True)
parser.add_argument("--kb-mapping", type=Path, required=True)
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
if args.out.exists():
parser.error("Output already exists")
manifest = json.loads(args.manifest.read_text())
with consistent_snapshot(args.sqlite) as (db, info):
plan = build_plan(db, args.kb_mapping)
expected = {k: v for k, v in manifest.items() if k != "source_snapshot_sha256"}
if safe_asset_summary(plan) != expected or info["source_mutated"]:
raise RuntimeError("Asset source differs from the reviewed manifest")
if (len(plan["agents"]), len(plan["skills"]), len(plan["bindings"]),
sum(b["effective"] for b in plan["bindings"])) != (15, 9, 39, 30):
raise RuntimeError("Asset counts differ from the reviewed acceptance batch")
plan["source_snapshot_sha256"] = info["sha256"]
query, expected_checks = regression_sql(plan, asset_sql(plan))
before, supporting_before = target_snapshot(), supporting_snapshot()
output = sql_run(query)
checks = [json.loads(line) for line in output.splitlines() if line.startswith("{")]
after, supporting_after = target_snapshot(), supporting_snapshot()
checks.extend((
{"check": "all_regressions_executed", "passed": len(checks) == expected_checks},
{"check": "all_test_changes_rolled_back", "passed": before == after and supporting_before == supporting_after},
{"check": "acceptance_scope", "passed": before["database"] == after["database"] == TARGET_DATABASE},
))
report = {"checked_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"database": TARGET_DATABASE, "tenant_id": TARGET_TENANT,
"committed": False, "source_mutated": info["source_mutated"],
"checks": checks, "passed": all(c["passed"] for c in checks)}
save_json(args.out, report)
print(json.dumps(report))
return 0 if report["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())
+272
View File
@@ -0,0 +1,272 @@
#!/usr/bin/env python3
"""Verify historical workbench imports through pinned local acceptance HTTP.
Run only after the business import and read-only protection deployment finish.
Reports contain only boolean checks and counts, never resource or login bodies.
"""
from __future__ import annotations
import argparse
from collections import Counter
import json
import secrets
import urllib.error
import urllib.request
import uuid
from pathlib import Path
from apply_weknora_business import target_snapshot
from apply_weknora_identities import sql_run
from weknora_business_etl import LEDGER, RESOURCE_KINDS, SOURCE
from weknora_category_etl import literal, save_json
from weknora_identity_etl import TARGET_DATABASE, TARGET_TENANT
BASE_URL = 'http://127.0.0.1:18080'
ROOT = '/api/v1/workbench/resources/'
EXPECTED_COUNTS = {'canvas': 3, 'trainingmap': 8, 'campaign': 2}
ACTIONS = {
'flows': ('generate',), # Flows have no executable action in the native API.
'training-maps': ('generate', 'revise', 'generate-day', 'confirm-day', 'confirm'),
'content-tasks': ('decompose', 'generate', 'revise', 'accept', 'skip', 'review', 'compose'),
}
READ_ONLY_TEXT = '\u53ea\u8bfb\u5386\u53f2\u8d44\u6599'
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--credentials', type=Path, required=True)
parser.add_argument('--out', type=Path, required=True)
args = parser.parse_args()
if args.out.exists():
parser.error('Report exists')
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
checks, counts = {}, Counter()
before, fixture_user, fixture_tenant = None, None, None
def check(name, passed):
checks[name] = checks.get(name, True) and bool(passed)
counts[name] += 1
return bool(passed)
def require(name, passed):
if not check(name, passed):
raise RuntimeError('Acceptance precondition failed')
def request(path, token=None, data=None, method=None, tenant=TARGET_TENANT):
headers = {'Content-Type': 'application/json'}
if tenant is not None:
headers['X-Tenant-ID'] = str(tenant)
if token:
headers['Authorization'] = 'Bearer ' + token
body = json.dumps(data, ensure_ascii=False).encode() if data is not None else None
if body is not None and len(body) > 2 * 1024 * 1024:
raise RuntimeError('Verification payload exceeds the native request limit')
req = urllib.request.Request(BASE_URL + path, headers=headers, data=body, method=method)
counts['http_requests'] += 1
try:
response = opener.open(req, timeout=30)
except urllib.error.HTTPError as error:
response = error
with response:
payload = json.load(response)
if not isinstance(payload, dict):
raise RuntimeError('Unexpected acceptance response shape')
return response.status, payload
def listed(kind, token):
result = {}
for page in range(1, 1001):
status, body = request(ROOT + kind + f'?page={page}&page_size=100', token)
if status != 200 or not isinstance(body.get('data'), list):
return False, result
rows, total = body['data'], body.get('total')
if type(total) is not int or total < 0:
return False, result
old_count = len(result)
for row in rows:
result[row['id']] = row
if len(result) >= total:
return len(result) == total, result
if len(result) == old_count:
return False, result
return False, result
def denied(name, path, token, data=None, method=None, tenant=TARGET_TENANT,
statuses=(403, 404), readonly=False):
status, body = request(path, token, data, method, tenant)
passed = status in statuses and (not readonly or READ_ONLY_TEXT in body.get('message', ''))
check(name, passed)
# A successful mutation ends the run immediately; never attempt repairs.
if method in ('PUT', 'DELETE', 'POST') and 200 <= status < 300:
raise RuntimeError('A protected mutation unexpectedly succeeded')
try:
before = target_snapshot()
require('pinned_acceptance_database', before['database'] == TARGET_DATABASE)
expected = json.loads(sql_run(
f"SELECT coalesce(json_agg(json_build_object('kind',source_kind,'id',target_id,"
f"'native',native_snapshot) ORDER BY source_kind,source_id),'[]') FROM {LEDGER} "
f"WHERE source_system={literal(SOURCE)} AND source_organization_id=1 "
f"AND target_tenant_id={TARGET_TENANT} AND source_kind IN ('canvas','trainingmap','campaign');"))
require('exact_13_migrated_resources_in_ledger',
Counter(item['kind'] for item in expected) == EXPECTED_COUNTS
and len({item['id'] for item in expected}) == 13)
for source_kind, count in EXPECTED_COUNTS.items():
counts['migrated_' + RESOURCE_KINDS[source_kind]] = count
require('all_13_native_snapshots_are_read_only', all(
item['native']['data'].get('legacy_import', {}).get('read_only') is True
and item['native']['tenant_id'] == TARGET_TENANT
and item['native']['id'] == item['id']
and item['native']['kind'] == RESOURCE_KINDS[item['kind']]
for item in expected))
credentials = json.loads(args.credentials.read_text())
status, auth = request('/api/v1/auth/login', data={k: credentials[k] for k in ('email', 'password')})
require('administrator_login_succeeded', status == 200 and bool(auth.get('token')))
token, administrator_id = auth['token'], auth.get('user', {}).get('id')
require('administrator_identity_available', bool(administrator_id))
counts['migrated_distinct_owners'] = len({item['native']['owner_id'] for item in expected})
counts['migrated_administrator_owned_resources'] = sum(
item['native']['owner_id'] == administrator_id for item in expected)
counts['migrated_other_owner_resources'] = len(expected) - counts['migrated_administrator_owned_resources']
for kind in RESOURCE_KINDS.values():
ok, resources = listed(kind, token)
selected = [item for item in expected if item['native']['kind'] == kind]
require('administrator_lists_all_migrated_resources',
ok and all(item['id'] in resources for item in selected))
denied('anonymous_list_denied', ROOT + kind, None, statuses=(401, 403))
for item in expected:
native = item['native']
path = ROOT + native['kind'] + '/' + item['id']
status, body = request(path, token)
current = body.get('data', {})
require('native_data_owner_status_and_version_match', status == 200 and all(
current.get(field) == native[field]
for field in ('id', 'tenant_id', 'owner_id', 'kind', 'name', 'status', 'version', 'data')))
status, body = request(path + '/jobs?page_size=100', token)
require('imported_resources_have_no_jobs', status == 200 and body.get('data') == [] and body.get('total') == 0)
for suffix in ('', '/jobs?page_size=100'):
denied('anonymous_resource_and_jobs_read_denied', path + suffix, None, statuses=(401, 403))
payload = {key: native[key] for key in ('name', 'data', 'version')}
owned = native['owner_id'] == administrator_id
# PUT checks historical protection before locking. DELETE and supported
# actions lock by owner first, even for a tenant administrator.
denied('all_13_updates_rejected_as_read_only', path, token, payload, 'PUT', statuses=(400,), readonly=True)
for detach, label in (('', 'deletes'), ('?detach=true', 'detaching_deletes')):
denied(('owned_' + label + '_rejected_as_read_only') if owned else
('other_owner_' + label + '_rejected_by_ownership'), path + detach, token,
method='DELETE', statuses=(400,) if owned else (403, 404), readonly=owned)
for action in ACTIONS[native['kind']]:
parameters = {'version': native['version'], 'day': 1}
if action == 'revise':
parameters['advice'] = 'Verification only; preserve historical content.'
segments = native['data'].get('segments', [])
if segments:
parameters['segment_id'] = segments[0]['id']
is_flow = native['kind'] == 'flows'
action_check = 'flow_actions_unsupported' if is_flow else (
'owned_supported_actions_rejected_as_read_only' if owned else
'other_owner_supported_actions_rejected_by_ownership')
denied(action_check, path + '/actions/' + action, token, parameters, 'POST',
statuses=(400,) if is_flow or owned else (403, 404), readonly=owned and not is_flow)
denied('anonymous_update_denied', path, None, payload, 'PUT', statuses=(401, 403))
denied('anonymous_delete_denied', path, None, method='DELETE', statuses=(401, 403))
denied('anonymous_action_denied', path + '/actions/generate', None,
{'version': native['version']}, 'POST', statuses=(401, 403))
require('resources_and_execution_unchanged_after_admin_checks', before == target_snapshot())
suffix = secrets.token_hex(6)
password = secrets.token_urlsafe(18) + 'aA1!'
email = 'business-check-' + suffix + '@migration.invalid'
existing_tenants = set(json.loads(sql_run("SELECT coalesce(json_agg(id),'[]') FROM tenants;")))
status, body = request('/api/v1/system/admin/users/create', token, {
'username': 'business-check-' + suffix, 'email': email, 'password': password})
candidate = body.get('user', {}).get('id')
require('temporary_user_created', status == 201 and candidate and str(uuid.UUID(candidate)) == candidate)
fixture_user = candidate
home = int(sql_run(f"SELECT tenant_id FROM users WHERE id={literal(fixture_user)};"))
if home and home not in existing_tenants:
fixture_tenant = home
require('temporary_user_home_is_new_or_tenantless', home == 0 or fixture_tenant == home)
sql_run(f"INSERT INTO tenant_members(user_id,tenant_id,role,status) VALUES "
f"({literal(fixture_user)},{TARGET_TENANT},'contributor','active');")
status, auth = request('/api/v1/auth/login', data={'email': email, 'password': password})
require('contributor_login_succeeded', status == 200 and bool(auth.get('token')))
member_token = auth['token']
fixture_scope = json.loads(sql_run(f"SELECT json_build_object("
f"'is_admin',is_system_admin,'all_tenants',can_access_all_tenants,"
f"'memberships',(SELECT count(*) FROM tenant_members WHERE user_id=u.id AND deleted_at IS NULL),"
f"'has_personal_membership',EXISTS(SELECT 1 FROM tenant_members WHERE user_id=u.id "
f"AND tenant_id={fixture_tenant or 0} AND role='owner' AND status='active' AND deleted_at IS NULL),"
f"'is_contributor',EXISTS(SELECT 1 FROM tenant_members WHERE user_id=u.id "
f"AND tenant_id={TARGET_TENANT} AND role='contributor' AND status='active' AND deleted_at IS NULL)) "
f"FROM users u WHERE id={literal(fixture_user)};"))
require('temporary_user_is_isolated_contributor', fixture_scope == {
'is_admin': False, 'all_tenants': False, 'memberships': 1 + bool(fixture_tenant),
'has_personal_membership': bool(fixture_tenant), 'is_contributor': True})
for kind in RESOURCE_KINDS.values():
ok, resources = listed(kind, member_token)
require('contributor_lists_no_other_owners_resources', ok and not resources)
denied('cross_tenant_list_denied', ROOT + kind, member_token, tenant=10060)
for item in expected:
native = item['native']
path = ROOT + native['kind'] + '/' + item['id']
payload = {key: native[key] for key in ('name', 'data', 'version')}
for suffix in ('', '/jobs?page_size=100'):
denied('contributor_direct_resource_and_jobs_read_denied', path + suffix, member_token)
denied('cross_tenant_direct_resource_and_jobs_read_denied', path + suffix, member_token, tenant=10060)
denied('contributor_update_denied', path, member_token, payload, 'PUT')
denied('contributor_delete_denied', path, member_token, method='DELETE')
denied('cross_tenant_update_denied', path, member_token, payload, 'PUT', tenant=10060)
denied('cross_tenant_delete_denied', path, member_token, method='DELETE', tenant=10060)
denied('cross_tenant_action_denied', path + '/actions/generate', member_token,
{'version': native['version']}, 'POST', tenant=10060)
if native['kind'] != 'flows':
denied('contributor_direct_action_denied', path + '/actions/generate', member_token,
{'version': native['version']}, 'POST')
else:
denied('contributor_flow_action_unsupported', path + '/actions/generate', member_token,
{'version': native['version']}, 'POST', statuses=(400,))
check('all_api_checks_completed', True)
except Exception:
# Transport/database exceptions can contain private paths or response data.
check('all_api_checks_completed', False)
finally:
if fixture_user:
try:
disable_tenant = (f"UPDATE tenants SET status='inactive',updated_at=now() "
f"WHERE id={fixture_tenant}; ") if fixture_tenant else ''
sql_run(f"BEGIN; DELETE FROM tenant_members WHERE user_id={literal(fixture_user)}; "
f"DELETE FROM auth_tokens WHERE user_id={literal(fixture_user)}; "
f"UPDATE users SET is_active=false,updated_at=now() WHERE id={literal(fixture_user)}; "
+ disable_tenant + "COMMIT;")
cleaned = sql_run(f"SELECT NOT is_active AND NOT EXISTS(SELECT 1 FROM tenant_members "
f"WHERE user_id=u.id) AND NOT EXISTS(SELECT 1 FROM auth_tokens WHERE user_id=u.id) "
f"FROM users u WHERE id={literal(fixture_user)};") == 't'
check('temporary_user_deactivated_membership_and_tokens_removed', cleaned)
if fixture_tenant:
check('temporary_personal_tenant_deactivated', sql_run(
f"SELECT status='inactive' FROM tenants WHERE id={fixture_tenant};") == 't')
except Exception:
check('temporary_user_deactivated_membership_and_tokens_removed', False)
if before is not None:
try:
check('all_existing_resources_jobs_events_revisions_assets_and_knowledge_unchanged', before == target_snapshot())
except Exception:
check('all_existing_resources_jobs_events_revisions_assets_and_knowledge_unchanged', False)
result = {'passed': bool(checks) and all(checks.values()), 'checks': checks, 'counts': dict(counts),
'fixture_user_created': fixture_user is not None,
'fixture_personal_tenant_created': fixture_tenant is not None}
save_json(args.out, result)
print(json.dumps(result))
return 0 if result['passed'] else 1
if __name__ == '__main__':
raise SystemExit(main())
+153
View File
@@ -0,0 +1,153 @@
const assert = require('node:assert/strict')
const fs = require('node:fs')
const path = require('node:path')
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
async function main() {
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
const manifest = JSON.parse(fs.readFileSync(path.join(directory, 'business-apply-20260916.manifest.json'), 'utf8'))
const credentials = JSON.parse(fs.readFileSync(path.join(directory, 'super-admin-login.json'), 'utf8'))
const login = await fetch('http://127.0.0.1:18080/api/v1/auth/login', {
method: 'POST', headers: {'Content-Type': 'application/json'},
body: JSON.stringify({email: credentials.email, password: credentials.password}),
})
assert.equal(login.status, 200, 'Administrator login')
const auth = await login.json()
const headers = {'Content-Type': 'application/json', Authorization: `Bearer ${auth.token}`, 'X-Tenant-ID': '10059'}
async function request(url, method = 'GET', data) {
const response = await fetch(`http://127.0.0.1:18080/api/v1${url}`, {method, headers, body: data === undefined ? undefined : JSON.stringify(data)})
assert.ok(response.ok, `Acceptance HTTP ${response.status}`)
return response.json()
}
const checks = [], errors = []
const browser = await chromium.launch({headless: true, channel: 'chrome'})
let fixture, page, stage = 'starting'
const check = (name, passed) => {checks.push({check: name, passed: !!passed}); assert.ok(passed, name)}
try {
const context = await browser.newContext({viewport: {width: 1440, height: 1000}})
await context.addInitScript(a => {
const tenant = {...a.active_tenant, id: '10059'}
const data = {weknora_user: JSON.stringify({...a.user, tenant_id: String(a.user.tenant_id)}),
weknora_tenant: JSON.stringify(tenant), weknora_token: a.token,
weknora_refresh_token: a.refresh_token, weknora_memberships: JSON.stringify(a.memberships)}
for (const [key, value] of Object.entries(data)) localStorage.setItem(key, value)
localStorage.setItem('weknora:new-user-guide-done:v1', '1')
}, auth)
page = await context.newPage()
page.setDefaultTimeout(15000)
page.on('pageerror', () => errors.push('browser_error'))
async function visit(url) {
await page.goto('http://127.0.0.1:15173' + url)
await page.locator('.workbench-resource').first().waitFor({timeout: 30000})
const guide = page.locator('.guide__close')
if (await guide.isVisible()) await guide.click()
}
async function resize(width, height) {
stage = 'resize_viewport'
await page.setViewportSize({width, height})
stage = 'wait_sidebar_collapse'
if (width <= 640) await page.locator('.aside_box--collapsed').waitFor()
stage = 'wait_page_dimensions'
await page.waitForFunction(() => {
const bounds = document.querySelector('.workbench-page')?.getBoundingClientRect()
return bounds && bounds.width >= Math.min(300, innerWidth - 80) && bounds.right <= innerWidth + 2
}, undefined, {polling: 100})
stage = 'wait_layout_transition'
await page.evaluate(async () => {
await Promise.race([
Promise.all(document.getAnimations()
.filter(animation => Number.isFinite(animation.effect?.getComputedTiming().endTime))
.map(animation => animation.finished.catch(() => {}))),
new Promise(resolve => setTimeout(resolve, 1000)),
])
})
}
async function mobileCheck(kind) {
await resize(390, 844)
check(`${kind}_mobile_usable_width`, await page.locator('.workbench-page').evaluate(element => {
const bounds = element.getBoundingClientRect()
const detail = element.querySelector('.workbench-detail')?.getBoundingClientRect()
return bounds.width >= 300 && bounds.right <= innerWidth + 2 && detail && detail.width >= 260 && detail.right <= innerWidth + 2
}))
await page.locator('.workbench-page').evaluate(element => { element.scrollTop = 0 })
await page.screenshot({path: path.join(directory, `business-${kind}-mobile.png`), animations: 'disabled'})
check(`${kind}_mobile_no_overflow`, await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 2))
}
for (const [kind, route, tab] of [
['trainingmap', '/platform/workbench/content', 'maps'],
['campaign', '/platform/workbench/content', 'tasks'],
]) {
const ids = Object.values(manifest.resource_map[kind])
await visit(`${route}?tab=${tab}&resource=${ids.at(-1)}`)
check(`${kind}_list_count`, await page.locator('.workbench-resource').count() === ids.length)
check(`${kind}_history_warning`, await page.locator('.workbench-detail .t-alert').isVisible())
check(`${kind}_delete_hidden`, await page.locator('.workbench-detail').getByRole('button', {name: '删除', exact: true}).count() === 0)
const aiButtons = page.locator('.workbench-detail').getByRole('button', {name: kind === 'campaign' ? '重新拆解' : '重新生成方案', exact: true})
check(`${kind}_execution_disabled`, await aiButtons.isDisabled())
await page.screenshot({path: path.join(directory, `business-${kind}-desktop.png`), animations: 'disabled'})
await page.getByRole('button', {name: '全屏查看', exact: true}).click()
await page.locator('.t-dialog:visible .workbench-preview').waitFor()
check(`${kind}_fullscreen_readable`, await page.locator('.t-dialog:visible .workbench-preview').isVisible())
stage = 'close_desktop_preview'
await page.locator('.t-dialog:visible .t-dialog__close').click()
stage = 'wait_desktop_preview_closed'
await page.locator('.t-dialog:visible').waitFor({state: 'hidden'})
await mobileCheck(kind)
await page.getByRole('button', {name: '全屏查看', exact: true}).click()
await page.locator('.t-dialog:visible .workbench-preview').waitFor()
check(`${kind}_mobile_fullscreen_fits`, await page.locator('.t-dialog:visible').evaluate(element => {
const bounds = element.getBoundingClientRect()
return bounds.left >= 0 && bounds.right <= innerWidth + 2
}))
await page.screenshot({path: path.join(directory, `business-${kind}-mobile-fullscreen.png`), animations: 'disabled'})
await page.locator('.t-dialog:visible .t-dialog__close').click()
await page.locator('.t-dialog:visible').waitFor({state: 'hidden'})
await resize(1440, 1000)
}
await visit('/platform/workbench/flows')
check('three_historical_flows_listed', await page.locator('.workbench-resource').count() === 3)
await page.locator('.workbench-resource').first().click()
check('flow_history_warning', await page.locator('.workbench-detail .t-alert').isVisible())
check('flow_delete_and_save_hidden', await page.locator('.workbench-detail').getByRole('button', {name: /^(保存|解绑并删除)$/}).count() === 0)
await page.screenshot({path: path.join(directory, 'business-flow-desktop.png'), animations: 'disabled'})
await mobileCheck('flow')
await resize(1440, 1000)
const agents = (await request('/agents')).data
const agent = agents.find(a => a.enabled && a.config.model_id && a.created_by === auth.user.id)
assert.ok(agent, 'Runnable owned Agent available for branch editor fixture')
const edges = [{source: 'a', target: 'b'}, {source: 'a', target: 'c'}]
fixture = (await request('/workbench/resources/flows', 'POST', {
name: `Migration branch check ${Date.now()}`,
data: {description: 'Disposable branch preservation check',
nodes: ['a', 'b', 'c'].map(id => ({id, label: `Branch ${id}`, agent_id: agent.id})), edges},
})).data
await visit('/platform/workbench/flows')
await page.locator('.workbench-resource').filter({hasText: fixture.name}).click()
await page.locator('.flow-node').nth(2).getByRole('button').nth(0).click()
const saved = page.waitForResponse(response => response.url().endsWith(fixture.id) && response.request().method() === 'PUT')
await page.getByRole('button', {name: '保存', exact: true}).click()
check('native_branch_save_success', (await saved).status() === 200)
const current = (await request(`/workbench/resources/flows/${fixture.id}`)).data
check('node_display_reordered', current.data.nodes.map(n => n.id).join(',') === 'a,c,b')
check('branch_edges_preserved_after_reorder_and_save', JSON.stringify(current.data.edges) === JSON.stringify(edges))
check('no_browser_javascript_errors', errors.length === 0)
} finally {
if (checks.length < 25 && page && !page.isClosed()) {
console.log(JSON.stringify({stage, layout: await page.evaluate(() => ({
dialogs: document.querySelectorAll('.t-dialog').length,
collapsed: !!document.querySelector('.aside_box--collapsed'),
width: innerWidth,
page: document.querySelector('.workbench-page')?.getBoundingClientRect().toJSON(),
}))}))
await page.screenshot({path: path.join(directory, 'business-browser-debug.png'), animations: 'disabled'})
}
if (fixture) await request(`/workbench/resources/flows/${fixture.id}?detach=true`, 'DELETE')
await browser.close()
fs.writeFileSync(path.join(directory, 'business-browser-20260916.json'), JSON.stringify({
passed: checks.length === 25 && checks.every(c => c.passed), checks, fixture_removed: !!fixture,
}, null, 2) + '\n')
}
console.log(JSON.stringify({checks, passed: checks.every(c => c.passed)}))
}
main().catch(error => { console.error(error instanceof assert.AssertionError ? error.message : `Business browser verification failed (${error.name})`); process.exitCode = 1 })
+73
View File
@@ -0,0 +1,73 @@
#!/usr/bin/env python3
"""Exercise historical business imports in PostgreSQL and roll back every case."""
import argparse
import json
from pathlib import Path
from apply_weknora_business import build_plan, target_snapshot, PRESERVED_TABLES
from apply_weknora_identities import sql_run
from verify_asset_etl import unchanged_check
from verify_identity_etl import rejected_case
from weknora_business_etl import LEDGER, TARGET_DATABASE, TARGET_TENANT, business_sql
from weknora_category_etl import literal, save_json
from weknora_etl import consistent_snapshot
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--sqlite', type=Path, required=True)
parser.add_argument('--kb-mapping', type=Path, required=True)
parser.add_argument('--artifact-root', type=Path, required=True)
parser.add_argument('--out', type=Path, required=True)
args = parser.parse_args()
if args.out.exists():
parser.error('Report exists')
with consistent_snapshot(args.sqlite) as (db, info):
plan = build_plan(db, args.kb_mapping, args.artifact_root)
plan['source_snapshot_sha256'] = info['sha256']
query = business_sql(plan)
first = plan['resources'][0]['target']
rid = literal(first['id'])
owner = literal(next(uid for uid in plan['identity_mapping'].values() if uid != first['owner_id']))
scope = f"target_id={rid} AND target_tenant_id={TARGET_TENANT}"
cases = [
('wrong_resource_tenant_rejected', f'UPDATE workbench_resources SET tenant_id={TARGET_TENANT + 1} WHERE id={rid};', 'Mapped business resource removed'),
('wrong_resource_owner_rejected', f'UPDATE workbench_resources SET owner_id={owner} WHERE id={rid};', 'Mapped business resource removed'),
('deleted_resource_not_recreated', f'DELETE FROM workbench_resources WHERE id={rid};', 'Mapped business resource removed'),
('ledger_target_conflict_rejected', f"UPDATE {LEDGER} SET target_id='rollback-only' WHERE {scope};", 'Business durable mapping conflict'),
('ledger_source_conflict_rejected', f"UPDATE {LEDGER} SET source_snapshot='{{}}' WHERE {scope};", 'Business durable mapping conflict'),
('ledger_projection_conflict_rejected', f"UPDATE {LEDGER} SET native_snapshot='{{}}' WHERE {scope};", 'Business durable mapping conflict'),
('id_without_provenance_rejected', f'DELETE FROM {LEDGER} WHERE {scope};', 'Native business ID or owner/name collision'),
('inactive_tenant_rejected', f"UPDATE tenants SET status='suspended' WHERE id={TARGET_TENANT};", 'Business migration requires active target tenant'),
]
before = target_snapshot()
ledger_before = sql_run(f"SELECT to_regclass('public.{LEDGER}') IS NOT NULL;")
sql = "BEGIN ISOLATION LEVEL SERIALIZABLE; SET LOCAL statement_timeout='45s'; SET LOCAL lock_timeout='5s';\n"
preserved = [(table, f'_preserved_{i}') for i, table in enumerate(PRESERVED_TABLES)]
for table, name in preserved:
sql += f'CREATE TEMP TABLE {name} ON COMMIT DROP AS SELECT * FROM {table};\n'
sql += query
pairs = [('workbench_resources', '_resource_repeat'), (LEDGER, '_ledger_repeat')]
for table, name in pairs:
sql += f'CREATE TEMP TABLE {name} ON COMMIT DROP AS SELECT * FROM {table};\n'
sql += query + unchanged_check('exact_repeat_noop', pairs + preserved)
for name, mutation, expected in cases:
sql += rejected_case(name, mutation, expected, query)
sql += f"UPDATE workbench_resources SET name='Rollback native edit',status='paused',version=99,data=jsonb_set(data,'{{description}}','\"Local edit\"'::jsonb) WHERE id={rid};\n"
sql += 'CREATE TEMP TABLE _native_edit ON COMMIT DROP AS SELECT * FROM workbench_resources;\n'
sql += query + unchanged_check('native_edits_preserved', [('workbench_resources', '_native_edit'), pairs[1], *preserved])
sql += 'ROLLBACK;'
checks = [json.loads(line) for line in sql_run(sql).splitlines() if line.startswith('{')]
checks.extend([
{'check': 'all_cases_executed', 'passed': len(checks) == len(cases) + 2},
{'check': 'all_regressions_rolled_back', 'passed': before == target_snapshot() and ledger_before == sql_run(f"SELECT to_regclass('public.{LEDGER}') IS NOT NULL;")},
{'check': 'source_unchanged', 'passed': not info['source_mutated']},
])
result = {'database': TARGET_DATABASE, 'committed': False, 'passed': all(c['passed'] for c in checks), 'checks': checks}
save_json(args.out, result)
print(json.dumps(result))
return 0 if result['passed'] else 1
if __name__ == '__main__':
raise SystemExit(main())
+164
View File
@@ -0,0 +1,164 @@
const assert = require('node:assert/strict')
const crypto = require('node:crypto')
const fs = require('node:fs')
const path = require('node:path')
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
const root = path.resolve(__dirname, '../..')
const origin = process.env.VERIFY_FRONTEND_ORIGIN || 'https://agent.pducv.cn'
const apiOrigin = process.env.VERIFY_API_ORIGIN || 'http://127.0.0.1:18080'
const kbID = '16c3e59c-4bb6-41cf-ae0a-6d0941b3d032'
const report = { checked_at: new Date().toISOString(), origin, api_origin: apiOrigin, checks: [], failures: [] }
let auth, browser
function check(name, value) {
report.checks.push({ name, passed: !!value })
assert.ok(value, name)
}
async function api(route, requestOrigin = apiOrigin) {
const response = await fetch(`${requestOrigin}/api/v1${route}`, {
headers: { Authorization: `Bearer ${auth.token}`, 'X-Tenant-ID': '10059' },
signal: AbortSignal.timeout(30000),
})
assert.equal(response.status, 200, `GET ${route}: HTTP ${response.status}`)
const result = await response.json()
assert.notEqual(result.success, false, `GET ${route} failed`)
return result.data
}
async function main() {
const credentials = JSON.parse(fs.readFileSync(path.join(root, 'acceptance/super-admin-login.json'), 'utf8'))
const login = await fetch(`${apiOrigin}/api/v1/auth/login`, {
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(credentials),
signal: AbortSignal.timeout(30000),
})
assert.equal(login.status, 200, 'authentication failed')
const loginResult = await login.json()
auth = loginResult.data || loginResult
const kbs = await api('/knowledge-bases')
report.knowledge_bases = []
for (const kb of kbs) {
const config = await api(`/initialization/config/${kb.id}`)
const initialized = !!(kb.summary_model_id && kb.embedding_model_id && config.llm?.modelName && config.embedding?.modelName)
report.knowledge_bases.push({ id: kb.id, name: kb.name, initialized, graph_enabled: !!kb.extract_config?.enabled,
llm_model: config.llm?.modelName, embedding_model: config.embedding?.modelName })
check(`models_resolve:${kb.name}`, initialized)
}
const overview = await api(`/knowledge-bases/${kbID}/evidence-graph`)
report.graph = { counts: overview.counts, sources: overview.sources, mapped_sources: overview.mapped_sources }
check('product_has_extracted_entities_and_relations', overview.counts.knowledgeentity > 2 && overview.counts.knowledgerelation > 0)
const nodes = [...overview.nodes]
let cursor = overview.next_cursor
for (let pages = 0; cursor && pages < 50; pages++) {
const next = await api(`/knowledge-bases/${kbID}/evidence-graph?cursor=${cursor}`)
nodes.push(...next.nodes)
assert.notEqual(next.next_cursor, cursor, 'graph cursor did not advance')
cursor = next.next_cursor
}
assert.equal(cursor, 0, 'graph exceeded verification pagination bound')
check('all_entity_ids_are_exact_in_browser', nodes.every(node => Number.isSafeInteger(node.id)))
const nativeNodes = nodes.filter(node => node.payload.origin === 'native_extraction_v1')
check('native_extraction_is_visible', nativeNodes.length > 0)
check('example_entities_are_absent', !nativeNodes.some(node => ['基础概念', '实践练习'].includes(node.payload.canonical_name)))
const chunks = new Map(), documents = new Map(), relations = new Map(), evidenceIDs = new Set()
let selected
for (const node of nativeNodes) {
const detail = await api(`/knowledge-bases/${kbID}/evidence-graph?entity=${node.id}`)
assert.equal(detail.nodes[0]?.id, node.id, 'entity detail must resolve exact ID')
const nativeEvidence = detail.evidence.filter(item => item.payload.origin === 'native_extraction_v1')
assert.ok(nativeEvidence.length, `native entity ${node.id} has no evidence`)
if (!selected && detail.relations.length) selected = { node, evidence: nativeEvidence[0] }
for (const relation of detail.relations) relations.set(relation.id, relation)
for (const evidence of nativeEvidence) {
assert.equal(evidence.mapping_state, 'native_extraction', 'evidence no longer maps to its original chunk')
assert.ok(evidence.native_chunk_id && evidence.native_knowledge_id, 'evidence source link is missing')
if (!chunks.has(evidence.native_chunk_id)) chunks.set(evidence.native_chunk_id, await api(`/chunks/by-id/${evidence.native_chunk_id}`))
if (!documents.has(evidence.native_knowledge_id)) documents.set(evidence.native_knowledge_id, await api(`/knowledge/${evidence.native_knowledge_id}`))
const chunk = chunks.get(evidence.native_chunk_id), document = documents.get(evidence.native_knowledge_id)
assert.equal(chunk.knowledge_base_id, kbID, 'chunk belongs to another knowledge base')
assert.equal(chunk.knowledge_id, evidence.native_knowledge_id, 'chunk does not belong to the source document')
assert.equal(document.knowledge_base_id, kbID, 'source document belongs to another knowledge base')
assert.equal(crypto.createHash('sha256').update(chunk.content).digest('hex'), evidence.payload.source_content_hash,
'evidence content hash differs from actual chunk')
assert.ok(evidence.payload.excerpt && chunk.content.includes(evidence.payload.excerpt), 'evidence excerpt is not from the original chunk')
evidenceIDs.add(evidence.id)
}
}
check('native_evidence_resolves_to_unchanged_source', evidenceIDs.size > 0)
check('example_relation_is_absent', [...relations.values()].every(relation =>
!(relation.payload.from_name === '基础概念' && relation.payload.to_name === '实践练习')))
const nativeRelations = [...relations.values()].filter(relation => relation.payload.origin === 'native_extraction_v1')
for (const relation of nativeRelations) {
assert.ok(typeof relation.payload.evidence === 'string' && relation.payload.evidence.trim(),
`native relation ${relation.id} has no source quotation`)
assert.ok([...chunks.values()].some(chunk => chunk.content.includes(relation.payload.evidence)),
`native relation ${relation.id} quotation is absent from verified source chunks`)
}
check('native_relations_have_verbatim_source_quotations', nativeRelations.length > 0)
report.verified_native = { entities: nativeNodes.length, entity_evidence: evidenceIDs.size,
relations: nativeRelations.length, quoted_relations: nativeRelations.length,
chunks: chunks.size, documents: documents.size }
report.sample_relations = [...relations.values()].slice(0, 5).map(relation => ({
from: relation.payload.from_name, relation: relation.payload.relation_type, to: relation.payload.to_name,
}))
const publicOverview = await api(`/knowledge-bases/${kbID}/evidence-graph`, origin)
check('public_graph_api_exposes_completed_extractions', publicOverview.counts.knowledgeentity >= overview.counts.knowledgeentity)
const denied = await fetch(`${origin}/api/v1/knowledge-bases/${kbID}/evidence-graph`, { signal: AbortSignal.timeout(30000) })
check('public_graph_api_requires_auth', denied.status === 401)
browser = await chromium.launch({ headless: true, channel: 'chrome' })
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } })
await context.addInitScript(value => {
const local = { weknora_user: JSON.stringify({ ...value.user, tenant_id: String(value.user.tenant_id) }),
weknora_tenant: JSON.stringify({ ...value.active_tenant, id: '10059' }), weknora_token: value.token,
weknora_refresh_token: value.refresh_token, weknora_memberships: JSON.stringify(value.memberships), weknora_selected_tenant_id: '10059' }
for (const [key, item] of Object.entries(local)) localStorage.setItem(key, item)
localStorage.setItem('weknora:new-user-guide-done:v1', '1')
}, auth)
const page = await context.newPage(), errors = []
page.on('pageerror', error => errors.push(error.message))
page.on('response', response => {
const pathname = new URL(response.url()).pathname
if (response.status() >= 400 && /\/(?:api|assets)\//.test(pathname)) errors.push(`${response.status()} ${pathname}`)
})
await page.goto(`${origin}/platform/knowledge-bases/${kbID}?tab=evidence`)
await page.locator('.entity-row').first().waitFor({ timeout: 30000 })
assert.ok(selected, 'missing native entity with relation for browser verification')
await page.getByRole('textbox', { name: '搜索实体' }).fill(selected.node.payload.canonical_name)
await page.getByRole('button', { name: '搜索', exact: true }).click()
await page.locator('.entity-row').filter({ hasText: `#${selected.node.id}` }).click()
await page.locator('.evidence-row').first().waitFor({ timeout: 30000 })
check('native_entity_relations_visible', await page.locator('.relation-list li').count() > 0)
check('source_evidence_marked_verified', await page.getByText('已核验原生分块', { exact: true }).count() > 0)
await page.screenshot({ path: path.join(root, 'acceptance/business-graph-live-desktop.png'), animations: 'disabled' })
await page.getByRole('button', { name: '查看原生文档', exact: true }).first().click()
await page.locator('.t-drawer--open').first().waitFor({ timeout: 30000 })
const originalDocument = documents.get(selected.evidence.native_knowledge_id)
const drawer = page.locator('.t-drawer--open').first()
const rawTitle = originalDocument.file_name || originalDocument.title || originalDocument.source
const extensionIndex = rawTitle.lastIndexOf('.')
const displayTitle = originalDocument.type === 'file' && extensionIndex > 0 ? rawTitle.slice(0, extensionIndex) : rawTitle
await page.waitForFunction(title => [...document.querySelectorAll('.t-drawer--open .doc-drawer-header-title')]
.some(element => element.textContent.trim() === title), displayTitle, { timeout: 30000 })
check('source_document_opens_in_drawer', (await drawer.locator('.doc-drawer-header-title').innerText()).trim() === displayTitle)
await page.screenshot({ path: path.join(root, 'acceptance/business-graph-live-source.png'), animations: 'disabled' })
await page.keyboard.press('Escape')
await page.setViewportSize({ width: 390, height: 844 })
await page.screenshot({ path: path.join(root, 'acceptance/business-graph-live-mobile.png'), animations: 'disabled' })
check('mobile_graph_has_no_overflow', await page.locator('.evidence-graph').evaluate(element => element.scrollWidth <= element.clientWidth + 2))
check('mobile_page_has_no_overflow', await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 2))
await page.goto(`${origin}/platform/knowledge-bases`)
await page.waitForLoadState('networkidle', { timeout: 30000 })
check('knowledge_base_list_has_no_initialization_warning', await page.locator('.warning-banner').count() === 0 && await page.locator('.uninitialized').count() === 0)
report.browser_errors = errors
check('browser_has_no_api_or_runtime_errors', errors.length === 0)
}
main().catch(error => { report.failures.push(error.message); process.exitCode = 1 }).finally(async () => {
if (browser) await browser.close()
report.passed = report.failures.length === 0
fs.writeFileSync(path.join(root, 'acceptance/business-graph-live.json'), JSON.stringify(report, null, 2) + '\n')
console.log(JSON.stringify(report))
})
+183
View File
@@ -0,0 +1,183 @@
const assert = require('node:assert/strict')
const crypto = require('node:crypto')
const { execFileSync } = require('node:child_process')
const fs = require('node:fs')
const path = require('node:path')
const kb = '16c3e59c-4bb6-41cf-ae0a-6d0941b3d032'
const problemChunk = '1144acfb-18de-4687-8bf5-df0e9c8f1dd5'
const outputPath = path.resolve(__dirname, '../../acceptance/business-graph-semantics.json')
const report = { checked_at: new Date().toISOString(), production_write: false, checks: [], failures: [] }
const query = `BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY;
SELECT json_build_object(
'config',(SELECT extract_config FROM knowledge_bases WHERE tenant_id=10059 AND id='${kb}' AND deleted_at IS NULL),
'chunks',COALESCE((SELECT json_agg(json_build_object('id',c.id,'knowledge_id',c.knowledge_id,'content',c.content,'file_name',k.file_name) ORDER BY c.id)
FROM chunks c JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id
WHERE c.tenant_id=10059 AND c.knowledge_base_id='${kb}' AND c.chunk_type='text' AND c.is_enabled=true AND c.deleted_at IS NULL AND k.deleted_at IS NULL),'[]'),
'records',COALESCE((SELECT json_agg(json_build_object('id',legacy_id,'table',source_table,'payload',payload) ORDER BY source_table,legacy_id)
FROM native_graph_records WHERE tenant_id=10059 AND knowledge_base_id='${kb}' AND payload->>'origin'='native_extraction_v1'),'[]'),
'sources',COALESCE((SELECT json_agg(json_build_object('source_chunk_id',source_chunk_id,'native_chunk_id',native_chunk_id,'native_knowledge_id',native_knowledge_id,'source_content_hash',source_content_hash,'native_content_sha256',native_content_sha256,'mapping_state',mapping_state))
FROM native_graph_sources WHERE tenant_id=10059 AND knowledge_base_id='${kb}' AND mapping_state='native_extraction'),'[]')
);
COMMIT;`
function check(name, passed) {
report.checks.push({ name, passed: !!passed })
if (!passed) report.failures.push(name)
}
try {
const input = JSON.parse(execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres',
'psql', '-X', '-qAt', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915'],
{ input: query, encoding: 'utf8', maxBuffer: 20 * 1024 * 1024 }))
const chunks = new Map(input.chunks.map(chunk => [chunk.id, chunk]))
const hashes = new Map(input.chunks.map(chunk => [chunk.id, crypto.createHash('sha256').update(chunk.content).digest('hex')]))
const sources = new Map(input.sources.map(source => [source.source_chunk_id, source]))
const entities = new Map(input.records.filter(record => record.table === 'knowledgeentity').map(record => [record.id, record]))
const relations = input.records.filter(record => record.table === 'knowledgerelation')
const states = input.records.filter(record => record.table === 'knowledgegraphchunkstate')
const evidence = input.records.filter(record => record.table === 'knowledgegraphevidence')
const labels = new Set(input.config.tags)
report.counts = { source_chunks: chunks.size, checkpoints: states.length, entities: entities.size,
relations: relations.length, evidence_records: evidence.length }
check('all_51_product_chunks_are_present', chunks.size === 51)
check('all_51_product_chunks_are_checkpointed', states.length === 51 && new Set(states.map(state => state.payload.native_chunk_id)).size === 51)
for (const state of states) {
const chunk = chunks.get(state.payload.native_chunk_id)
check(`checkpoint_current:${state.payload.native_chunk_id}`, chunk
&& state.payload.native_knowledge_id === chunk.knowledge_id
&& state.payload.source_content_hash === hashes.get(chunk.id)
&& state.payload.extraction_status === 'complete')
}
const relationEvidence = new Map()
for (const record of evidence) {
const item = record.payload, source = sources.get(item.source_chunk_id), chunk = chunks.get(source?.native_chunk_id)
check(`evidence_grounded:${record.id}`, chunk && source.mapping_state === 'native_extraction'
&& source.native_knowledge_id === chunk.knowledge_id && item.native_knowledge_id === chunk.knowledge_id
&& item.source_content_hash === hashes.get(chunk.id) && source.source_content_hash === hashes.get(chunk.id)
&& source.native_content_sha256 === hashes.get(chunk.id)
&& typeof item.excerpt === 'string' && item.excerpt.trim().length > 0 && chunk.content.includes(item.excerpt))
if (item.subject_type === 'relation') {
if (!relationEvidence.has(item.subject_id)) relationEvidence.set(item.subject_id, [])
relationEvidence.get(item.subject_id).push({ evidence_id: record.id, chunk_id: chunk?.id, quote: item.excerpt })
}
}
report.relations = relations.map(record => {
const item = record.payload, quotations = relationEvidence.get(record.id) || []
const from = entities.get(item.from_entity_id), to = entities.get(item.to_entity_id)
check(`relation_has_grounded_quote:${record.id}`, from && to && labels.has(item.relation_type)
&& typeof item.evidence === 'string' && item.evidence.trim().length > 0
&& quotations.length > 0 && quotations.some(quotation => quotation.quote === item.evidence))
return { id: record.id, from_id: item.from_entity_id, to_id: item.to_entity_id,
from: from?.payload.canonical_name, relation: item.relation_type,
to: to?.payload.canonical_name, evidence: quotations }
})
const relationChunks = new Set(report.relations.flatMap(relation => relation.evidence.map(item => item.chunk_id)))
report.counts.chunks_with_relations = relationChunks.size
report.counts.checkpointed_without_relations = states.filter(state => !relationChunks.has(state.payload.native_chunk_id)).length
report.counts.unresolved_chunks = [...chunks.keys()].filter(id => !relationChunks.has(id)).length
for (const state of states) {
check(`completed_checkpoint_has_relations:${state.payload.native_chunk_id}`, relationChunks.has(state.payload.native_chunk_id))
}
const problemRelations = report.relations.filter(relation => relation.evidence.some(item => item.chunk_id === problemChunk))
const expectedRestrictions = [
['过敏者', ['慎用于']],
['粘膜破损处', ['禁用于', '不适用于', '不能用于']],
['性生活中对性病的预防', ['禁用于', '不适用于', '不能用于']],
]
for (const [target, allowed] of expectedRestrictions) {
const candidates = problemRelations.filter(relation => relation.to === target || relation.from === target)
check(`known_restriction_preserved:${target}`, candidates.length > 0 && candidates.every(relation =>
relation.from === '本品' && relation.to === target && allowed.includes(relation.relation)))
}
const compact = value => value.replace(/\s+/g, '')
const quantities = value => new Set((compact(value).toLowerCase().match(/[0-9]+(?:[..·][0-9]+)*(?:ppm|ppb|kg|mg|ug|ml|cm|mm|min|g|l|%|%|毫克|千克|克|毫升|升|分钟|小时|天|颗|粒|瓶|支|片)?/g) || [])
.map(value => value.replace(/[.·]/g, '.').replace(/%/g, '%')))
const quantityGrounded = (endpoint, source) => [...quantities(endpoint)].every(value => quantities(source).has(value))
const prohibition = /禁|不得|不能|不可|不宜|不应|不建议|不适用|不适合|不用于|不具有|不具备|避开|请勿|勿用|停用|停止使用/
const affirmative = new Set(['适用于', '适合', '用于', '可用于', '条件适用于'])
const genericEntities = new Map()
for (const relation of report.relations) {
const endpoints = compact(relation.from + relation.to)
for (const [id, name] of [[relation.from_id, relation.from], [relation.to_id, relation.to]]) {
const localName = compact(name).toLowerCase()
if (!['本品', '该产品', '本产品', '产品', '本商品', '此产品', '本公司', '该公司', '本文', '本方法', '该方法', 'thisproduct', 'theproduct'].includes(localName)
&& !localName.endsWith('未名产品') && !localName.startsWith('本品(')) continue
if (!genericEntities.has(id)) genericEntities.set(id, new Set())
for (const item of relation.evidence) genericEntities.get(id).add(item.chunk_id)
}
for (const item of relation.evidence) {
const quote = compact(item.quote)
const source = chunks.get(item.chunk_id)?.content || ''
check(`endpoint_quantities_grounded:${relation.id}:${item.chunk_id}`,
quantityGrounded(relation.from, source) && quantityGrounded(relation.to, item.quote))
if (relation.relation === '抑制') {
const target = compact(relation.to)
check(`suppression_has_target_predicate:${relation.id}:${item.chunk_id}`, quote.includes(target)
&& /抑制|杀灭|杀菌|抑菌|抗菌/.test(quote.split(target).join('')))
}
if (affirmative.has(relation.relation)) {
check(`suitability_has_use_claim:${relation.id}:${item.chunk_id}`, /适用|适合|用于|可用|使用|涂于|涂抹|推入|推进|擦拭|清洁|饮用|服用|外用|内服|口服/.test(quote))
check(`storage_not_suitability:${relation.id}:${item.chunk_id}`, !/贮存|贮藏|储存|储藏|保存|存放/.test(relation.to)
&& !(/阴凉|干燥处|冷藏|常温|避光|遮光|密封/.test(relation.to) && /贮存|贮藏|储存|储藏|保存|存放/.test(quote)))
}
if (['禁用于', '不适用于', '不能用于'].includes(relation.relation)) {
check(`storage_not_usage_prohibition:${relation.id}:${item.chunk_id}`,
!(/阳光|直晒|阴凉|干燥|冷藏|常温|避光|遮光|密封/.test(relation.to)
&& /贮存|贮藏|储存|储藏|保存|存放/.test(quote)))
}
if (relation.relation === '约束') {
if (/避免.{0,16}(?:同用|合用)|不[宜可得能].{0,16}(?:同用|合用)/.test(quote) && /同用|合用|拮抗/.test(endpoints)) {
check(`constraint_preserves_combination_prohibition:${relation.id}:${item.chunk_id}`, /避免|禁止|不得|不能|不可|不宜|不要|勿/.test(endpoints))
}
if (/停止使用|立即停用/.test(quote) && /不适|使用|停用/.test(endpoints)) {
check(`constraint_preserves_stop_action:${relation.id}:${item.chunk_id}`, /停止使用|立即停用/.test(endpoints))
}
if (/儿童不[易宜能]触及|儿童触及不到/.test(quote) && /儿童/.test(endpoints)) {
check(`constraint_preserves_child_storage_action:${relation.id}:${item.chunk_id}`, /不[易宜能]触及|触及不到|远离|避免/.test(endpoints))
}
}
const nameAt = source.indexOf(relation.from), quoteAt = source.indexOf(item.quote)
if (nameAt >= 0 && quoteAt >= 0 && quoteAt + item.quote.length < nameAt && !source.slice(nameAt).includes(item.quote)) {
const boundary = source.lastIndexOf('\n\n', nameAt)
check(`no_later_product_attribution:${relation.id}:${item.chunk_id}`, boundary < quoteAt + item.quote.length
|| !/产品名称|品名/.test(source.slice(boundary)))
}
if (item.chunk_id === '3626de4e-88a2-41e4-8e09-05c51f23848a' && /避开女性崆期|产期/.test(quote)) {
check(`avoidance_not_permission:${relation.id}`, !affirmative.has(relation.relation)
&& relation.relation !== '慎用于')
if (['禁用于', '不适用于', '不能用于', '约束'].includes(relation.relation)) {
check(`vaginal_restriction_scope_preserved:${relation.id}`, endpoints.includes('阴道'))
}
}
if (quote.includes('哺乳期请勿在乳头使用') && /哺乳期|乳头/.test(endpoints)
&& ['禁用于', '不适用于', '不能用于', '约束'].includes(relation.relation)) {
check(`lactation_restriction_scope_preserved:${relation.id}`, endpoints.includes('哺乳期') && endpoints.includes('乳头'))
}
if (/抑制|杀灭|抑菌|杀菌|抗菌/.test(quote) && /菌$|病毒$|微生物$/.test(compact(relation.to))
&& !/抑制|杀灭|抑菌|杀菌|抗菌/.test(compact(relation.to))) {
check(`suppression_not_support:${relation.id}`, !['支持', '促进', '增强'].includes(relation.relation))
}
if (/告别冷淡|改善粗糙与干燥/.test(quote) && !prohibition.test(quote)) {
check(`benefit_not_prohibition:${relation.id}`, !['禁用于', '不适用于', '不能用于', '慎用于', '不支持', '不导致'].includes(relation.relation))
}
if (quote.includes('停止使用') && relation.to.includes('不适')) {
check(`stop_use_not_caution:${relation.id}`, !affirmative.has(relation.relation) && relation.relation !== '慎用于')
}
}
}
for (const [id, chunkIDs] of genericEntities) {
check(`generic_product_is_chunk_scoped:${id}`, chunkIDs.size <= 1)
}
report.problem_chunk_relations = problemRelations
report.chunks = input.chunks.map(chunk => ({ ...chunk, sha256: hashes.get(chunk.id),
relations: report.relations.filter(relation => relation.evidence.some(item => item.chunk_id === chunk.id)) }))
report.semantic_review_note = 'Automated checks cover exact-source evidence, original contraindications, numeric endpoints, product attribution and generic-entity scope, explicit suitability versus storage, and demonstrated polarity reversals. Full semantic correctness still requires reviewing each listed relation with its source chunk.'
} catch (error) {
report.failures.push(error.message)
}
report.passed = report.failures.length === 0
fs.writeFileSync(outputPath, JSON.stringify(report, null, 2) + '\n')
console.log(JSON.stringify({ checked_at: report.checked_at, production_write: report.production_write,
passed: report.passed, counts: report.counts, failures: report.failures, report_path: outputPath }))
if (!report.passed) process.exitCode = 1
+66
View File
@@ -0,0 +1,66 @@
const fs = require('node:fs')
const path = require('node:path')
const assert = require('node:assert/strict')
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
const directory = path.resolve(__dirname, '../../acceptance')
const origin = 'https://agent.pducv.cn'
const checks = [], failures = [], snapshots = []
let auth, browser
function check(name, value) { checks.push({name, passed: !!value}); assert.ok(value, name) }
async function api(route, method = 'GET', body) {
const response = await fetch(origin + '/api/v1' + route, {
method, headers: {'Content-Type': 'application/json', ...(auth ? {Authorization: `Bearer ${auth.token}`, 'X-Tenant-ID': '10059'} : {})},
body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(45000),
})
assert.equal(response.status, 200, route)
return response.json()
}
async function main() {
const credentials = JSON.parse(fs.readFileSync(path.join(directory, 'super-admin-login.json'), 'utf8'))
auth = await api('/auth/login', 'POST', {username: credentials.username, password: credentials.password})
check('admin_login_opens_business_workspace', Number(auth.active_tenant.id) === 10059)
const baseline = JSON.parse(fs.readFileSync(path.join(directory, 'business-wiki-backfill.json'), 'utf8'))
for (const kb of baseline.before) {
const current = (await api(`/knowledge-bases/${kb.id}`)).data
check(`${kb.name}:rag_and_wiki_enabled`, current.indexing_strategy.wiki_enabled && current.indexing_strategy.vector_enabled && current.indexing_strategy.keyword_enabled)
const stats = await api(`/knowledgebase/${kb.id}/wiki/stats`)
const pages = await api(`/knowledgebase/${kb.id}/wiki/pages?page=1&page_size=5`)
check(`${kb.name}:wiki_readable`, typeof stats.total_pages === 'number' && Array.isArray(pages.pages))
snapshots.push({id: kb.id, name: kb.name, pages: stats.total_pages, pending_tasks: stats.pending_tasks, active: stats.is_active})
}
check('wiki_pages_actually_generated', snapshots.some(s => s.pages > 0))
const product = baseline.before.find(k => k.name === '\u4ea7\u54c1\u4fe1\u606f')
const start = Date.now()
const hits = (await api(`/knowledge-bases/${product.id}/hybrid-search`, 'POST', {
query_text: '\u4ea7\u54c1\u4fe1\u606f', match_count: 5, vector_threshold: 0, keyword_threshold: 0,
})).data
check('existing_rag_still_returns_source_chunks', Array.isArray(hits) && hits.some(hit => hit.knowledge_id))
snapshots.push({rag_query_ms: Date.now() - start, rag_hits: hits.length})
browser = await chromium.launch({headless: true, channel: 'chrome'})
const page = await browser.newPage({viewport: {width: 1440, height: 1000}})
await page.addInitScript(a => {
const values = {weknora_user: JSON.stringify(a.user), weknora_tenant: JSON.stringify(a.active_tenant), weknora_token: a.token, weknora_refresh_token: a.refresh_token, weknora_memberships: JSON.stringify(a.memberships)}
Object.entries(values).forEach(([k,v]) => localStorage.setItem(k,v))
localStorage.setItem('weknora:new-user-guide-done:v1', '1')
}, auth)
const errors = []
page.on('pageerror', e => errors.push(e.message))
page.on('response', response => {if(response.status() >= 400 && /\/(api|assets)\//.test(new URL(response.url()).pathname)) errors.push(`${response.status()} ${new URL(response.url()).pathname}`)})
await page.goto(`${origin}/platform/knowledge-bases/${product.id}?tab=wiki`)
await page.locator('.wiki-browser').waitFor({timeout: 30000})
await page.locator('.wiki-tab').first().waitFor({timeout: 30000})
await page.locator('.wiki-directory-item, .wiki-page-item').first().waitFor({timeout: 30000})
check('generated_wiki_entries_visible', await page.locator('.wiki-tab-count').first().innerText() !== '0')
await page.screenshot({path: path.join(directory, 'business-wiki-desktop.png'), animations: 'disabled'})
check('public_wiki_browser_visible', await page.locator('.wiki-browser').isVisible())
await page.setViewportSize({width: 390, height: 844})
await page.screenshot({path: path.join(directory, 'business-wiki-mobile.png'), animations: 'disabled'})
check('wiki_no_horizontal_overflow', await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 2))
check('wiki_no_failed_requests_or_page_errors', errors.length === 0)
}
main().catch(e => {failures.push(e.message); process.exitCode = 1}).finally(async () => {
if (browser) await browser.close()
const report = {checked_at: new Date().toISOString(), origin, checks, snapshots, failures, passed: failures.length === 0}
fs.writeFileSync(path.join(directory, 'business-wiki-verification.json'), JSON.stringify(report, null, 2))
console.log(JSON.stringify(report))
})
@@ -0,0 +1,319 @@
// Creates disposable fixtures only. No generation actions or paid model calls.
// VERIFY_API_ORIGIN / VERIFY_FRONTEND_ORIGIN default to the public deployment.
// CANVAS_BROWSER=0 runs API checks only. PostgreSQL must back the selected API.
const assert = require('node:assert/strict')
const crypto = require('node:crypto')
const fs = require('node:fs')
const path = require('node:path')
const { execFileSync } = require('node:child_process')
const root = path.resolve(__dirname, '../..')
const directory = path.join(root, 'acceptance')
const origin = (process.env.VERIFY_API_ORIGIN || 'https://agent.pducv.cn').replace(/\/$/, '')
const frontend = (process.env.VERIFY_FRONTEND_ORIGIN || origin).replace(/\/$/, '')
const reportName = process.env.VERIFY_REPORT_NAME || 'canvas-configuration-live'
const tenant = 10059
const database = 'yawan_native_acceptance_20260915'
const prefix = `canvas-check-${Date.now()}-${crypto.randomBytes(3).toString('hex')}`
const checks = [], failures = [], resources = []
const knowledgeKeys = ['knowledge_base_ids', 'knowledge_ids', 'knowledge_folders', 'knowledge_tags']
const browserEnabled = process.env.CANVAS_BROWSER !== '0'
let admin, member, role, user, agent, kb, browser, foreignTenant, databaseVerified = false
assert.match(reportName, /^[a-zA-Z0-9_-]+$/)
const literal = value => `'${String(value).replaceAll("'", "''")}'`
const sql = query => execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres',
'psql', '-X', '-q', '-U', 'postgres', '-d', database, '-At', '-v', 'ON_ERROR_STOP=1'],
{ input: query, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim()
function check(name, value) {
checks.push({ name, passed: !!value })
assert.ok(value, name)
console.log(`PASS ${name}`)
}
async function api(route, method = 'GET', body, identity = admin, expected = 200) {
const response = await fetch(origin + '/api/v1' + route, { method, headers: {
'Content-Type': 'application/json',
...(identity ? { Authorization: `Bearer ${identity.token}`, 'X-Tenant-ID': String(tenant) } : {}),
}, body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(30000) })
const data = await response.json()
assert.ok((Array.isArray(expected) ? expected : [expected]).includes(response.status),
`${method} ${route}: HTTP ${response.status}; ${data.message || data.error?.message || ''}`)
return data
}
function resourcePath(resource) {
assert.ok(resources.some(row => row.id === resource.id && row.kind === resource.kind), 'Only recorded fixtures may be changed')
return `/workbench/resources/${resource.kind}/${resource.id}`
}
async function create(kind, label, data, identity = member) {
const resource = (await api(`/workbench/resources/${kind}`, 'POST', { name: `${prefix}-${label}`, data }, identity, 201)).data
resources.push({ ...resource, identity })
assert.equal(resource.owner_id, identity.user.id)
return resource
}
function fixtureWhere(resource) {
resourcePath(resource)
assert.ok(resource.name.startsWith(prefix + '-'))
return `id=${literal(resource.id)} AND tenant_id=${Number(resource.tenant_id)} AND kind=${literal(resource.kind)} AND name LIKE ${literal(prefix + '-%')}`
}
function seed(resource, data, ownerID = resource.owner_id, status = resource.status) {
assert.ok(databaseVerified, 'API and database identity must match before seeding')
const id = sql(`UPDATE workbench_resources SET data=${literal(JSON.stringify(data))}::jsonb,
owner_id=${literal(ownerID)},status=${literal(status)} WHERE ${fixtureWhere(resource)} RETURNING id;`)
assert.equal(id, resource.id, 'Exactly the recorded fixture must be seeded')
}
async function read(resource, identity = admin) {
return (await api(resourcePath(resource), 'GET', undefined, identity)).data
}
async function absent(resource, identity = admin) {
await api(resourcePath(resource), 'GET', undefined, identity, 404)
const listed = await api(`/workbench/resources/${resource.kind}?q=${encodeURIComponent(prefix)}&page_size=100`, 'GET', undefined, identity)
assert.ok(!listed.data.some(row => row.id === resource.id), 'Deleted fixture must not reappear in a fresh list request')
}
function scopeIsEmpty(data, keys = knowledgeKeys) {
return keys.every(key => data[key] == null || Object.keys(data[key]).length === 0)
}
function manuscript(task) {
return { ...task.data, segments: [
{ id: 'accepted', title: 'Accepted fixture', agent_id: agent.id, status: 'accepted', content: 'Keep this accepted fixture prose.' },
{ id: 'skipped', title: 'Skipped fixture', agent_id: agent.id, status: 'skipped', content: 'Keep this skipped fixture prose.' },
], review: 'Keep the fixture review.', composed: 'Keep the fixture final manuscript.',
legacy_history: { fixture: prefix, opaque: { retained: true } }, custom_fixture_field: ['preserve', 123] }
}
async function checkDetached(before, actor) {
const after = await read(before)
assert.deepEqual(after.data, { ...before.data, flow_id: '' }, 'Detachment must preserve all other fields, including prose and accepted states')
assert.equal(after.owner_id, before.owner_id)
assert.equal(after.version, before.version + 1)
assert.equal(after.status, 'needs_configuration')
const revisions = JSON.parse(sql(`SELECT coalesce(json_agg(r ORDER BY version),'[]') FROM workbench_revisions r
WHERE tenant_id=${tenant} AND resource_id=${literal(before.id)} AND version IN (${before.version},${after.version});`))
const next = revisions.find(row => row.version === after.version)
assert.ok(next && next.actor_id === actor.user.id, 'Detachment revision must record the actual actor')
assert.deepEqual(next.data, after.data)
return after
}
async function browserPage(identity, resource) {
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } })
await context.addInitScript(({ auth, tenantID }) => {
for (const [key, value] of Object.entries({
weknora_user: JSON.stringify({ ...auth.user, tenant_id: String(auth.user.tenant_id) }),
weknora_tenant: JSON.stringify({ ...auth.active_tenant, id: String(tenantID) }),
weknora_token: auth.token, weknora_refresh_token: auth.refresh_token,
weknora_memberships: JSON.stringify(auth.memberships), 'weknora:new-user-guide-done:v1': '1',
})) localStorage.setItem(key, value)
}, { auth: identity, tenantID: tenant })
const page = await context.newPage()
page.setDefaultTimeout(20000)
page.on('pageerror', error => failures.push(`browser: ${error.message}`))
await page.route('**/api/v1/workbench/resources/**', async route => {
const request = route.request()
if (!['GET', 'HEAD'].includes(request.method())) {
const pathname = new URL(request.url()).pathname
const allowed = resources.some(row => pathname === '/api/v1' + resourcePath(row))
if (!allowed) {
failures.push(`blocked browser mutation outside recorded fixtures: ${pathname}`)
return route.abort()
}
}
return route.continue()
})
await page.goto(frontend + '/platform/workbench/flows')
const row = page.locator('.workbench-resource').filter({ hasText: resource.name })
await row.waitFor()
await row.click()
await page.getByRole('button', { name: '\u5220\u9664\u753b\u5e03', exact: true }).waitFor()
return { page, context }
}
async function deleteInBrowser(page, resource) {
const [response] = await Promise.all([
page.waitForResponse(value => value.request().method() === 'DELETE' &&
new URL(value.url()).pathname === '/api/v1' + resourcePath(resource)),
(async () => {
await page.getByRole('button', { name: '\u5220\u9664\u753b\u5e03', exact: true }).click()
await page.locator('.t-popconfirm:visible').getByRole('button', { name: /^(\u786e\u5b9a|\u786e\u8ba4)$/ }).click()
})(),
])
assert.equal(response.status(), 200)
const row = page.locator('.workbench-resource').filter({ hasText: resource.name })
await row.waitFor({ state: 'hidden' })
const [reloaded] = await Promise.all([
page.waitForResponse(value => value.request().method() === 'GET' &&
new URL(value.url()).pathname === '/api/v1/workbench/resources/flows'),
page.reload(),
])
assert.equal(reloaded.status(), 200)
await page.getByRole('heading', { name: '\u753b\u5e03\u7f16\u6392', exact: true }).waitFor()
await page.waitForLoadState('networkidle', { timeout: 30000 })
assert.equal(await row.count(), 0)
}
async function main() {
const credentials = JSON.parse(fs.readFileSync(path.join(directory, 'super-admin-login.json'), 'utf8'))
admin = await api('/auth/login', 'POST', { email: credentials.email, password: credentials.password }, null)
check('same_tenant_administrator_available', (await api('/access/me')).data.administrator === true)
role = (await api('/access/roles', 'POST', { name: prefix, permissions: [
'flows.read', 'flows.write', 'content.read', 'content.write', 'agents.read', 'agents.write', 'knowledge.read', 'knowledge.write',
] })).data
check('api_uses_expected_fixture_database', sql(`SELECT count(*) FROM workbench_access_roles
WHERE tenant_id=${tenant} AND id=${literal(role.id)} AND name=${literal(prefix)}`) === '1')
databaseVerified = true
const password = crypto.randomBytes(18).toString('base64url')
const email = `${prefix}@example.test`
user = (await api('/access/users', 'POST', { username: prefix, email, password, role: 'contributor', status: 'active', custom_role_id: role.id })).data
member = await api('/auth/login', 'POST', { email, password }, null)
check('fixture_member_is_not_an_administrator', (await api('/access/me', 'GET', undefined, member)).data.administrator === false)
const source = (await api('/agents')).data.find(row => row.config?.model_id)
assert.ok(source, 'An existing configured model ID is needed for the Agent selector; it will not be invoked')
kb = (await api('/knowledge-bases', 'POST', { name: prefix + '-knowledge', type: 'document',
indexing_strategy: { vector_enabled: false, keyword_enabled: false, wiki_enabled: false, graph_enabled: false },
}, member, [200, 201])).data
agent = (await api('/agents', 'POST', { name: prefix + '-agent', config: {
role: 'production', agent_mode: 'quick-answer', model_id: source.config.model_id,
kb_selection_mode: 'selected', knowledge_bases: [kb.id], system_prompt: 'Disposable canvas fixture. No generation requested.',
fusion_agent_ids: [], instruction_skill_ids: [],
} }, member, 201)).data
const staleKB = crypto.randomUUID()
const stale = { knowledge_base_ids: [staleKB], knowledge_ids: [crypto.randomUUID()],
knowledge_folders: { [staleKB]: ['removed-folder'] }, knowledge_tags: { [staleKB]: [crypto.randomUUID()] } }
const flowData = { nodes: [{ id: 'writer', label: 'Fixture writer', agent_id: agent.id }], edges: [], ...stale }
const own = await create('flows', 'own', flowData)
check('creating_canvas_removes_all_four_legacy_knowledge_fields', knowledgeKeys.every(key => !Object.hasOwn(own.data, key)))
seed(own, { ...own.data, ...stale })
const historicalScope = (await read(own, member)).data
assert.deepEqual(Object.fromEntries(knowledgeKeys.map(key => [key, historicalScope[key]])), stale)
let ownTask = await create('content-tasks', 'own-task', { source_text: 'Disposable outline', flow_id: own.id, ...stale })
assert.deepEqual(ownTask.data.knowledge_base_ids, [kb.id])
assert.ok(scopeIsEmpty(ownTask.data, knowledgeKeys.slice(1)))
assert.ok(ownTask.data.flow_snapshot && scopeIsEmpty(ownTask.data.flow_snapshot))
assert.equal(ownTask.data.agent_id, agent.id)
check('stale_canvas_and_task_scopes_are_replaced_by_node_agent_scope', true)
seed(ownTask, manuscript(ownTask))
ownTask = await read(ownTask, member)
await api(resourcePath(own), 'DELETE', undefined, member, 409)
if (browserEnabled) {
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
browser = await chromium.launch({ headless: true, channel: 'chrome' })
const { page, context } = await browserPage(member, own)
check('canvas_has_no_knowledge_selector', await page.locator('.knowledge-scope').count() === 0 &&
!/\u753b\u5e03\u77e5\u8bc6\u8303\u56f4/.test(await page.locator('.workbench-detail').innerText()))
const [saved] = await Promise.all([
page.waitForResponse(value => value.request().method() === 'PUT' && new URL(value.url()).pathname === '/api/v1' + resourcePath(own)),
page.getByRole('button', { name: '\u4fdd\u5b58', exact: true }).click(),
])
assert.equal(saved.status(), 200)
const savedData = (await read(own, member)).data
check('saving_canvas_removes_all_four_legacy_knowledge_fields', knowledgeKeys.every(key => !Object.hasOwn(savedData, key)))
await page.setViewportSize({ width: 390, height: 844 })
await page.screenshot({ path: path.join(directory, `${reportName}-mobile.png`), animations: 'disabled' })
check('canvas_mobile_has_no_horizontal_overflow', await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 2))
await page.setViewportSize({ width: 1440, height: 1000 })
await deleteInBrowser(page, own)
check('owner_delete_survives_browser_reload', true)
await context.close()
} else {
await api(resourcePath(own) + '?detach=true', 'DELETE', undefined, member)
}
await absent(own, member)
await checkDetached(ownTask, member)
check('owner_delete_detaches_safely_and_does_not_reappear', true)
const old = await create('flows', 'historical', flowData)
let oldTask = await create('content-tasks', 'historical-task', { source_text: 'Disposable historical outline', flow_id: old.id })
const blocker = await create('content-tasks', 'other-owner-reference', { source_text: 'Disposable blocker', flow_id: old.id })
seed(blocker, blocker.data, admin.user.id)
const blockerBefore = await read(blocker)
assert.equal(blockerBefore.owner_id, admin.user.id)
const orphanOwner = crypto.randomUUID()
assert.equal(sql(`SELECT count(*) FROM users WHERE id=${literal(orphanOwner)}`), '0')
seed(old, { ...old.data, legacy_import: { read_only: true, fixture: prefix } }, orphanOwner, 'needs_configuration')
seed(oldTask, manuscript(oldTask), orphanOwner, 'completed')
const historical = await read(old)
oldTask = await read(oldTask)
await api(resourcePath(old), 'PUT', { name: old.name + '-forbidden', data: historical.data, version: historical.version }, admin, [403, 404])
await api(resourcePath(old), 'DELETE', undefined, member, [403, 404])
await api(resourcePath(oldTask), 'DELETE', undefined, admin, [403, 404])
assert.deepEqual(await read(old), historical)
check('administrator_delete_permission_does_not_allow_editing_or_task_deletion', true)
await api(resourcePath(old) + '?detach=true', 'DELETE', undefined, admin, 409)
assert.deepEqual(await read(oldTask), oldTask)
assert.deepEqual(await read(blocker), blockerBefore)
check('another_owners_reference_blocks_deletion_without_changing_tasks', true)
await api(resourcePath(blocker), 'DELETE')
await api(resourcePath(old), 'DELETE', undefined, admin, 409)
if (browserEnabled) {
const { page, context } = await browserPage(admin, old)
check('administrator_sees_delete_but_not_save_for_historical_canvas',
await page.getByRole('button', { name: '\u5220\u9664\u753b\u5e03', exact: true }).isEnabled() &&
await page.getByRole('button', { name: '\u4fdd\u5b58', exact: true }).count() === 0)
await page.screenshot({ path: path.join(directory, `${reportName}-admin.png`), animations: 'disabled' })
await deleteInBrowser(page, old)
await context.close()
} else {
await api(resourcePath(old) + '?detach=true', 'DELETE')
}
await absent(old)
await checkDetached(oldTask, admin)
assert.equal(sql(`SELECT count(*) FROM workbench_events WHERE tenant_id=${tenant} AND resource_id=${literal(old.id)}
AND action='delete' AND owner_id=${literal(admin.user.id)}`), '1')
check('administrator_deletes_orphaned_canvas_preserving_manuscript_and_audit_actor', true)
foreignTenant = Number(sql(`INSERT INTO tenants(name,business,status)
VALUES (${literal(prefix)},'canvas-fixture','active') RETURNING id;`))
assert.ok(Number.isSafeInteger(foreignTenant) && foreignTenant > 0 && foreignTenant !== tenant)
const foreign = { id: crypto.randomUUID(), tenant_id: foreignTenant, owner_id: admin.user.id, kind: 'flows', name: prefix + '-foreign' }
resources.push(foreign)
sql(`INSERT INTO workbench_resources(id,tenant_id,owner_id,owner_name,kind,name,status,version,data,created_at,updated_at)
VALUES(${literal(foreign.id)},${foreignTenant},${literal(admin.user.id)},${literal(prefix)},'flows',${literal(foreign.name)},'draft',1,'{}',now(),now());`)
await api(resourcePath(foreign), 'GET', undefined, admin, 404)
await api(resourcePath(foreign) + '?detach=true', 'DELETE', undefined, admin, 404)
check('administrator_cannot_delete_same_owner_canvas_in_another_tenant',
sql(`SELECT count(*) FROM workbench_resources WHERE ${fixtureWhere(foreign)}`) === '1')
check('no_generation_jobs_or_model_usage', sql(`SELECT
(SELECT count(*) FROM workbench_jobs WHERE resource_id IN (${resources.map(row => literal(row.id)).join(',')})) +
(SELECT count(*) FROM workbench_usage WHERE tenant_id=${tenant} AND agent_id=${literal(agent.id)})`) === '0')
}
async function cleanup() {
async function attempt(label, action) {
try { await action() } catch (error) { failures.push(`cleanup ${label}: ${error.message}`) }
}
await attempt('browser', async () => { await browser?.close() })
if (databaseVerified) {
for (const resource of [...resources].reverse()) await attempt(resource.id, async () => {
assert.equal(sql(`SELECT count(*) FROM workbench_jobs WHERE resource_id=${literal(resource.id)} AND status IN ('queued','running')`), '0')
sql(`BEGIN;
DELETE FROM workbench_events WHERE tenant_id=${Number(resource.tenant_id)} AND resource_id=${literal(resource.id)};
DELETE FROM workbench_revisions WHERE tenant_id=${Number(resource.tenant_id)} AND resource_id=${literal(resource.id)};
DELETE FROM workbench_jobs WHERE tenant_id=${Number(resource.tenant_id)} AND resource_id=${literal(resource.id)};
DELETE FROM workbench_resources WHERE ${fixtureWhere(resource)};
COMMIT;`)
})
if (foreignTenant) await attempt('foreign tenant', async () => {
assert.equal(sql(`SELECT count(*) FROM workbench_resources WHERE tenant_id=${foreignTenant}`), '0')
sql(`DELETE FROM tenants WHERE id=${foreignTenant} AND name=${literal(prefix)} AND business='canvas-fixture';`)
})
}
if (agent) await attempt('agent', () => api(`/agents/${agent.id}?detach=true`, 'DELETE', undefined, member, [200, 404]))
if (kb) await attempt('knowledge', () => api(`/knowledge-bases/${kb.id}`, 'DELETE', undefined, member, [200, 202, 404]))
if (databaseVerified) await attempt('fixture publications', async () => {
const ids = [...resources.map(row => row.id), agent?.id, kb?.id].filter(Boolean)
if (ids.length) for (const table of ['workbench_access_grants', 'workbench_access_publications']) {
sql(`DELETE FROM ${table} WHERE tenant_id=${tenant} AND resource_id IN (${ids.map(literal).join(',')});`)
}
})
if (user) await attempt('user', () => api(`/access/users/${user.id}`, 'DELETE', undefined, admin, [200, 404]))
if (role) await attempt('role', () => api(`/access/roles/${role.id}`, 'DELETE', undefined, admin, [200, 404]))
if (databaseVerified && resources.length) await attempt('verify resources removed', async () => {
check('all_recorded_canvas_and_task_fixtures_removed', sql(`SELECT count(*) FROM workbench_resources
WHERE id IN (${resources.map(row => literal(row.id)).join(',')})`) === '0')
})
}
main().catch(error => { failures.push(error.message) }).finally(async () => {
await cleanup()
const report = { checked_at: new Date().toISOString(), origin, frontend, prefix, browser_enabled: browserEnabled,
checks, failures, passed: failures.length === 0, fixtures: { role_id: role?.id, user_id: user?.id, agent_id: agent?.id,
knowledge_base_id: kb?.id, foreign_tenant: foreignTenant, resources: resources.map(({ id, kind, tenant_id }) => ({ id, kind, tenant_id })) } }
fs.writeFileSync(path.join(directory, `${reportName}.json`), JSON.stringify(report, null, 2), { mode: 0o600 })
console.log(JSON.stringify(report, null, 2))
if (failures.length) process.exitCode = 1
})
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""Verify category migration through authenticated, read-only acceptance APIs."""
import argparse
import datetime
import json
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from weknora_category_etl import TENANT, save_json
BASE = "http://127.0.0.1:18080"
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--credentials", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
manifest = json.loads(args.manifest.read_text())
credentials = json.loads(args.credentials.read_text())
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
def request(path, token=None, payload=None):
headers = {"Content-Type": "application/json"}
if token:
headers["Authorization"] = "Bearer " + token
req = urllib.request.Request(BASE + path, data=json.dumps(payload).encode() if payload else None, headers=headers)
try:
with opener.open(req, timeout=20) as response:
return response.status, json.load(response)
except urllib.error.HTTPError as error:
return error.code, None
status, auth = request("/api/v1/auth/login", payload={k: credentials[k] for k in ("email", "password")})
if status != 200 or auth.get("active_tenant", {}).get("id") != TENANT:
raise RuntimeError("Verification login failed or outside migration tenant")
token = auth["token"]
report = {"checked_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "tenant_id": TENANT, "checks": []}
def check(name, passed, **counts):
report["checks"].append({"check": name, "passed": bool(passed), **counts})
for kb in manifest["knowledge_bases"]:
files = [f for f in manifest["files"] if f["knowledge_base_id"] == kb]
tags = [t for t in manifest["tags"] if t["knowledge_base_id"] == kb]
status, response = request(f"/api/v1/knowledge-bases/{kb}/knowledge/folders", token)
tree = response.get("data", {}) if response else {}
nodes = {}
def walk(items):
for node in items:
nodes[node["path"]] = node
walk(node.get("children") or [])
walk(tree.get("folders") or [])
counts_match = all(t["folder_path"] in nodes and nodes[t["folder_path"]]["total_count"] == sum(
f["folder_path"] == t["folder_path"] or f["folder_path"].startswith(t["folder_path"] + "/") for f in files) for t in tags)
check("folder_tree", status == 200 and tree.get("total_document_count") == len(files) and counts_match,
knowledge_base_id=kb, folders=len(nodes), documents=tree.get("total_document_count"))
status, response = request(f"/api/v1/knowledge-bases/{kb}/tags?page=1&page_size=100", token)
data = response.get("data", {}) if response else {}
items = data.get("data", [])
check("tag_list", status == 200 and {t["id"] for t in items} == {t["id"] for t in tags},
knowledge_base_id=kb, tags=len(items))
for tag in tags:
params = urllib.parse.urlencode({"page": 1, "page_size": 100, "tag_ids": tag["id"]})
status, response = request(f"/api/v1/knowledge-bases/{kb}/knowledge?{params}", token)
data = response.get("data", []) if response else []
if isinstance(data, dict):
data = data.get("items", [])
expected = {r["knowledge_id"] for r in manifest["relations"] if r["tag_id"] == tag["id"]}
actual = {f["id"] for f in data}
check("tag_document_scope", status == 200 and actual == expected,
legacy_category_id=tag["legacy_id"], expected=len(expected), returned=len(actual))
status, _ = request(f"/api/v1/knowledge-bases/{kb}/knowledge/folders")
check("anonymous_denied", status in (401, 403), knowledge_base_id=kb)
report["passed"] = all(c["passed"] for c in report["checks"])
save_json(args.out, report)
print(json.dumps({"passed": report["passed"], "checks": len(report["checks"]),
"failed": [c for c in report["checks"] if not c["passed"]], "out": str(args.out)}))
return 0 if report["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())
+103
View File
@@ -0,0 +1,103 @@
const assert = require('node:assert/strict')
const fs = require('node:fs')
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
const origin = process.env.VERIFY_FRONTEND_ORIGIN || 'http://127.0.0.1:15173'
const apiOrigin = process.env.VERIFY_API_ORIGIN || 'http://127.0.0.1:18080'
async function main() {
const credentials = JSON.parse(fs.readFileSync(`${directory}/super-admin-login.json`, 'utf8'))
const response = await fetch(`${apiOrigin}/api/v1/auth/login`, {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ email: credentials.email, password: credentials.password }),
})
assert.equal(response.status, 200)
const auth = await response.json()
const browser = await chromium.launch({ headless: true, channel: 'chrome' })
const checks = [], errors = [], blockedWrites = [], writes = []
const check = (name, passed) => { checks.push({ name, passed: !!passed }); assert.ok(passed, name) }
try {
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } })
await context.addInitScript(a => {
const entries = {
weknora_user: JSON.stringify({ ...a.user, tenant_id: String(a.user.tenant_id) }),
weknora_tenant: JSON.stringify({ ...a.active_tenant, id: '10059' }),
weknora_token: a.token, weknora_refresh_token: a.refresh_token,
weknora_memberships: JSON.stringify(a.memberships), 'weknora:new-user-guide-done:v1': '1',
}
for (const [key, value] of Object.entries(entries)) localStorage.setItem(key, value)
}, auth)
const fixture = (id, kind, name, data) => ({ id, kind, name, data, tenant_id: 10059, owner_id: auth.user.id, owner_name: 'UI verification', status: kind === 'training-maps' ? 'confirmed' : 'draft', version: 1, created_at: new Date().toISOString(), updated_at: new Date().toISOString() })
const map = fixture('duration-map', 'training-maps', '时长验证导图', { days: [{ day: 1, title: '基础与实践' }], agent_id: 'duration-agent' })
const flow = fixture('duration-flow', 'flows', '时长验证画布', { nodes: [{ id: 'node', label: '内容作者', agent_id: 'duration-agent' }], edges: [] })
let task = null
// All business writes below are in-memory fixtures; no task or AI job reaches the API.
await context.route('**/api/v1/**', async route => {
const request = route.request(), path = new URL(request.url()).pathname, method = request.method()
const ok = body => route.fulfill({ status: 200, json: { success: true, ...body } })
if (path === '/api/v1/access/me') return ok({ data: { administrator: false, restricted: true, permissions: ['content.read', 'content.write', 'flows.read'] } })
const match = path.match(/^\/api\/v1\/workbench\/resources\/(content-tasks|training-maps|flows)(?:\/([^/]+))?(?:\/(jobs|actions\/.*))?$/)
if (match) {
const [, kind, id, action] = match
if (method === 'GET' && action === 'jobs') return ok({ data: [], total: 0 })
if (method === 'GET') {
const items = kind === 'training-maps' ? [map] : kind === 'flows' ? [flow] : task ? [task] : []
return ok(id ? { data: items.find(item => item.id === id) } : { data: items, total: items.length })
}
if (kind === 'content-tasks' && !action && ['POST', 'PUT'].includes(method)) {
const body = request.postDataJSON()
writes.push({ method, data: body.data })
task = { ...fixture('duration-task', kind, body.name, body.data), version: (task?.version || 0) + 1 }
return ok({ data: task })
}
}
if (!['GET', 'HEAD', 'OPTIONS'].includes(method)) {
blockedWrites.push({ path, method })
return route.fulfill({ status: 409, json: { error: 'Verification prevents live writes' } })
}
return route.continue()
})
const page = await context.newPage()
page.on('pageerror', error => errors.push(error.message))
await page.goto(`${origin}/platform/workbench/content?tab=tasks`)
await page.getByRole('button', { name: '创建内容任务', exact: true }).click()
const drawer = page.locator('.t-drawer--open')
const duration = drawer.locator('[aria-label="直播时长(分钟)"] input')
await duration.waitFor()
check('new_task_defaults_to_60_minutes', await duration.inputValue() === '60')
await drawer.locator('.t-form__item').filter({ has: page.locator('.t-form__label').filter({ hasText: /^名称$/ }) }).locator('input').fill('直播时长验证任务')
await duration.fill('135')
await duration.press('Tab')
await drawer.locator('.t-form__item').filter({ hasText: '已确认的训练营导图' }).locator('.t-select').click()
await page.getByText('时长验证导图', { exact: true }).last().click()
await drawer.locator('.t-form__item').filter({ hasText: '画布模板' }).locator('.t-select').click()
await page.getByText('时长验证画布', { exact: true }).last().click()
await page.screenshot({ path: `${directory}/content-duration-desktop.png`, animations: 'disabled' })
await drawer.getByRole('button', { name: '保存草稿', exact: true }).click()
await page.getByText('直播时长:135 分钟', { exact: true }).waitFor()
check('draft_payload_and_detail_keep_duration', writes[0]?.data.duration_minutes === 135)
await page.getByRole('button', { name: '编辑配置', exact: true }).click()
await duration.waitFor()
check('editing_reopens_saved_duration', await duration.inputValue() === '135')
await page.setViewportSize({ width: 390, height: 844 })
await duration.scrollIntoViewIfNeeded()
await page.screenshot({ path: `${directory}/content-duration-mobile.png`, animations: 'disabled' })
check('duration_control_fits_mobile', await duration.evaluate(element => { const r = element.getBoundingClientRect(); return r.left >= 0 && r.right <= innerWidth }))
check('drawer_fits_mobile', await drawer.locator('.t-drawer__content-wrapper').evaluate(element => { const r = element.getBoundingClientRect(); return r.left >= -1 && r.right <= innerWidth + 1 }))
await duration.fill('225')
await duration.press('Tab')
await drawer.getByRole('button', { name: '保存配置', exact: true }).click()
await page.getByText('直播时长:225 分钟', { exact: true }).waitFor()
check('edited_duration_saved', writes.at(-1)?.data.duration_minutes === 225)
check('no_live_business_or_ai_writes', blockedWrites.length === 0 && writes.length === 2)
check('no_runtime_errors', errors.length === 0)
fs.writeFileSync(`${directory}/content-duration-ui.json`, JSON.stringify({ origin, checks, errors, businessWrites: 'mocked in browser only' }, null, 2))
console.log(JSON.stringify({ checks, errors }, null, 2))
} catch (error) {
console.error(error.stack)
process.exitCode = 1
} finally {
await browser.close()
}
}
main().catch(error => { console.error(error.stack); process.exitCode = 1 })
@@ -0,0 +1,62 @@
const assert = require('node:assert/strict')
const fs = require('node:fs')
const path = require('node:path')
const crypto = require('node:crypto')
const { execFileSync } = require('node:child_process')
const schema = `content_permissions_${crypto.randomBytes(6).toString('hex')}`
const migration = fs.readFileSync(path.join(__dirname, '../migrations/versioned/000207_content_action_permissions.up.sql'), 'utf8')
const checks = []
const literal = value => `'${String(value).replaceAll("'", "''")}'`
function sql(query) {
return execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres',
'psql', '-X', '-q', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-At', '-v', 'ON_ERROR_STOP=1'],
{ input: `SET search_path TO ${schema}, public;\n${query}`, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim()
}
function permissions(id) { return JSON.parse(sql(`SELECT permissions FROM workbench_access_roles WHERE id=${literal(id)}`)) }
function check(name, actual, expected) { assert.deepEqual(actual, expected, name); checks.push(name) }
function legacyRole(id, sourcePermissions, nativePermissions = [], version = 1) {
const spec = { id, source_role: id, name: id, source_permissions: sourcePermissions, permissions: nativePermissions, review_notes: [] }
return `INSERT INTO workbench_access_roles(id,tenant_id,name,permissions,version) VALUES(${literal(id)},10059,${literal(id)},${literal(JSON.stringify(nativePermissions))},${version});
INSERT INTO yawan_legacy_access_mappings(source_system,target_tenant_id,entity_key,source_spec,plan_sha256)
VALUES('yawan-workbench',10059,${literal('role/' + id)},${literal(JSON.stringify(spec))},'test');`
}
try {
sql(`CREATE SCHEMA ${schema};
CREATE TABLE ${schema}.workbench_access_roles (LIKE public.workbench_access_roles INCLUDING ALL);
CREATE TABLE ${schema}.yawan_legacy_access_mappings (LIKE public.yawan_legacy_access_mappings INCLUDING ALL);
CREATE TABLE ${schema}.workbench_content_permission_migrations(version integer PRIMARY KEY, changes jsonb NOT NULL, created_at timestamptz NOT NULL DEFAULT now());
INSERT INTO workbench_access_roles(id,tenant_id,name,permissions) VALUES
('producer',10059,'producer','["content.read","content.write"]'),
('reader',10059,'reader','["content.read"]'),
('other-module',10059,'other-module','["agents.read"]');
${legacyRole('reviewer', ['audit:read', 'read', 'review'])}
${legacyRole('viewer', ['read'])}
${legacyRole('edited-reviewer', ['review'], [], 2)}
${legacyRole('editor', ['tasks:review', 'tasks:write', 'tasks:deliver'], ['content.read', 'content.write'])}
${legacyRole('changed-reviewer', ['review'])}
UPDATE workbench_access_roles SET permissions='["agents.read"]' WHERE id='changed-reviewer';`)
const identityBefore = sql(`SELECT md5(coalesce(jsonb_agg(to_jsonb(u) ORDER BY id)::text,'')) FROM public.users u`)
const membershipBefore = sql(`SELECT md5(coalesce(jsonb_agg(to_jsonb(m) ORDER BY id)::text,'')) FROM public.tenant_members m`)
sql(`BEGIN; ${migration} COMMIT;`)
check('native_producer_preserves_review_and_delivery', permissions('producer'), ['content.deliver', 'content.read', 'content.review', 'content.write'])
check('native_reader_preserves_delivery_without_review', permissions('reader'), ['content.deliver', 'content.read'])
check('legacy_reviewer_restored_without_generation_or_delivery', permissions('reviewer'), ['content.read', 'content.review'])
check('generic_legacy_read_does_not_invent_task_access', permissions('viewer'), [])
check('edited_import_role_is_preserved', permissions('edited-reviewer'), [])
check('changed_import_permissions_are_preserved', permissions('changed-reviewer'), ['agents.read'])
check('legacy_editor_preserves_all_existing_content_actions', permissions('editor'), ['content.deliver', 'content.read', 'content.review', 'content.write'])
check('unrelated_module_unchanged', permissions('other-module'), ['agents.read'])
check('original_legacy_ledger_is_not_rewritten', JSON.parse(sql(`SELECT source_spec->'permissions' FROM yawan_legacy_access_mappings WHERE entity_key='role/reviewer'`)), [])
check('backfill_provenance_records_new_mapping', JSON.parse(sql(`SELECT c->'mapped_permissions' FROM workbench_content_permission_migrations,jsonb_array_elements(changes)c WHERE c->>'role_id'='reviewer'`)), ['content.read', 'content.review'])
sql(`UPDATE workbench_access_roles SET permissions='[]',version=version+1 WHERE id='reviewer';
INSERT INTO workbench_access_roles(id,tenant_id,name,permissions) VALUES('new-reader',10059,'new-reader','["content.read"]');
BEGIN; ${migration} COMMIT;`)
check('rerun_preserves_explicit_revocation', permissions('reviewer'), [])
check('rerun_does_not_expand_post_migration_roles', permissions('new-reader'), ['content.read'])
check('users_unchanged', sql(`SELECT md5(coalesce(jsonb_agg(to_jsonb(u) ORDER BY id)::text,'')) FROM public.users u`), identityBefore)
check('memberships_unchanged', sql(`SELECT md5(coalesce(jsonb_agg(to_jsonb(m) ORDER BY id)::text,'')) FROM public.tenant_members m`), membershipBefore)
console.log(JSON.stringify({ ok: true, checks }, null, 2))
} finally {
sql(`DROP SCHEMA IF EXISTS ${schema} CASCADE;`)
}

Some files were not shown because too many files have changed in this diff Show More