WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes
Anydoc / anydoc parser engine (push) Canceled after 0s
App / Format, vet, test, and build (push) Canceled after 0s
CLI / test (macos-latest / go 1.26) (push) Canceled after 0s
CLI / test (ubuntu-latest / go 1.26) (push) Canceled after 0s
CLI / test (windows-latest / go 1.26) (push) Canceled after 0s
Build and Push Docker Image / build-ui (push) Canceled after 0s
Build and Push Docker Image / build-docreader (push) Canceled after 0s
Build and Push Docker Image / build-sandbox (push) Canceled after 0s
Build and Push Docker Image / build-app (amd64, linux/amd64, ubuntu-latest) (push) Canceled after 0s
Build and Push Docker Image / build-app (arm64, linux/arm64, ubuntu-24.04-arm) (push) Canceled after 0s
DocReader / Python tests and Go client tests (push) Canceled after 0s
Build & Publish (dsh-weknora) / Typecheck, test, pack (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (pinned) (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (latest, advisory) (push) Canceled after 0s
Frontend / Test, type-check, and build (push) Canceled after 0s
Go Lint Cache / warm lint cache (.) (push) Canceled after 0s
Go Lint Cache / warm lint cache (cli) (push) Canceled after 0s
Go Lint Cache / warm lint cache (client) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.10) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.11) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.12) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.13) (push) Canceled after 0s
Build and Push Docker Image / merge (push) Canceled after 0s
Build & Publish (dsh-weknora) / Publish to npm (push) Canceled after 0s
Build & Publish (mcp-server) / Build sdist + wheel (push) Canceled after 0s
Build & Publish (mcp-server) / Publish to PyPI (push) Canceled after 0s
Anydoc / anydoc parser engine (push) Canceled after 0s
App / Format, vet, test, and build (push) Canceled after 0s
CLI / test (macos-latest / go 1.26) (push) Canceled after 0s
CLI / test (ubuntu-latest / go 1.26) (push) Canceled after 0s
CLI / test (windows-latest / go 1.26) (push) Canceled after 0s
Build and Push Docker Image / build-ui (push) Canceled after 0s
Build and Push Docker Image / build-docreader (push) Canceled after 0s
Build and Push Docker Image / build-sandbox (push) Canceled after 0s
Build and Push Docker Image / build-app (amd64, linux/amd64, ubuntu-latest) (push) Canceled after 0s
Build and Push Docker Image / build-app (arm64, linux/arm64, ubuntu-24.04-arm) (push) Canceled after 0s
DocReader / Python tests and Go client tests (push) Canceled after 0s
Build & Publish (dsh-weknora) / Typecheck, test, pack (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (pinned) (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (latest, advisory) (push) Canceled after 0s
Frontend / Test, type-check, and build (push) Canceled after 0s
Go Lint Cache / warm lint cache (.) (push) Canceled after 0s
Go Lint Cache / warm lint cache (cli) (push) Canceled after 0s
Go Lint Cache / warm lint cache (client) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.10) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.11) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.12) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.13) (push) Canceled after 0s
Build and Push Docker Image / merge (push) Canceled after 0s
Build & Publish (dsh-weknora) / Publish to npm (push) Canceled after 0s
Build & Publish (mcp-server) / Build sdist + wheel (push) Canceled after 0s
Build & Publish (mcp-server) / Publish to PyPI (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
FROM yawan-weknora-app:native-final2
|
||||
COPY WeKnora-native-acceptance-next /app/WeKnora
|
||||
COPY migrations /app/migrations
|
||||
@@ -0,0 +1,4 @@
|
||||
*
|
||||
!WeKnora-native-acceptance-next
|
||||
!migrations/
|
||||
!migrations/**
|
||||
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Back up, rehearse and import operational history into local acceptance."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from apply_weknora_identities import sql_run as mapping_sql
|
||||
from weknora_asset_etl import LEDGER as ASSET_LEDGER
|
||||
from weknora_business_etl import LEDGER as BUSINESS_LEDGER
|
||||
from weknora_category_etl import save_json
|
||||
from weknora_etl import consistent_snapshot
|
||||
from weknora_identity_etl import IDENTITY_LEDGER, TARGET_DATABASE, TARGET_TENANT
|
||||
from weknora_operations_etl import LEDGER, TABLES, operations_plan, operations_sql, safe_operations_summary
|
||||
|
||||
PRESERVED = ("workbench_resources", "workbench_jobs", "workbench_revisions", "workbench_balance_holds")
|
||||
DOCKER = ["docker", "--context", "colima-yawan", "exec", "-i", "WeKnora-postgres"]
|
||||
|
||||
|
||||
def sql_run(query):
|
||||
result = subprocess.run(DOCKER + ["psql", "-X", "-qAt", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", TARGET_DATABASE],
|
||||
input=query, text=True, capture_output=True, timeout=600)
|
||||
if result.returncode:
|
||||
# Do not expose source text included in SQL error contexts.
|
||||
lines = [line for line in result.stderr.splitlines() if line.startswith("ERROR:")]
|
||||
error = lines[0][:180] if lines and "Operations " in lines[0] else "database rejected operations transaction"
|
||||
raise RuntimeError(error)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def digest(path):
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def quiescent(path):
|
||||
if any(Path(str(path) + suffix).exists() for suffix in ("-wal", "-journal")):
|
||||
raise RuntimeError("Operations fallback requires no source WAL or journal")
|
||||
check = subprocess.run(["lsof", "-t", str(path)], capture_output=True, text=True, timeout=15)
|
||||
if check.returncode != 1 or check.stdout.strip():
|
||||
raise RuntimeError("Operations fallback requires no process holding the source database")
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def source_snapshot(path):
|
||||
"""Use SQLite backup; compensate for missing WAL sidecars on a quiet source.
|
||||
|
||||
Some SQLite builds cannot read a WAL-mode file read-only when sidecars are
|
||||
absent. Only a byte-identical, verified quiet copy is opened read-write to
|
||||
initialize its temporary sidecars. The original is never opened for writing.
|
||||
"""
|
||||
path = path.resolve()
|
||||
manager = consistent_snapshot(path)
|
||||
try:
|
||||
db, info = manager.__enter__()
|
||||
except sqlite3.OperationalError as error:
|
||||
if "unable to open database file" not in str(error):
|
||||
raise
|
||||
else:
|
||||
try:
|
||||
yield db, info
|
||||
finally:
|
||||
manager.__exit__(None, None, None)
|
||||
return
|
||||
quiescent(path)
|
||||
before = path.stat()
|
||||
with tempfile.TemporaryDirectory(prefix="operations-frozen-source-") as root:
|
||||
copied = Path(root) / "source.sqlite"
|
||||
shutil.copyfile(path, copied)
|
||||
fingerprint = digest(copied)
|
||||
if digest(path) != fingerprint or before.st_mtime_ns != path.stat().st_mtime_ns:
|
||||
raise RuntimeError("Operations source changed during quiet copy")
|
||||
quiescent(path)
|
||||
with sqlite3.connect(copied) as initializer:
|
||||
initializer.execute("SELECT count(*) FROM sqlite_master").fetchone()
|
||||
with consistent_snapshot(copied) as (db, info):
|
||||
info.update(snapshot_method="verified_quiescent_copy_then_sqlite_backup", source_bytes=before.st_size,
|
||||
source_mtime_ns=before.st_mtime_ns, original_file_sha256=fingerprint)
|
||||
yield db, info
|
||||
quiescent(path)
|
||||
if (before.st_size, before.st_mtime_ns) != (path.stat().st_size, path.stat().st_mtime_ns) or digest(path) != fingerprint:
|
||||
raise RuntimeError("Operations source changed during snapshot read")
|
||||
|
||||
|
||||
def build_plan(db, mapping_file):
|
||||
ids = json.loads(mapping_sql(f"SELECT json_object_agg(source_user_id,target_user_id) FROM {IDENTITY_LEDGER} WHERE source_system='yawan-workbench' AND source_organization_id=1 AND target_tenant_id={TARGET_TENANT};"))
|
||||
agents = json.loads(mapping_sql(f"SELECT json_object_agg(source_id,target_id) FROM {ASSET_LEDGER} WHERE source_system='yawan-workbench' AND source_organization_id=1 AND source_kind='agent' AND target_tenant_id={TARGET_TENANT};"))
|
||||
business = json.loads(mapping_sql(f"SELECT json_object_agg(source_kind||':'||source_id,target_id) FROM {BUSINESS_LEDGER} WHERE source_system='yawan-workbench' AND source_organization_id=1 AND target_tenant_id={TARGET_TENANT};"))
|
||||
model_rows = json.loads(mapping_sql(f"SELECT coalesce(json_agg(json_build_object('provider_id',source_provider_id,'snapshot',source_snapshot,'target_id',target_model_id)),'[]') FROM yawan_legacy_model_mappings WHERE source_system='yawan-workbench' AND source_organization_id=1 AND target_tenant_id={TARGET_TENANT};"))
|
||||
models = {}
|
||||
for row in model_rows:
|
||||
name = row["snapshot"].get("model_name", row["snapshot"].get("model", ""))
|
||||
if name:
|
||||
models[f"{row['provider_id']}:{name}"] = row["target_id"]
|
||||
state = json.loads(mapping_file.read_text())
|
||||
if state.get("tenant_id") != TARGET_TENANT:
|
||||
raise ValueError("Operations KB mapping tenant differs")
|
||||
return operations_plan(db, ids, agents, {k: v["id"] for k, v in state["knowledge_bases"].items()}, business, models)
|
||||
|
||||
|
||||
def target_snapshot():
|
||||
tables = (*TABLES, "workbench_history", *PRESERVED)
|
||||
entries = ["'database',current_database()"]
|
||||
for table in tables:
|
||||
key = "tenant_id::text||':'||user_id" if table == "workbench_balances" else "id"
|
||||
entries.append(f"'{table}',(SELECT coalesce(json_agg(json_build_object('id',{key},'hash',md5(to_jsonb(r)::text)) ORDER BY {key}),'[]') FROM {table} r)")
|
||||
return json.loads(sql_run("SELECT json_build_object(" + ",".join(entries) + ");"))
|
||||
|
||||
|
||||
def backup(path):
|
||||
if path.exists():
|
||||
raise RuntimeError("Operations backup exists")
|
||||
args = DOCKER + ["pg_dump", "-U", "postgres", "-d", TARGET_DATABASE, "--format=custom"]
|
||||
for table in (*TABLES, "workbench_history", *PRESERVED):
|
||||
args.extend(["--table", table])
|
||||
if mapping_sql(f"SELECT to_regclass('{LEDGER}') IS NOT NULL;") == "t":
|
||||
args.extend(["--table", LEDGER])
|
||||
with path.open("xb") as output:
|
||||
os.chmod(path, 0o600)
|
||||
result = subprocess.run(args, stdout=output, stderr=subprocess.PIPE, timeout=120)
|
||||
if result.returncode or not path.stat().st_size:
|
||||
raise RuntimeError("Operations target backup failed")
|
||||
os.chmod(path, 0o400)
|
||||
return {"path": str(path), "bytes": path.stat().st_size, "sha256": digest(path)}
|
||||
|
||||
|
||||
def export_plan(path, plan):
|
||||
with path.open("x") as out:
|
||||
os.chmod(path, 0o600)
|
||||
json.dump(plan, out, ensure_ascii=True, sort_keys=True, separators=(",", ":"))
|
||||
os.chmod(path, 0o400)
|
||||
return {"path": str(path), "bytes": path.stat().st_size, "sha256": digest(path), "mode": "0400"}
|
||||
|
||||
|
||||
def transaction(query, commit=False):
|
||||
sql = "BEGIN ISOLATION LEVEL SERIALIZABLE; SET LOCAL statement_timeout='180s'; SET LOCAL lock_timeout='5s';\n"
|
||||
for i, table in enumerate((*TABLES, "workbench_history", *PRESERVED)):
|
||||
sql += f"CREATE TEMP TABLE _ops_before_{i} ON COMMIT DROP AS SELECT * FROM {table};\n"
|
||||
sql += query
|
||||
for i, table in enumerate((*TABLES, "workbench_history", *PRESERVED)):
|
||||
if table == "workbench_balances":
|
||||
join = "n.tenant_id=b.tenant_id AND n.user_id=b.user_id"
|
||||
else:
|
||||
join = "n.id=b.id"
|
||||
assertion = f"EXISTS (SELECT 1 FROM _ops_before_{i} b LEFT JOIN {table} n ON {join} WHERE to_jsonb(n) IS DISTINCT FROM to_jsonb(b))"
|
||||
if table in PRESERVED:
|
||||
assertion += f" OR (SELECT count(*) FROM {table})<>(SELECT count(*) FROM _ops_before_{i})"
|
||||
sql += f"DO $preserve$ BEGIN IF {assertion} THEN RAISE EXCEPTION 'Operations existing data changed in {table}'; END IF; END $preserve$;\n"
|
||||
return sql + ("COMMIT;" if commit else "ROLLBACK;")
|
||||
|
||||
|
||||
def verify(plan, before, after):
|
||||
checks = {"acceptance_only": before["database"] == after["database"] == TARGET_DATABASE}
|
||||
for table in (*TABLES, "workbench_history", *PRESERVED):
|
||||
old = {r["id"]: r["hash"] for r in before[table]}
|
||||
new = {r["id"]: r["hash"] for r in after[table]}
|
||||
checks[table + "_existing_unchanged"] = all(new.get(k) == v for k, v in old.items())
|
||||
if table in PRESERVED:
|
||||
checks[table + "_no_new"] = old == new
|
||||
else:
|
||||
if table == "workbench_history":
|
||||
expected = {i["history_id"] for i in plan["items"]}
|
||||
else:
|
||||
expected = {f"{TARGET_TENANT}:{i['target_key']}" if table == "workbench_balances" else i["target_key"]
|
||||
for i in plan["items"] if i["target_table"] == table}
|
||||
checks[table + "_planned_additions_only"] = set(new) - set(old) == expected - set(old)
|
||||
checks[table + "_all_planned_present"] = expected <= set(new)
|
||||
return {"passed": all(checks.values()), "checks": checks}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("command", choices=("plan", "apply", "repeat"))
|
||||
parser.add_argument("--sqlite", type=Path)
|
||||
parser.add_argument("--kb-mapping", type=Path)
|
||||
parser.add_argument("--export", type=Path)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.out.exists():
|
||||
parser.error("Operations checkpoint already exists")
|
||||
if args.command == "repeat":
|
||||
plan = json.loads(args.export.read_text())
|
||||
info = {"sha256": plan["source_snapshot_sha256"], "source_mutated": False, "snapshot_method": "frozen_verified_export"}
|
||||
else:
|
||||
with source_snapshot(args.sqlite) as (db, info):
|
||||
plan = build_plan(db, args.kb_mapping)
|
||||
plan["source_snapshot_sha256"] = info["sha256"]
|
||||
if info["source_mutated"]:
|
||||
raise RuntimeError("Operations source snapshot changed")
|
||||
query = operations_sql(plan)
|
||||
result = {"checked_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "summary": safe_operations_summary(plan),
|
||||
"snapshot": info, "committed": False}
|
||||
if args.command != "repeat":
|
||||
result["private_export"] = export_plan(args.out.with_suffix(".private.json"), plan)
|
||||
save_json(args.out.with_suffix(".manifest.json"), result)
|
||||
result["backup"] = backup(args.out.with_suffix(".backup.dump"))
|
||||
before = target_snapshot()
|
||||
save_json(args.out.with_suffix(".before.json"), before)
|
||||
result["rehearsal"] = [json.loads(s) for s in sql_run(transaction(query)).splitlines() if s.startswith("{")]
|
||||
result["rollback_unchanged"] = before == target_snapshot()
|
||||
if not result["rollback_unchanged"]:
|
||||
raise RuntimeError("Operations rollback rehearsal changed target")
|
||||
if args.command in ("apply", "repeat"):
|
||||
save_json(args.out.with_suffix(".attempt.json"), result)
|
||||
result["applied"] = [json.loads(s) for s in sql_run(transaction(query, True)).splitlines() if s.startswith("{")]
|
||||
result["committed"] = True
|
||||
save_json(args.out.with_suffix(".committed.json"), result)
|
||||
after = target_snapshot()
|
||||
save_json(args.out.with_suffix(".after.json"), after)
|
||||
result["verification"] = verify(plan, before, after)
|
||||
if args.command == "repeat":
|
||||
result["repeat_no_change"] = before == after and all(r.get("inserted_history") == 0 and r.get("inserted_native") == 0 for r in result["applied"])
|
||||
save_json(args.out, result)
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
return 0 if result.get("verification", {}).get("passed", True) and result.get("repeat_no_change", True) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Apply a reviewed access plan only to the local acceptance database."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from apply_weknora_identities import sql_run
|
||||
from weknora_access_etl import ACCESS_TABLES, LEDGER, SOURCE, canonical, migration_sql
|
||||
from weknora_category_etl import save_json
|
||||
from weknora_identity_etl import TARGET_DATABASE, TARGET_TENANT
|
||||
|
||||
|
||||
def snapshot():
|
||||
fields = ["'database',current_database()",
|
||||
"""'users',(SELECT coalesce(json_agg(json_build_object('id',id,'fingerprint',md5(to_jsonb(u)::text)) ORDER BY id),'[]') FROM users u)""",
|
||||
"""'memberships',(SELECT coalesce(json_agg(json_build_object('id',id,'fingerprint',md5(to_jsonb(m)::text)) ORDER BY id),'[]') FROM tenant_members m)"""]
|
||||
fields += [f"'{table}',(SELECT coalesce(json_agg(to_jsonb(a) ORDER BY to_jsonb(a)::text),'[]') FROM {table} a)" for table in ACCESS_TABLES]
|
||||
result = json.loads(sql_run("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; SELECT json_build_object(" + ",".join(fields) + "); COMMIT;"))
|
||||
exists = sql_run(f"SELECT to_regclass('{LEDGER}') IS NOT NULL;") == "t"
|
||||
result[LEDGER] = json.loads(sql_run(f"SELECT coalesce(json_agg(to_jsonb(a) ORDER BY entity_key),'[]') FROM {LEDGER} a;")) if exists else []
|
||||
return result
|
||||
|
||||
|
||||
def verification(plan, before, after):
|
||||
checks = {"acceptance_scope": before["database"] == after["database"] == TARGET_DATABASE,
|
||||
"all_users_unchanged": before["users"] == after["users"],
|
||||
"all_memberships_unchanged": before["memberships"] == after["memberships"],
|
||||
"all_explicit_grants_unchanged": before[ACCESS_TABLES[3]] == after[ACCESS_TABLES[3]]}
|
||||
expected = {
|
||||
ACCESS_TABLES[0]: {r["id"] for r in plan["roles"]},
|
||||
ACCESS_TABLES[1]: {(TARGET_TENANT, a["user_id"]) for a in plan["assignments"]},
|
||||
ACCESS_TABLES[2]: {(TARGET_TENANT, p["kind"], p["resource_id"]) for p in plan["publications"]},
|
||||
}
|
||||
def key(table, row):
|
||||
if table == ACCESS_TABLES[0]:
|
||||
return row["id"]
|
||||
return (row["tenant_id"], row["user_id"]) if table == ACCESS_TABLES[1] else (row["tenant_id"], row["kind"], row["resource_id"])
|
||||
for table in expected:
|
||||
old, new = ({key(table, r): r for r in state[table]} for state in (before, after))
|
||||
checks[table + "_existing_unchanged"] = all(new.get(k) == r for k, r in old.items())
|
||||
checks[table + "_planned_only"] = set(new) - set(old) == expected[table] - set(old)
|
||||
saved = {r["entity_key"]: r["source_spec"] for r in after[LEDGER]
|
||||
if r["source_system"] == SOURCE and r["target_tenant_id"] == TARGET_TENANT}
|
||||
planned = {"role/" + r["source_role"]: r for r in plan["roles"]}
|
||||
planned.update({"assignment/" + a["user_id"]: a for a in plan["assignments"]})
|
||||
planned.update({"publication/" + p["kind"] + "/" + p["resource_id"]: p for p in plan["publications"]})
|
||||
checks["durable_provenance"] = all(saved.get(k) == v for k, v in planned.items())
|
||||
return {"passed": all(checks.values()), "checks": checks}
|
||||
|
||||
|
||||
def rollback_sql(plan, before, after, commit=False):
|
||||
migration_sql(plan) # Reuse the scope and plan-integrity validation.
|
||||
inserted = {table: [r for r in after[table] if r not in before[table]] for table in (*ACCESS_TABLES, LEDGER)}
|
||||
payload = "'" + canonical(inserted).replace("'", "''") + "'::jsonb"
|
||||
parts = [f"""-- Removes only unchanged rows inserted by this reviewed application.
|
||||
-- Default rehearsal. Emit COMMIT only after reviewing the current native edits.
|
||||
BEGIN ISOLATION LEVEL SERIALIZABLE;
|
||||
SET LOCAL statement_timeout='30s'; SET LOCAL lock_timeout='5s';
|
||||
DO $scope$ BEGIN
|
||||
IF current_database()<>'{TARGET_DATABASE}' THEN RAISE EXCEPTION 'Wrong rollback database'; END IF;
|
||||
END $scope$;
|
||||
SELECT pg_advisory_xact_lock({TARGET_TENANT},11905);
|
||||
LOCK TABLE users,tenant_members,workbench_access_roles,workbench_access_assignments,
|
||||
workbench_access_publications,workbench_access_grants,{LEDGER} IN SHARE ROW EXCLUSIVE MODE;
|
||||
DO $suspended$ BEGIN
|
||||
IF EXISTS(SELECT 1 FROM users u JOIN yawan_legacy_identity_mappings l ON l.target_user_id=u.id
|
||||
JOIN tenant_members m ON m.user_id=u.id
|
||||
WHERE l.target_tenant_id={TARGET_TENANT} AND l.source_user_id<>1
|
||||
AND (u.is_active OR m.status<>'suspended' OR m.role<>'contributor' OR m.tenant_id<>{TARGET_TENANT}))
|
||||
THEN RAISE EXCEPTION 'Account activation changed; review rollback policy first'; END IF;
|
||||
END $suspended$;
|
||||
CREATE TEMP TABLE _access_rollback_payload ON COMMIT DROP AS SELECT {payload} AS data;"""]
|
||||
for table in (*ACCESS_TABLES, LEDGER):
|
||||
parts.append(f"""DO $unchanged$ BEGIN
|
||||
IF EXISTS(SELECT 1 FROM _access_rollback_payload p, jsonb_array_elements(p.data->'{table}') expected
|
||||
WHERE NOT EXISTS(SELECT 1 FROM {table} actual WHERE to_jsonb(actual)=expected))
|
||||
THEN RAISE EXCEPTION 'Native edits or missing rows in {table}; rollback aborted'; END IF;
|
||||
END $unchanged$;""")
|
||||
parts.append("""DO $references$ BEGIN
|
||||
IF EXISTS(SELECT 1 FROM workbench_access_grants g JOIN workbench_access_publications p USING(tenant_id,kind,resource_id)
|
||||
WHERE to_jsonb(p) IN (SELECT value FROM _access_rollback_payload, jsonb_array_elements(data->'workbench_access_publications')))
|
||||
OR EXISTS(SELECT 1 FROM workbench_access_assignments a JOIN workbench_access_roles r ON r.id=a.role_id
|
||||
WHERE to_jsonb(r) IN (SELECT value FROM _access_rollback_payload, jsonb_array_elements(data->'workbench_access_roles'))
|
||||
AND to_jsonb(a) NOT IN (SELECT value FROM _access_rollback_payload, jsonb_array_elements(data->'workbench_access_assignments')))
|
||||
THEN RAISE EXCEPTION 'New references to migrated access rows; rollback aborted'; END IF;
|
||||
END $references$;""")
|
||||
for table in (ACCESS_TABLES[1], ACCESS_TABLES[2], ACCESS_TABLES[0], LEDGER):
|
||||
parts.append(f"DELETE FROM {table} actual WHERE to_jsonb(actual) IN (SELECT value FROM _access_rollback_payload,jsonb_array_elements(data->'{table}'));")
|
||||
parts.append("COMMIT;" if commit else "ROLLBACK;")
|
||||
return "\n".join(parts) + "\n"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("command", choices=("plan", "apply"))
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
suffixes = (".before.json", ".after.json", ".attempt.json", ".committed.json", ".rollback.sql")
|
||||
if args.out.exists() or any(args.out.with_suffix(s).exists() for s in suffixes):
|
||||
parser.error("Checkpoint exists; choose a new output name")
|
||||
plan = json.loads(args.manifest.read_text())
|
||||
sql = migration_sql(plan)
|
||||
before = snapshot()
|
||||
save_json(args.out.with_suffix(".before.json"), before)
|
||||
rehearsal = [json.loads(line) for line in sql_run(sql).splitlines() if line.startswith("{")]
|
||||
if snapshot() != before:
|
||||
raise RuntimeError("Rehearsal changed state or concurrent changes detected")
|
||||
result = {"checked_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "database": TARGET_DATABASE,
|
||||
"tenant_id": TARGET_TENANT, "manifest": str(args.manifest), "plan_sha256": plan["plan_sha256"],
|
||||
"rehearsal": rehearsal, "committed": False}
|
||||
if args.command == "apply":
|
||||
save_json(args.out.with_suffix(".attempt.json"), {**result, "commit_requested": True,
|
||||
"recovery": "Reconcile durable access ledger and checkpoints; never recreate records blindly"})
|
||||
changes = [json.loads(line) for line in sql_run(migration_sql(plan, True)).splitlines() if line.startswith("{")]
|
||||
result.update(committed=True, changes=changes)
|
||||
save_json(args.out.with_suffix(".committed.json"), result)
|
||||
after = snapshot()
|
||||
save_json(args.out.with_suffix(".after.json"), after)
|
||||
result["verification"] = verification(plan, before, after)
|
||||
args.out.with_suffix(".rollback.sql").write_text(rollback_sql(plan, before, after), encoding="utf-8")
|
||||
save_json(args.out, result)
|
||||
print(json.dumps(result))
|
||||
return 0 if result.get("verification", {}).get("passed", True) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Checkpoint Agent/skill migration without writing private bodies to reports."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from apply_weknora_identities import sql_run
|
||||
from weknora_asset_etl import LEDGER, BINDING_LEDGER, asset_plan, asset_sql, safe_asset_summary
|
||||
from weknora_category_etl import save_json
|
||||
from weknora_etl import consistent_snapshot
|
||||
from weknora_identity_etl import IDENTITY_LEDGER, TARGET_DATABASE, TARGET_TENANT
|
||||
from weknora_model_etl import model_plan
|
||||
|
||||
|
||||
def target_snapshot():
|
||||
return json.loads(sql_run("""SELECT json_build_object(
|
||||
'database',current_database(),
|
||||
'agents',(SELECT coalesce(json_agg(json_build_object('id',id,'tenant_id',tenant_id,
|
||||
'fingerprint',md5(to_jsonb(a)::text)) ORDER BY id),'[]') FROM custom_agents a),
|
||||
'skills',(SELECT coalesce(json_agg(json_build_object('id',id,'tenant_id',tenant_id,
|
||||
'fingerprint',md5(to_jsonb(s)::text)) ORDER BY id),'[]') FROM tenant_skill_catalog s),
|
||||
'models',(SELECT coalesce(json_agg(json_build_object('id',id,
|
||||
'fingerprint',md5(to_jsonb(m)::text)) ORDER BY id),'[]') FROM models m),
|
||||
'knowledge_count',(SELECT count(*) FROM knowledges),
|
||||
'chunk_count',(SELECT count(*) FROM chunks));"""))
|
||||
|
||||
|
||||
def build_plan(db, mapping_file):
|
||||
identities = json.loads(sql_run(f"SELECT json_object_agg(source_user_id,target_user_id) FROM {IDENTITY_LEDGER} "
|
||||
f"WHERE source_system='yawan-workbench' AND source_organization_id=1 AND target_tenant_id={TARGET_TENANT};"))
|
||||
state = json.loads(mapping_file.read_text())
|
||||
if state.get('tenant_id') != TARGET_TENANT:
|
||||
raise ValueError('Knowledge mapping tenant mismatch')
|
||||
kbs = {key: item['id'] for key, item in state['knowledge_bases'].items()}
|
||||
return asset_plan(db, identities, kbs, model_plan(db, TARGET_TENANT))
|
||||
|
||||
|
||||
def verification_sql():
|
||||
return f"""SELECT json_build_object(
|
||||
'agents',(SELECT count(*) FROM {LEDGER} WHERE target_tenant_id={TARGET_TENANT} AND source_kind='agent'),
|
||||
'skills',(SELECT count(*) FROM {LEDGER} WHERE target_tenant_id={TARGET_TENANT} AND source_kind='skill'),
|
||||
'bindings',(SELECT count(*) FROM {BINDING_LEDGER} WHERE target_tenant_id={TARGET_TENANT}),
|
||||
'effective_bindings',(SELECT coalesce(sum(jsonb_array_length(coalesce(a.config->'instruction_skill_ids','[]'))),0)
|
||||
FROM custom_agents a JOIN {LEDGER} l ON l.source_kind='agent' AND l.target_id=a.id WHERE l.target_tenant_id={TARGET_TENANT}),
|
||||
'disabled_agents',(SELECT count(*) FROM custom_agents a JOIN {LEDGER} l ON l.source_kind='agent' AND l.target_id=a.id
|
||||
WHERE l.target_tenant_id={TARGET_TENANT} AND NOT a.enabled),
|
||||
'private_agents',(SELECT count(*) FROM custom_agents a JOIN {LEDGER} l ON l.source_kind='agent' AND l.target_id=a.id
|
||||
WHERE l.target_tenant_id={TARGET_TENANT} AND a.visibility='private'),
|
||||
'private_skills',(SELECT count(*) FROM tenant_skill_catalog s JOIN {LEDGER} l ON l.source_kind='skill' AND l.target_id=s.id
|
||||
WHERE l.target_tenant_id={TARGET_TENANT} AND s.visibility='private'),
|
||||
'rules_match',(SELECT bool_and(a.config->>'system_prompt'=coalesce(l.source_snapshot->>'rules',''))
|
||||
FROM custom_agents a JOIN {LEDGER} l ON l.source_kind='agent' AND l.target_id=a.id WHERE l.target_tenant_id={TARGET_TENANT}),
|
||||
'skill_bodies_match',(SELECT bool_and(s.instructions=l.source_snapshot->>'instructions')
|
||||
FROM tenant_skill_catalog s JOIN {LEDGER} l ON l.source_kind='skill' AND l.target_id=s.id WHERE l.target_tenant_id={TARGET_TENANT}));"""
|
||||
|
||||
|
||||
def transaction(query, commit=False):
|
||||
return f"""BEGIN ISOLATION LEVEL SERIALIZABLE;
|
||||
SET LOCAL statement_timeout='30s';
|
||||
SET LOCAL lock_timeout='5s';
|
||||
CREATE TEMP TABLE _agents_before ON COMMIT DROP AS SELECT * FROM custom_agents;
|
||||
CREATE TEMP TABLE _skills_before ON COMMIT DROP AS SELECT * FROM tenant_skill_catalog;
|
||||
{query}
|
||||
DO $preservation$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM _agents_before b LEFT JOIN custom_agents a ON a.id=b.id
|
||||
WHERE a.id IS NULL OR to_jsonb(a) IS DISTINCT FROM to_jsonb(b))
|
||||
OR EXISTS (SELECT 1 FROM _skills_before b LEFT JOIN tenant_skill_catalog s ON s.id=b.id
|
||||
WHERE s.id IS NULL OR to_jsonb(s) IS DISTINCT FROM to_jsonb(b))
|
||||
THEN RAISE EXCEPTION 'Asset import changed an existing row'; END IF;
|
||||
END $preservation$;
|
||||
SELECT json_build_object(
|
||||
'inserted_agents',(SELECT count(*) FROM custom_agents a WHERE NOT EXISTS (SELECT 1 FROM _agents_before b WHERE b.id=a.id)),
|
||||
'inserted_skills',(SELECT count(*) FROM tenant_skill_catalog s WHERE NOT EXISTS (SELECT 1 FROM _skills_before b WHERE b.id=s.id)),
|
||||
'existing_assets_unchanged',true);
|
||||
{verification_sql()}
|
||||
{'COMMIT' if commit else 'ROLLBACK'};
|
||||
"""
|
||||
|
||||
|
||||
def verify(plan, before, after, results):
|
||||
checks = {'acceptance_scope': before['database'] == after['database'] == TARGET_DATABASE}
|
||||
for kind in ('agents', 'skills'):
|
||||
old, new = ({v['id']: v for v in state[kind]} for state in (before, after))
|
||||
expected = {item['target']['id'] for item in plan[kind]}
|
||||
checks[kind + '_existing_unchanged'] = all(new.get(k) == v for k, v in old.items())
|
||||
checks[kind + '_planned_only'] = set(new) - set(old) == expected - set(old)
|
||||
checks[kind + '_all_present'] = expected <= set(new)
|
||||
checks['knowledge_and_models_unchanged'] = all(before[k] == after[k] for k in ('models', 'knowledge_count', 'chunk_count'))
|
||||
summary = next(item for item in results if 'effective_bindings' in item)
|
||||
checks['durable_mapping_counts'] = all(summary[k] == len(plan[k]) for k in ('agents', 'skills', 'bindings'))
|
||||
checks['effective_binding_count'] = summary['effective_bindings'] == sum(len(a['target']['config']['instruction_skill_ids']) for a in plan['agents'])
|
||||
checks['disabled_and_private_preserved'] = summary['disabled_agents'] == 1 and summary['private_agents'] == 2 and summary['private_skills'] == 2
|
||||
# Native edits may differ on future repeats; first import must preserve all bodies.
|
||||
if not any(x['target']['id'] in {a['id'] for a in before['agents']} for x in plan['agents']):
|
||||
checks['source_bodies_preserved'] = summary['rules_match'] and summary['skill_bodies_match']
|
||||
return {'passed': all(checks.values()), 'checks': checks}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('command', choices=('plan', 'apply'))
|
||||
parser.add_argument('--sqlite', type=Path, required=True)
|
||||
parser.add_argument('--kb-mapping', type=Path, required=True)
|
||||
parser.add_argument('--out', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
suffixes = ('.before.json', '.manifest.json', '.attempt.json', '.committed.json', '.after.json')
|
||||
if any(p.exists() for p in (args.out, *(args.out.with_suffix(s) for s in suffixes))):
|
||||
parser.error('Checkpoint exists; use a new output name')
|
||||
with consistent_snapshot(args.sqlite) as (db, info):
|
||||
plan = build_plan(db, args.kb_mapping)
|
||||
if info['source_mutated']:
|
||||
raise RuntimeError('Source changed while taking the snapshot')
|
||||
plan['source_snapshot_sha256'] = info['sha256']
|
||||
query = asset_sql(plan)
|
||||
before = target_snapshot()
|
||||
save_json(args.out.with_suffix('.before.json'), before)
|
||||
save_json(args.out.with_suffix('.manifest.json'), {**safe_asset_summary(plan), 'source_snapshot_sha256': info['sha256']})
|
||||
rehearsal = [json.loads(line) for line in sql_run(transaction(query)).splitlines() if line.startswith('{')]
|
||||
result = {'checked_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'database': TARGET_DATABASE, 'tenant_id': TARGET_TENANT,
|
||||
'source_snapshot_sha256': info['sha256'], 'source_mutated': info['source_mutated'],
|
||||
'rehearsal': rehearsal, 'committed': False}
|
||||
if args.command == 'apply':
|
||||
save_json(args.out.with_suffix('.attempt.json'), {**result, 'commit_requested': True,
|
||||
'recovery': 'Reconcile asset ledger and committed receipt; do not delete or recreate assets automatically'})
|
||||
results = [json.loads(line) for line in sql_run(transaction(query, True)).splitlines() if line.startswith('{')]
|
||||
result.update(committed=True, changes=results)
|
||||
save_json(args.out.with_suffix('.committed.json'), result)
|
||||
after = target_snapshot()
|
||||
save_json(args.out.with_suffix('.after.json'), after)
|
||||
result['verification'] = verify(plan, before, after, results)
|
||||
save_json(args.out, result)
|
||||
print(json.dumps(result))
|
||||
return 0 if result.get('verification', {}).get('passed', True) else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Checkpoint historical workbench imports into the pinned acceptance database."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from apply_weknora_identities import sql_run
|
||||
from weknora_asset_etl import LEDGER as ASSET_LEDGER
|
||||
from weknora_business_etl import LEDGER, business_plan, business_sql, safe_business_summary
|
||||
from weknora_category_etl import save_json
|
||||
from weknora_etl import consistent_snapshot
|
||||
from weknora_identity_etl import IDENTITY_LEDGER, TARGET_DATABASE, TARGET_TENANT
|
||||
|
||||
PRESERVED_TABLES = ('workbench_jobs', 'workbench_events', 'workbench_revisions',
|
||||
'custom_agents', 'tenant_skill_catalog', 'models')
|
||||
|
||||
|
||||
def build_plan(db, mapping_file, artifact_root):
|
||||
identities = json.loads(sql_run(f"SELECT json_object_agg(source_user_id,target_user_id) FROM {IDENTITY_LEDGER} "
|
||||
f"WHERE source_system='yawan-workbench' AND source_organization_id=1 AND target_tenant_id={TARGET_TENANT};"))
|
||||
agents = json.loads(sql_run(f"SELECT json_object_agg(source_id,target_id) FROM {ASSET_LEDGER} "
|
||||
f"WHERE source_system='yawan-workbench' AND source_organization_id=1 AND source_kind='agent' AND target_tenant_id={TARGET_TENANT};"))
|
||||
state = json.loads(mapping_file.read_text())
|
||||
if state.get('tenant_id') != TARGET_TENANT:
|
||||
raise ValueError('Knowledge mapping tenant mismatch')
|
||||
kbs = {key: item['id'] for key, item in state['knowledge_bases'].items()}
|
||||
return business_plan(db, identities, agents, kbs, artifact_root)
|
||||
|
||||
|
||||
def target_snapshot():
|
||||
parts = ["'database',current_database()", "'knowledge_count',(SELECT count(*) FROM knowledges)",
|
||||
"'chunk_count',(SELECT count(*) FROM chunks)"]
|
||||
for table in ('workbench_resources', *PRESERVED_TABLES):
|
||||
parts.append(f"'{table}',(SELECT coalesce(json_agg(json_build_object('id',id,'hash',md5(to_jsonb(r)::text)) ORDER BY id),'[]') FROM {table} r)")
|
||||
return json.loads(sql_run('SELECT json_build_object(' + ','.join(parts) + ');'))
|
||||
|
||||
|
||||
def verification_sql():
|
||||
return f"""SELECT json_build_object(
|
||||
'ledger_counts',(SELECT json_object_agg(source_kind,n) FROM (SELECT source_kind,count(*) n
|
||||
FROM {LEDGER} WHERE target_tenant_id={TARGET_TENANT} GROUP BY source_kind) q),
|
||||
'resources',(SELECT count(*) FROM {LEDGER} l JOIN workbench_resources r ON r.id=l.target_id
|
||||
WHERE l.target_tenant_id={TARGET_TENANT} AND l.source_kind IN ('canvas','trainingmap','campaign')),
|
||||
'protected',(SELECT count(*) FROM {LEDGER} l JOIN workbench_resources r ON r.id=l.target_id
|
||||
WHERE l.target_tenant_id={TARGET_TENANT} AND r.data->'legacy_import'->>'read_only'='true'),
|
||||
'segments',(SELECT coalesce(sum(jsonb_array_length(r.data->'segments')),0)
|
||||
FROM {LEDGER} l JOIN workbench_resources r ON r.id=l.target_id
|
||||
WHERE l.target_tenant_id={TARGET_TENANT} AND l.source_kind='campaign'),
|
||||
'initial_data_match',(SELECT bool_and(r.data=l.native_snapshot->'data' AND r.owner_id=l.native_snapshot->>'owner_id'
|
||||
AND r.status=l.native_snapshot->>'status' AND r.version=(l.native_snapshot->>'version')::integer)
|
||||
FROM {LEDGER} l JOIN workbench_resources r ON r.id=l.target_id WHERE l.target_tenant_id={TARGET_TENANT}));"""
|
||||
|
||||
|
||||
def transaction(query, commit=False):
|
||||
sql = "BEGIN ISOLATION LEVEL SERIALIZABLE; SET LOCAL statement_timeout='45s'; SET LOCAL lock_timeout='5s';\n"
|
||||
tables = ('workbench_resources', *PRESERVED_TABLES)
|
||||
for i, table in enumerate(tables):
|
||||
sql += f'CREATE TEMP TABLE _business_before_{i} ON COMMIT DROP AS SELECT * FROM {table};\n'
|
||||
sql += query
|
||||
assertions = ["EXISTS (SELECT 1 FROM _business_before_0 b LEFT JOIN workbench_resources r ON r.id=b.id WHERE r.id IS NULL OR to_jsonb(r) IS DISTINCT FROM to_jsonb(b))"]
|
||||
for i, table in enumerate(tables[1:], 1):
|
||||
assertions.extend((f'EXISTS (SELECT * FROM {table} EXCEPT SELECT * FROM _business_before_{i})',
|
||||
f'EXISTS (SELECT * FROM _business_before_{i} EXCEPT SELECT * FROM {table})'))
|
||||
sql += "DO $preserve$ BEGIN IF " + ' OR '.join(assertions) + " THEN RAISE EXCEPTION 'Business import changed existing data or execution state'; END IF; END $preserve$;\n"
|
||||
sql += "SELECT json_build_object('inserted_resources',(SELECT count(*) FROM workbench_resources r WHERE NOT EXISTS (SELECT 1 FROM _business_before_0 b WHERE b.id=r.id)), 'existing_data_unchanged',true);\n"
|
||||
return sql + verification_sql() + ('COMMIT;' if commit else 'ROLLBACK;')
|
||||
|
||||
|
||||
def verify(plan, before, after, results):
|
||||
old = {r['id']: r for r in before['workbench_resources']}
|
||||
new = {r['id']: r for r in after['workbench_resources']}
|
||||
planned = {r['target']['id'] for r in plan['resources']}
|
||||
summary = next(r for r in results if 'ledger_counts' in r)
|
||||
counts = safe_business_summary(plan)['counts']
|
||||
checks = {
|
||||
'acceptance_scope': before['database'] == after['database'] == TARGET_DATABASE,
|
||||
'existing_resources_unchanged': all(new.get(k) == v for k, v in old.items()),
|
||||
'only_planned_resources_added': set(new) - set(old) == planned - set(old),
|
||||
'all_resources_present': planned <= set(new),
|
||||
'no_jobs_events_revisions_or_other_assets_changed': all(before[k] == after[k] for k in PRESERVED_TABLES),
|
||||
'knowledge_unchanged': all(before[k] == after[k] for k in ('knowledge_count', 'chunk_count')),
|
||||
'all_history_counts_match': all(summary['ledger_counts'].get(k, 0) == v for k, v in counts.items()),
|
||||
'all_segments_present': summary['segments'] == counts['segment'],
|
||||
'historical_execution_protected': summary['protected'] == len(plan['resources']),
|
||||
}
|
||||
if not planned.intersection(old):
|
||||
checks['initial_content_status_and_owner_match'] = summary['initial_data_match']
|
||||
return {'passed': all(checks.values()), 'checks': checks}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('command', choices=('plan', 'apply'))
|
||||
parser.add_argument('--sqlite', type=Path, required=True)
|
||||
parser.add_argument('--kb-mapping', type=Path, required=True)
|
||||
parser.add_argument('--artifact-root', type=Path, required=True)
|
||||
parser.add_argument('--out', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if any(p.exists() for p in [args.out, *(args.out.with_suffix(s) for s in ('.before.json', '.manifest.json', '.attempt.json', '.committed.json', '.after.json'))]):
|
||||
parser.error('Checkpoint exists; choose a fresh output name')
|
||||
with consistent_snapshot(args.sqlite) as (db, info):
|
||||
plan = build_plan(db, args.kb_mapping, args.artifact_root)
|
||||
if info['source_mutated']:
|
||||
raise RuntimeError('Source changed during snapshot')
|
||||
plan['source_snapshot_sha256'] = info['sha256']
|
||||
query = business_sql(plan)
|
||||
before = target_snapshot()
|
||||
save_json(args.out.with_suffix('.before.json'), before)
|
||||
save_json(args.out.with_suffix('.manifest.json'), {**safe_business_summary(plan), 'source_snapshot_sha256': info['sha256']})
|
||||
rehearsal = [json.loads(line) for line in sql_run(transaction(query)).splitlines() if line.startswith('{')]
|
||||
if before != target_snapshot():
|
||||
raise RuntimeError('Target changed during rollback rehearsal')
|
||||
result = {'checked_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'database': TARGET_DATABASE,
|
||||
'tenant_id': TARGET_TENANT, 'source_mutated': info['source_mutated'],
|
||||
'source_snapshot_sha256': info['sha256'], 'rehearsal': rehearsal, 'committed': False}
|
||||
if args.command == 'apply':
|
||||
save_json(args.out.with_suffix('.attempt.json'), {**result, 'commit_requested': True})
|
||||
results = [json.loads(line) for line in sql_run(transaction(query, True)).splitlines() if line.startswith('{')]
|
||||
result.update(committed=True, changes=results)
|
||||
save_json(args.out.with_suffix('.committed.json'), result)
|
||||
after = target_snapshot()
|
||||
save_json(args.out.with_suffix('.after.json'), after)
|
||||
result['verification'] = verify(plan, before, after, results)
|
||||
save_json(args.out, result)
|
||||
print(json.dumps(result))
|
||||
return 0 if result.get('verification', {}).get('passed', True) else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Add the deleted historical owner's disabled identity with rollback rehearsal."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from apply_weknora_identities import sql_run, target_snapshot
|
||||
from weknora_category_etl import save_json
|
||||
from weknora_etl import consistent_snapshot
|
||||
from weknora_historical_identity_etl import historical_identity_plan, historical_identity_sql
|
||||
from weknora_identity_etl import TARGET_TENANT
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--sqlite', type=Path, required=True)
|
||||
parser.add_argument('--out', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if any(p.exists() for p in [args.out, args.out.with_suffix('.before.json'), args.out.with_suffix('.attempt.json')]):
|
||||
parser.error('Checkpoint exists')
|
||||
with consistent_snapshot(args.sqlite) as (db, info):
|
||||
plan = historical_identity_plan(db, TARGET_TENANT)
|
||||
if info['source_mutated']:
|
||||
raise RuntimeError('Source changed during snapshot')
|
||||
plan['source_snapshot_sha256'] = info['sha256']
|
||||
query = historical_identity_sql(plan)
|
||||
before = target_snapshot()
|
||||
save_json(args.out.with_suffix('.before.json'), before)
|
||||
sql = """BEGIN ISOLATION LEVEL SERIALIZABLE;
|
||||
SET LOCAL statement_timeout='30s'; SET LOCAL lock_timeout='5s';
|
||||
SET LOCAL log_statement='none'; SET LOCAL log_min_error_statement='panic';
|
||||
CREATE TEMP TABLE _old_users ON COMMIT DROP AS SELECT * FROM users;
|
||||
CREATE TEMP TABLE _old_members ON COMMIT DROP AS SELECT * FROM tenant_members;
|
||||
""" + query + """
|
||||
DO $verify$ BEGIN
|
||||
IF EXISTS(SELECT 1 FROM _old_users b LEFT JOIN users a ON a.id=b.id WHERE a.id IS NULL OR to_jsonb(a) IS DISTINCT FROM to_jsonb(b))
|
||||
OR EXISTS(SELECT 1 FROM _old_members b LEFT JOIN tenant_members a ON a.id=b.id WHERE a.id IS NULL OR to_jsonb(a) IS DISTINCT FROM to_jsonb(b))
|
||||
THEN RAISE EXCEPTION 'Historical identity changed existing users'; END IF;
|
||||
END $verify$;
|
||||
"""
|
||||
sql_run(sql + 'ROLLBACK;')
|
||||
if before != target_snapshot():
|
||||
raise RuntimeError('Rollback rehearsal changed identities')
|
||||
save_json(args.out.with_suffix('.attempt.json'), {'source_snapshot_sha256': info['sha256'], 'commit_requested': True})
|
||||
sql_run(sql + 'COMMIT;')
|
||||
after = target_snapshot()
|
||||
uid = plan['users'][0]['target_id']
|
||||
user = next(u for u in after['users'] if u['id'] == uid)
|
||||
members = [m for m in after['memberships'] if m['user_id'] == uid]
|
||||
tokens = [t for t in after['token_counts'] if t['user_id'] == uid]
|
||||
checks = {'source_not_mutated': not info['source_mutated'], 'login_disabled': not user['is_active'],
|
||||
'not_admin': not user['is_system_admin'] and not user['can_access_all_tenants'],
|
||||
'membership_suspended': len(members) == 1 and members[0]['tenant_id'] == TARGET_TENANT and members[0]['status'] == 'suspended',
|
||||
'no_tokens': not tokens}
|
||||
sql_run(sql + 'COMMIT;')
|
||||
checks['repeat_no_change'] = after == target_snapshot()
|
||||
result = {'committed': True, 'source_user_id': 2, 'target_id': uid,
|
||||
'passed': all(checks.values()), 'checks': checks}
|
||||
save_json(args.out, result)
|
||||
print(json.dumps(result))
|
||||
return 0 if result['passed'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Checkpointed identity import restricted to the local acceptance database."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from weknora_category_etl import save_json
|
||||
from weknora_etl import consistent_snapshot
|
||||
from weknora_identity_etl import (ADMIN_TARGET_ID, IDENTITY_LEDGER, ORGANIZATION_LEDGER,
|
||||
TARGET_DATABASE, TARGET_TENANT, identity_plan, identity_sql)
|
||||
|
||||
|
||||
def sql_run(sql):
|
||||
result = subprocess.run([
|
||||
"docker", "--context", "colima-yawan", "exec", "-i", "WeKnora-postgres",
|
||||
"psql", "-X", "-qAt", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", TARGET_DATABASE,
|
||||
], input=sql, text=True, capture_output=True, timeout=60)
|
||||
if result.returncode:
|
||||
# PostgreSQL can include SQL payloads in errors. Do not print them.
|
||||
raise RuntimeError("Identity transaction not confirmed; reconcile ledger and checkpoints before retrying")
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def target_snapshot():
|
||||
snapshot = json.loads(sql_run(f"""
|
||||
BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY;
|
||||
SELECT json_build_object(
|
||||
'database',current_database(),
|
||||
'tenant',(SELECT json_build_object('id',id,'status',status) FROM tenants WHERE id={TARGET_TENANT}),
|
||||
'users',(SELECT coalesce(json_agg(row_to_json(x) ORDER BY id),'[]') FROM
|
||||
(SELECT id,username,email,avatar,tenant_id,is_active,is_system_admin,can_access_all_tenants,
|
||||
created_at,updated_at,deleted_at FROM users) x),
|
||||
'memberships',(SELECT coalesce(json_agg(row_to_json(x) ORDER BY id),'[]') FROM tenant_members x),
|
||||
'token_counts',(SELECT coalesce(json_agg(row_to_json(x) ORDER BY user_id),'[]') FROM
|
||||
(SELECT user_id,count(*) AS count FROM auth_tokens GROUP BY user_id) x)
|
||||
);
|
||||
COMMIT;
|
||||
"""))
|
||||
snapshot["ledgers"] = {}
|
||||
for table in (ORGANIZATION_LEDGER, IDENTITY_LEDGER):
|
||||
exists = sql_run(f"SELECT to_regclass('public.{table}') IS NOT NULL;") == "t"
|
||||
order = "source_system,source_organization_id" + (",source_user_id" if table == IDENTITY_LEDGER else "")
|
||||
snapshot["ledgers"][table] = json.loads(sql_run(f"SELECT coalesce(json_agg(x ORDER BY {order}),'[]') FROM {table} x;")) if exists else []
|
||||
return snapshot
|
||||
|
||||
|
||||
def transaction(manifest, commit=False):
|
||||
# Save complete rows only inside PostgreSQL; credentials never enter reports.
|
||||
return f"""
|
||||
BEGIN ISOLATION LEVEL SERIALIZABLE;
|
||||
SET LOCAL statement_timeout = '30s';
|
||||
SET LOCAL lock_timeout = '5s';
|
||||
CREATE TEMP TABLE _identity_before_users ON COMMIT DROP AS SELECT * FROM users;
|
||||
CREATE TEMP TABLE _identity_before_members ON COMMIT DROP AS SELECT * FROM tenant_members;
|
||||
{identity_sql(manifest)}
|
||||
DO $identity_preservation$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM _identity_before_users b LEFT JOIN users u ON u.id=b.id
|
||||
WHERE u.id IS NULL OR to_jsonb(u) IS DISTINCT FROM to_jsonb(b))
|
||||
THEN RAISE EXCEPTION 'Identity import changed an existing user'; END IF;
|
||||
IF EXISTS (SELECT 1 FROM _identity_before_members b LEFT JOIN tenant_members m ON m.id=b.id
|
||||
WHERE m.id IS NULL OR to_jsonb(m) IS DISTINCT FROM to_jsonb(b))
|
||||
THEN RAISE EXCEPTION 'Identity import changed an existing membership'; END IF;
|
||||
END $identity_preservation$;
|
||||
SELECT json_build_object(
|
||||
'inserted_users',(SELECT count(*) FROM users u WHERE NOT EXISTS (SELECT 1 FROM _identity_before_users b WHERE b.id=u.id)),
|
||||
'inserted_memberships',(SELECT count(*) FROM tenant_members m WHERE NOT EXISTS (SELECT 1 FROM _identity_before_members b WHERE b.id=m.id)),
|
||||
'identity_mappings',(SELECT count(*) FROM {IDENTITY_LEDGER} WHERE target_tenant_id={TARGET_TENANT}),
|
||||
'organization_mappings',(SELECT count(*) FROM {ORGANIZATION_LEDGER} WHERE target_tenant_id={TARGET_TENANT}),
|
||||
'existing_users_unchanged',true,'existing_memberships_unchanged',true
|
||||
);
|
||||
{'COMMIT' if commit else 'ROLLBACK'};
|
||||
"""
|
||||
|
||||
|
||||
def verify_states(manifest, before, after):
|
||||
checks = []
|
||||
|
||||
def check(name, passed):
|
||||
checks.append({"check": name, "passed": bool(passed)})
|
||||
|
||||
check("acceptance_scope", before["database"] == after["database"] == TARGET_DATABASE
|
||||
and before["tenant"] == after["tenant"] == {"id": TARGET_TENANT, "status": "active"})
|
||||
old_users = {u["id"]: u for u in before["users"]}
|
||||
users = {u["id"]: u for u in after["users"]}
|
||||
old_members = {m["id"]: m for m in before["memberships"]}
|
||||
members = {m["id"]: m for m in after["memberships"]}
|
||||
check("existing_users_unchanged", all(users.get(key) == value for key, value in old_users.items()))
|
||||
check("existing_memberships_unchanged", all(members.get(key) == value for key, value in old_members.items()))
|
||||
check("existing_admin_unchanged", ADMIN_TARGET_ID in old_users and users.get(ADMIN_TARGET_ID) == old_users[ADMIN_TARGET_ID])
|
||||
expected_ids = {u["target_id"] for u in manifest["users"] if u["action"] == "create_suspended"}
|
||||
check("only_planned_users_added", set(users) - set(old_users) == expected_ids - set(old_users))
|
||||
added_members = [m for key, m in members.items() if key not in old_members]
|
||||
check("only_planned_memberships_added", len(added_members) == len(expected_ids - set(old_users))
|
||||
and {m["user_id"] for m in added_members} == expected_ids - set(old_users))
|
||||
token_counts = {t["user_id"]: t["count"] for t in after["token_counts"]}
|
||||
for item in manifest["users"]:
|
||||
if item["action"] != "create_suspended":
|
||||
continue
|
||||
uid = item["target_id"]
|
||||
user = users.get(uid, {})
|
||||
scoped = [m for m in members.values() if m["user_id"] == uid and m["deleted_at"] is None]
|
||||
expected_contact = old_users.get(uid, {"username": item["target_username"], "email": item["target_email"]})
|
||||
check(f"user_{item['source_user']['id']}_identity", user.get("username") == expected_contact["username"]
|
||||
and user.get("email") == expected_contact["email"] and user.get("tenant_id") == TARGET_TENANT
|
||||
and user.get("deleted_at") is None and user.get("is_system_admin") is False
|
||||
and user.get("can_access_all_tenants") is False)
|
||||
check(f"user_{item['source_user']['id']}_isolated", len(scoped) == 1
|
||||
and scoped[0]["tenant_id"] == TARGET_TENANT and scoped[0]["role"] == "contributor")
|
||||
# Exact reruns preserve later activation and password-reset decisions.
|
||||
if uid not in old_users:
|
||||
check(f"user_{item['source_user']['id']}_login_disabled", user.get("is_active") is False
|
||||
and len(scoped) == 1 and scoped[0]["status"] == "suspended" and token_counts.get(uid, 0) == 0)
|
||||
skipped_names = {u["source_user"]["username"].casefold() for u in manifest["skipped"]}
|
||||
check("disabled_source_accounts_not_added", all(u["username"].casefold() not in skipped_names
|
||||
for uid, u in users.items() if uid not in old_users))
|
||||
identities = {r["source_user_id"]: r for r in after["ledgers"][IDENTITY_LEDGER] if r["target_tenant_id"] == TARGET_TENANT}
|
||||
organizations = [r for r in after["ledgers"][ORGANIZATION_LEDGER] if r["target_tenant_id"] == TARGET_TENANT]
|
||||
check("durable_identity_mapping", set(identities) == {u["source_user"]["id"] for u in manifest["users"]}
|
||||
and all(identities[u["source_user"]["id"]]["target_user_id"] == u["target_id"]
|
||||
and identities[u["source_user"]["id"]]["source_snapshot"] == u["source_user"] for u in manifest["users"]))
|
||||
check("durable_organization_and_roles", len(organizations) == 1
|
||||
and organizations[0]["source_snapshot"] == manifest["source_organization"]
|
||||
and organizations[0]["source_role_definitions"] == manifest["source_role_definitions"])
|
||||
return {"passed": all(c["passed"] for c in checks), "checks": checks}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("command", choices=("plan", "apply"))
|
||||
parser.add_argument("--sqlite", type=Path, required=True)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
suffixes = (".before.json", ".manifest.json", ".attempt.json", ".committed.json", ".after.json")
|
||||
if any(p.exists() for p in (args.out, *(args.out.with_suffix(s) for s in suffixes))):
|
||||
parser.error("Output checkpoint exists; use a new output name")
|
||||
with consistent_snapshot(args.sqlite) as (snapshot, info):
|
||||
manifest = identity_plan(snapshot, TARGET_TENANT)
|
||||
if info["source_mutated"]:
|
||||
raise RuntimeError("Source changed while taking snapshot; retry during a stable migration window")
|
||||
manifest["source_snapshot"] = info
|
||||
manifest["source_snapshot_sha256"] = info["sha256"]
|
||||
before = target_snapshot()
|
||||
save_json(args.out.with_suffix(".before.json"), before)
|
||||
save_json(args.out.with_suffix(".manifest.json"), manifest)
|
||||
rehearsal = [json.loads(line) for line in sql_run(transaction(manifest)).splitlines() if line.startswith("{")]
|
||||
result = {"checked_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"tenant_id": TARGET_TENANT, "database": TARGET_DATABASE,
|
||||
"source_snapshot_sha256": info["sha256"], "source_mutated": info["source_mutated"],
|
||||
"mapped_admins": 1, "planned_suspended_users": 2, "skipped_disabled_users": len(manifest["skipped"]),
|
||||
"rehearsal": rehearsal, "committed": False}
|
||||
if args.command == "apply":
|
||||
save_json(args.out.with_suffix(".attempt.json"), {**result, "commit_requested": True,
|
||||
"recovery": "Inspect identity ledger and use a new output name to reconcile; do not delete users automatically"})
|
||||
result["changes"] = [json.loads(line) for line in sql_run(transaction(manifest, True)).splitlines() if line.startswith("{")]
|
||||
result["committed"] = True
|
||||
save_json(args.out.with_suffix(".committed.json"), result)
|
||||
after = target_snapshot()
|
||||
save_json(args.out.with_suffix(".after.json"), after)
|
||||
result["verification"] = verify_states(manifest, before, after)
|
||||
save_json(args.out, result)
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
return 0 if result.get("verification", {}).get("passed", True) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Checkpointed model import into the isolated acceptance database."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from apply_weknora_identities import sql_run
|
||||
from weknora_category_etl import save_json
|
||||
from weknora_etl import consistent_snapshot
|
||||
from weknora_identity_etl import TARGET_DATABASE, TARGET_TENANT
|
||||
from weknora_model_etl import LEDGER, model_plan, model_sql, model_verification_sql, read_credentials
|
||||
|
||||
|
||||
def resolve_source_credential(value, provider_id):
|
||||
if not value.startswith('keychain:'):
|
||||
return value
|
||||
expected = f'keychain:yawen-workbench.providers:provider-{provider_id}'
|
||||
if value != expected:
|
||||
raise ValueError('Source credential reference does not match the reviewed provider')
|
||||
result = subprocess.run(['/usr/bin/security', 'find-generic-password', '-s', value[9:], '-w'],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
if result.returncode or not result.stdout.strip():
|
||||
raise RuntimeError('Source provider credential could not be resolved')
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def native_encryptor(environment: Path):
|
||||
env = dict(line.split("=", 1) for line in environment.read_text().splitlines()
|
||||
if line and not line.startswith("#") and "=" in line)
|
||||
key = env.get("SYSTEM_AES_KEY", "")
|
||||
if env.get("DB_NAME") != TARGET_DATABASE or len(key.encode()) != 32:
|
||||
raise ValueError("Acceptance encryption configuration is invalid")
|
||||
|
||||
def encrypt(value):
|
||||
# Node's standard crypto API matches internal/utils/crypto.go. Secrets
|
||||
# enter through stdin, never argv, an environment override, or a file.
|
||||
script = """const fs = require('node:fs'), crypto = require('node:crypto');
|
||||
const input = JSON.parse(fs.readFileSync(0, 'utf8'));
|
||||
const nonce = crypto.randomBytes(12);
|
||||
const cipher = crypto.createCipheriv('aes-256-gcm', Buffer.from(input.key), nonce);
|
||||
const data = Buffer.concat([nonce, cipher.update(input.value, 'utf8'), cipher.final(), cipher.getAuthTag()]);
|
||||
process.stdout.write('enc:v1:' + data.toString('base64url'));
|
||||
"""
|
||||
result = subprocess.run(["node", "-e", script], input=json.dumps({"key": key, "value": value}),
|
||||
capture_output=True, text=True, timeout=10)
|
||||
if result.returncode:
|
||||
raise RuntimeError("Native credential encryption failed")
|
||||
return result.stdout
|
||||
return encrypt
|
||||
|
||||
|
||||
def target_snapshot():
|
||||
return json.loads(sql_run("""SELECT json_build_object(
|
||||
'database',current_database(),
|
||||
'models',(SELECT coalesce(json_agg(json_build_object('id',id,'tenant_id',tenant_id,
|
||||
'name',name,'type',type,'fingerprint',md5(to_jsonb(m)::text)) ORDER BY id),'[]') FROM models m),
|
||||
'knowledge_count',(SELECT count(*) FROM knowledges),
|
||||
'chunk_count',(SELECT count(*) FROM chunks));"""))
|
||||
|
||||
|
||||
def transaction(query, commit=False):
|
||||
return f"""BEGIN ISOLATION LEVEL SERIALIZABLE;
|
||||
SET LOCAL statement_timeout='30s';
|
||||
SET LOCAL lock_timeout='5s';
|
||||
CREATE TEMP TABLE _model_before ON COMMIT DROP AS SELECT * FROM models;
|
||||
{query}
|
||||
DO $preservation$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM _model_before b LEFT JOIN models m ON m.id=b.id
|
||||
WHERE m.id IS NULL OR to_jsonb(m) IS DISTINCT FROM to_jsonb(b))
|
||||
THEN RAISE EXCEPTION 'Model import changed an existing row'; END IF;
|
||||
END $preservation$;
|
||||
SELECT json_build_object('inserted_models',
|
||||
(SELECT count(*) FROM models m WHERE NOT EXISTS (SELECT 1 FROM _model_before b WHERE b.id=m.id)),
|
||||
'existing_models_unchanged',true);
|
||||
{model_verification_sql()}
|
||||
{'COMMIT' if commit else 'ROLLBACK'};
|
||||
"""
|
||||
|
||||
|
||||
def verify(plan, before, after, summaries):
|
||||
old = {m['id']: m for m in before['models']}
|
||||
new = {m['id']: m for m in after['models']}
|
||||
expected = {m['target_id'] for m in plan['models']}
|
||||
checks = {
|
||||
'acceptance_scope': before['database'] == after['database'] == TARGET_DATABASE,
|
||||
'existing_models_unchanged': all(new.get(k) == v for k, v in old.items()),
|
||||
'only_planned_models_added': set(new) - set(old) == expected - set(old),
|
||||
'all_models_present': expected <= set(new),
|
||||
'knowledge_and_chunks_unchanged': all(before[k] == after[k] for k in ('knowledge_count', 'chunk_count')),
|
||||
'ledger_and_encrypted_credentials': any(s.get('mapped') == len(expected)
|
||||
and len(s.get('models', [])) == len(expected)
|
||||
and all(m['encrypted_credential'] for m in s['models']) for s in summaries),
|
||||
}
|
||||
return {'passed': all(checks.values()), 'checks': checks}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('command', choices=('plan', 'apply'))
|
||||
parser.add_argument('--sqlite', type=Path, required=True)
|
||||
parser.add_argument('--environment', type=Path, required=True)
|
||||
parser.add_argument('--out', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
suffixes = ('.before.json', '.manifest.json', '.attempt.json', '.committed.json', '.after.json')
|
||||
if any(p.exists() for p in (args.out, *(args.out.with_suffix(s) for s in suffixes))):
|
||||
parser.error('Checkpoint exists; use a new output name')
|
||||
with consistent_snapshot(args.sqlite) as (db, info):
|
||||
plan = model_plan(db, TARGET_TENANT)
|
||||
if info['source_mutated']:
|
||||
raise RuntimeError('Source changed while taking the snapshot')
|
||||
plan['source_snapshot_sha256'] = info['sha256']
|
||||
query = model_sql(plan, read_credentials(db, plan, resolve_source_credential), native_encryptor(args.environment))
|
||||
before = target_snapshot()
|
||||
save_json(args.out.with_suffix('.before.json'), before)
|
||||
save_json(args.out.with_suffix('.manifest.json'), plan)
|
||||
rehearsal = [json.loads(line) for line in sql_run(transaction(query)).splitlines() if line.startswith('{')]
|
||||
result = {'checked_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'database': TARGET_DATABASE, 'tenant_id': TARGET_TENANT,
|
||||
'source_snapshot_sha256': info['sha256'], 'source_mutated': info['source_mutated'],
|
||||
'planned_models': len(plan['models']), 'skipped_models': len(plan['skipped']),
|
||||
'rehearsal': rehearsal, 'committed': False}
|
||||
if args.command == 'apply':
|
||||
save_json(args.out.with_suffix('.attempt.json'), {**result, 'commit_requested': True,
|
||||
'recovery': 'Reconcile model ledger and committed receipt; never delete models automatically'})
|
||||
summaries = [json.loads(line) for line in sql_run(transaction(query, True)).splitlines() if line.startswith('{')]
|
||||
result.update(committed=True, changes=summaries)
|
||||
save_json(args.out.with_suffix('.committed.json'), result)
|
||||
after = target_snapshot()
|
||||
save_json(args.out.with_suffix('.after.json'), after)
|
||||
result['verification'] = verify(plan, before, after, summaries)
|
||||
save_json(args.out, result)
|
||||
print(json.dumps(result))
|
||||
return 0 if result.get('verification', {}).get('passed', True) else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Executable
+120
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the anydoc static archive that the `anydoc` build tag links.
|
||||
#
|
||||
# The archive is a Rust build artifact (~30 MB), so it is neither committed nor
|
||||
# downloaded: this script builds it from third_party/anydoc-go, which pins the
|
||||
# published anydoc crate, and drops it where cgo looks for it.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/build-anydoc-lib.sh # host platform
|
||||
# TARGET=aarch64-unknown-linux-musl scripts/build-anydoc-lib.sh
|
||||
set -euo pipefail
|
||||
|
||||
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
crate_dir="$repo_root/third_party/anydoc-go"
|
||||
|
||||
if ! command -v cargo >/dev/null 2>&1; then
|
||||
echo "error: cargo not found. Install Rust (https://rustup.rs) to build the anydoc archive." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The Rust target triple decides the archive; the directory name mirrors what
|
||||
# the cgo LDFLAGS in third_party/anydoc-go expect for that platform.
|
||||
target=${TARGET:-$(rustc -vV | sed -n 's/^host: //p')}
|
||||
case "$target" in
|
||||
x86_64-apple-darwin) lib_dir=darwin_amd64 ;;
|
||||
aarch64-apple-darwin) lib_dir=darwin_arm64 ;;
|
||||
x86_64-pc-windows-msvc) lib_dir=windows_amd64 ;;
|
||||
x86_64-unknown-linux-gnu) lib_dir=linux_amd64_gnu ;;
|
||||
aarch64-unknown-linux-gnu) lib_dir=linux_arm64_gnu ;;
|
||||
x86_64-unknown-linux-musl) lib_dir=linux_amd64_musl ;;
|
||||
aarch64-unknown-linux-musl) lib_dir=linux_arm64_musl ;;
|
||||
*)
|
||||
echo "error: unsupported target '$target'." >&2
|
||||
echo "Supported: {x86_64,aarch64}-{apple-darwin,unknown-linux-{gnu,musl}}, x86_64-pc-windows-msvc" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$target" in
|
||||
*windows-msvc) lib_name=anydoc_go.lib ;;
|
||||
*) lib_name=libanydoc_go.a ;;
|
||||
esac
|
||||
|
||||
# The crate version to patch is read from Cargo.toml rather than repeated here,
|
||||
# so an upgrade is a one-line change to the `anydoc = "=X.Y.Z"` pin.
|
||||
anydoc_version=$(sed -n 's/^anydoc = "=\([0-9][^"]*\)".*/\1/p' "$crate_dir/Cargo.toml" | head -1)
|
||||
if [ -z "$anydoc_version" ]; then
|
||||
echo "error: no pinned 'anydoc = \"=X.Y.Z\"' dependency in $crate_dir/Cargo.toml" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# document_to_markdown is private in published anydoc. Copy the pinned version
|
||||
# and re-export that one function so anydoc-go can keep the official serializer.
|
||||
prepare_patched_anydoc() {
|
||||
local dest="$crate_dir/patched-anydoc"
|
||||
# The marker records which version was patched: after a version bump the
|
||||
# copy left by the previous build is the wrong crate, and reusing it would
|
||||
# fail deep inside cargo's patch resolution instead of here. Cargo.toml is
|
||||
# required too: a truncated leftover with only the marker would otherwise
|
||||
# skip the copy and fail with "failed to read .../Cargo.toml".
|
||||
if [ -f "$dest/.weknora-patched" ] && [ "$(cat "$dest/.weknora-patched")" = "$anydoc_version" ] && [ -f "$dest/Cargo.toml" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
local src=""
|
||||
local cargo_home="${CARGO_HOME:-$HOME/.cargo}"
|
||||
src=$(find "$cargo_home/registry/src" -maxdepth 2 -type d -name "anydoc-$anydoc_version" 2>/dev/null | head -1 || true)
|
||||
|
||||
if [ -z "$src" ]; then
|
||||
local tarball="$crate_dir/.anydoc-$anydoc_version.crate"
|
||||
echo "Fetching anydoc $anydoc_version to patch document_to_markdown into the public API"
|
||||
if ! curl -fsSL "https://rsproxy.cn/api/v1/crates/anydoc/$anydoc_version/download" -o "$tarball"; then
|
||||
curl -fsSL "https://static.crates.io/crates/anydoc/anydoc-$anydoc_version.crate" -o "$tarball"
|
||||
fi
|
||||
local unpack="$crate_dir/.anydoc-unpack"
|
||||
rm -rf "$unpack"
|
||||
mkdir -p "$unpack"
|
||||
tar -xzf "$tarball" -C "$unpack"
|
||||
src=$(find "$unpack" -maxdepth 1 -type d -name "anydoc-$anydoc_version" | head -1)
|
||||
if [ -z "$src" ]; then
|
||||
echo "error: unpacked anydoc-$anydoc_version crate is missing" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
rm -rf "$dest"
|
||||
cp -R "$src" "$dest"
|
||||
if ! grep -q '^use render::markdown::document_to_markdown;$' "$dest/src/lib.rs"; then
|
||||
echo "error: anydoc $anydoc_version lib.rs no longer has the expected document_to_markdown import" >&2
|
||||
exit 1
|
||||
fi
|
||||
sed -i.bak 's/^use render::markdown::document_to_markdown;/pub use render::markdown::document_to_markdown;/' "$dest/src/lib.rs"
|
||||
rm -f "$dest/src/lib.rs.bak"
|
||||
printf '%s\n' "$anydoc_version" > "$dest/.weknora-patched"
|
||||
rm -rf "$crate_dir/.anydoc-unpack" "$crate_dir/.anydoc-$anydoc_version.crate"
|
||||
}
|
||||
|
||||
# version.go is what the parser reports as `anydoc_version` on every parsed
|
||||
# document, so a bump that misses it would mislabel the whole knowledge base.
|
||||
go_version=$(sed -n 's/^const Version = "\([^"]*\)".*/\1/p' "$crate_dir/version.go" | head -1)
|
||||
if [ "$go_version" != "$anydoc_version" ]; then
|
||||
echo "error: version.go says '$go_version' but Cargo.toml pins anydoc '$anydoc_version'." >&2
|
||||
echo "Bump both: the Go constant is the version WeKnora records for parsed documents." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Building anydoc $anydoc_version archive for $target"
|
||||
prepare_patched_anydoc
|
||||
|
||||
# --locked: build exactly the dependency versions in the committed Cargo.lock.
|
||||
# That lockfile is what pins the patched pdf-inspector/lopdf, so a silent
|
||||
# resolver drift must fail the build rather than ship an unaudited tree.
|
||||
cargo build --release --locked --manifest-path "$crate_dir/Cargo.toml" --target "$target"
|
||||
|
||||
dest="$crate_dir/lib/$lib_dir"
|
||||
mkdir -p "$dest"
|
||||
cp "$crate_dir/target/$target/release/$lib_name" "$dest/$lib_name"
|
||||
|
||||
echo "Wrote $dest/$lib_name"
|
||||
echo "Build WeKnora with the archive: go build -tags anydoc ./cmd/server"
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build frontend static assets for Docker / release packaging.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
if [ -z "${VITE_FRONTEND_COMMIT:-}" ]; then
|
||||
# shellcheck source=/dev/null
|
||||
eval "$("$PROJECT_ROOT/scripts/get_version.sh" env)"
|
||||
export VITE_FRONTEND_COMMIT="${COMMIT_ID:-unknown}"
|
||||
fi
|
||||
|
||||
export VITE_IS_DOCKER="${VITE_IS_DOCKER:-true}"
|
||||
|
||||
cd "$PROJECT_ROOT/frontend"
|
||||
npm ci
|
||||
npm run build
|
||||
Executable
+428
@@ -0,0 +1,428 @@
|
||||
#!/bin/bash
|
||||
# 该脚本用于从源码构建WeKnora的所有Docker镜像
|
||||
|
||||
# 设置颜色
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # 无颜色
|
||||
|
||||
# 获取项目根目录(脚本所在目录的上一级)
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
|
||||
|
||||
# 启用 BuildKit
|
||||
export DOCKER_BUILDKIT=1
|
||||
|
||||
# 版本信息
|
||||
VERSION="1.0.0"
|
||||
SCRIPT_NAME=$(basename "$0")
|
||||
|
||||
# 显示帮助信息
|
||||
show_help() {
|
||||
echo -e "${GREEN}WeKnora 镜像构建脚本 v${VERSION}${NC}"
|
||||
echo -e "${GREEN}用法:${NC} $0 [选项]"
|
||||
echo "选项:"
|
||||
echo " -h, --help 显示帮助信息"
|
||||
echo " -a, --all 构建所有镜像(默认)"
|
||||
echo " -p, --app 仅构建应用镜像"
|
||||
echo " -d, --docreader 仅构建文档读取器镜像"
|
||||
echo " -f, --frontend 仅构建前端镜像"
|
||||
echo " -s, --sandbox 仅构建沙箱镜像"
|
||||
echo " -c, --clean 清理所有本地镜像"
|
||||
echo " -v, --version 显示版本信息"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# 显示版本信息
|
||||
show_version() {
|
||||
echo -e "${GREEN}WeKnora 镜像构建脚本 v${VERSION}${NC}"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# 日志函数
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
# 检查Docker是否已安装
|
||||
check_docker() {
|
||||
log_info "检查Docker环境..."
|
||||
|
||||
if ! command -v docker &> /dev/null; then
|
||||
log_error "未安装Docker,请先安装Docker"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 检查Docker服务运行状态
|
||||
if ! docker info &> /dev/null; then
|
||||
log_error "Docker服务未运行,请启动Docker服务"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_success "Docker环境检查通过"
|
||||
return 0
|
||||
}
|
||||
|
||||
# 检测平台
|
||||
check_platform() {
|
||||
log_info "检测系统平台信息..."
|
||||
if [ "$(uname -m)" = "x86_64" ]; then
|
||||
export PLATFORM="linux/amd64"
|
||||
export TARGETARCH="amd64"
|
||||
elif [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then
|
||||
export PLATFORM="linux/arm64"
|
||||
export TARGETARCH="arm64"
|
||||
else
|
||||
log_warning "未识别的平台类型:$(uname -m),将使用默认平台 linux/amd64"
|
||||
export PLATFORM="linux/amd64"
|
||||
export TARGETARCH="amd64"
|
||||
fi
|
||||
log_info "当前平台:$PLATFORM"
|
||||
log_info "当前架构:$TARGETARCH"
|
||||
}
|
||||
|
||||
# 获取版本信息
|
||||
get_version_info() {
|
||||
# 从VERSION文件获取版本号
|
||||
if [ -f "VERSION" ]; then
|
||||
VERSION=$(cat VERSION | tr -d '\n\r')
|
||||
else
|
||||
VERSION="unknown"
|
||||
fi
|
||||
|
||||
# 获取commit ID
|
||||
if command -v git >/dev/null 2>&1; then
|
||||
COMMIT_ID=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||||
else
|
||||
COMMIT_ID="unknown"
|
||||
fi
|
||||
|
||||
# 获取构建时间
|
||||
BUILD_TIME=$(date -u '+%Y-%m-%d %H:%M:%S UTC')
|
||||
|
||||
# 获取Go版本
|
||||
if command -v go >/dev/null 2>&1; then
|
||||
GO_VERSION=$(go version 2>/dev/null || echo "unknown")
|
||||
else
|
||||
GO_VERSION="unknown"
|
||||
fi
|
||||
|
||||
log_info "版本信息: $VERSION"
|
||||
log_info "Commit ID: $COMMIT_ID"
|
||||
log_info "构建时间: $BUILD_TIME"
|
||||
log_info "Go版本: $GO_VERSION"
|
||||
}
|
||||
|
||||
# 构建应用镜像
|
||||
build_app_image() {
|
||||
log_info "构建应用镜像 (weknora-app)..."
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
# 获取版本信息
|
||||
get_version_info
|
||||
|
||||
docker build \
|
||||
--platform $PLATFORM \
|
||||
--build-arg GOPRIVATE_ARG=${GOPRIVATE:-""} \
|
||||
--build-arg GOPROXY_ARG=${GOPROXY:-"https://goproxy.cn,direct"} \
|
||||
--build-arg GOSUMDB_ARG=${GOSUMDB:-"off"} \
|
||||
--build-arg VERSION_ARG="$VERSION" \
|
||||
--build-arg COMMIT_ID_ARG="$COMMIT_ID" \
|
||||
--build-arg BUILD_TIME_ARG="$BUILD_TIME" \
|
||||
--build-arg GO_VERSION_ARG="$GO_VERSION" \
|
||||
--build-arg WITH_ANYDOC=${WITH_ANYDOC:-1} \
|
||||
-f docker/Dockerfile.app \
|
||||
-t wechatopenai/weknora-app:latest \
|
||||
.
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_success "应用镜像构建成功"
|
||||
return 0
|
||||
else
|
||||
log_error "应用镜像构建失败"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 构建文档读取器镜像
|
||||
build_docreader_image() {
|
||||
log_info "构建文档读取器镜像 (weknora-docreader)..."
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
docker build \
|
||||
--platform $PLATFORM \
|
||||
--build-arg PLATFORM=$PLATFORM \
|
||||
--build-arg TARGETARCH=$TARGETARCH \
|
||||
--build-arg APT_MIRROR=${APT_MIRROR:-} \
|
||||
-f docker/Dockerfile.docreader \
|
||||
-t wechatopenai/weknora-docreader:latest \
|
||||
.
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_success "文档读取器镜像构建成功"
|
||||
return 0
|
||||
else
|
||||
log_error "文档读取器镜像构建失败"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 构建前端镜像
|
||||
build_frontend_image() {
|
||||
log_info "构建前端镜像 (weknora-ui)..."
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
# 获取版本信息(用于注入前端 commit hash)
|
||||
get_version_info
|
||||
|
||||
log_info "构建前端静态资源..."
|
||||
VITE_IS_DOCKER=true VITE_FRONTEND_COMMIT="$COMMIT_ID" "$SCRIPT_DIR/build_frontend_dist.sh"
|
||||
|
||||
docker build \
|
||||
--platform $PLATFORM \
|
||||
-f frontend/Dockerfile \
|
||||
-t wechatopenai/weknora-ui:latest \
|
||||
frontend/
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_success "前端镜像构建成功"
|
||||
return 0
|
||||
else
|
||||
log_error "前端镜像构建失败"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 构建沙箱镜像
|
||||
build_sandbox_image() {
|
||||
log_info "构建沙箱镜像 (weknora-sandbox)..."
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
# 同时打上 main:默认镜像跟的是 main(见 sandbox.go 的 DefaultDockerImage),
|
||||
# 而 Docker 后端只在本地缺失时才去拉,本地不打这个标签就等于白构建。
|
||||
docker build \
|
||||
--platform $PLATFORM \
|
||||
-f docker/Dockerfile.sandbox \
|
||||
--target sandbox \
|
||||
-t wechatopenai/weknora-sandbox:latest \
|
||||
-t wechatopenai/weknora-sandbox:main \
|
||||
.
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
log_error "沙箱镜像构建失败"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Cube 从镜像直接构建模板,并以 :49983/health 探活,缺 envd 必然失败,
|
||||
# 因此 Cube 用的是注入了 envd 的变体镜像。详见 docs/sandbox-cluster.md。
|
||||
# 固定 linux/amd64:envd 的来源镜像 cubesandbox-base 不发布 arm64。
|
||||
log_info "构建沙箱镜像 Cube 变体 (weknora-sandbox:main-cube)..."
|
||||
|
||||
docker build \
|
||||
--platform linux/amd64 \
|
||||
-f docker/Dockerfile.sandbox \
|
||||
--target cube \
|
||||
-t wechatopenai/weknora-sandbox:latest-cube \
|
||||
-t wechatopenai/weknora-sandbox:main-cube \
|
||||
.
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_success "沙箱镜像构建成功"
|
||||
return 0
|
||||
else
|
||||
log_error "沙箱镜像 Cube 变体构建失败"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 构建所有镜像
|
||||
build_all_images() {
|
||||
log_info "开始构建所有镜像..."
|
||||
|
||||
local app_result=0
|
||||
local docreader_result=0
|
||||
local frontend_result=0
|
||||
local sandbox_result=0
|
||||
|
||||
# 构建应用镜像
|
||||
build_app_image
|
||||
app_result=$?
|
||||
|
||||
# 构建文档读取器镜像
|
||||
build_docreader_image
|
||||
docreader_result=$?
|
||||
|
||||
# 构建前端镜像
|
||||
build_frontend_image
|
||||
frontend_result=$?
|
||||
|
||||
# 构建沙箱镜像
|
||||
build_sandbox_image
|
||||
sandbox_result=$?
|
||||
|
||||
# 显示构建结果
|
||||
echo ""
|
||||
log_info "=== 构建结果 ==="
|
||||
if [ $app_result -eq 0 ]; then
|
||||
log_success "✓ 应用镜像构建成功"
|
||||
else
|
||||
log_error "✗ 应用镜像构建失败"
|
||||
fi
|
||||
|
||||
if [ $docreader_result -eq 0 ]; then
|
||||
log_success "✓ 文档读取器镜像构建成功"
|
||||
else
|
||||
log_error "✗ 文档读取器镜像构建失败"
|
||||
fi
|
||||
|
||||
if [ $frontend_result -eq 0 ]; then
|
||||
log_success "✓ 前端镜像构建成功"
|
||||
else
|
||||
log_error "✗ 前端镜像构建失败"
|
||||
fi
|
||||
|
||||
if [ $sandbox_result -eq 0 ]; then
|
||||
log_success "✓ 沙箱镜像构建成功"
|
||||
else
|
||||
log_error "✗ 沙箱镜像构建失败"
|
||||
fi
|
||||
|
||||
if [ $app_result -eq 0 ] && [ $docreader_result -eq 0 ] && [ $frontend_result -eq 0 ] && [ $sandbox_result -eq 0 ]; then
|
||||
log_success "所有镜像构建完成!"
|
||||
return 0
|
||||
else
|
||||
log_error "部分镜像构建失败"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 清理本地镜像
|
||||
clean_images() {
|
||||
log_info "清理本地WeKnora镜像..."
|
||||
|
||||
# 停止相关容器
|
||||
log_info "停止相关容器..."
|
||||
docker stop $(docker ps -q --filter "ancestor=wechatopenai/weknora-app:latest" 2>/dev/null) 2>/dev/null || true
|
||||
docker stop $(docker ps -q --filter "ancestor=wechatopenai/weknora-docreader:latest" 2>/dev/null) 2>/dev/null || true
|
||||
docker stop $(docker ps -q --filter "ancestor=wechatopenai/weknora-ui:latest" 2>/dev/null) 2>/dev/null || true
|
||||
|
||||
# 删除相关容器
|
||||
log_info "删除相关容器..."
|
||||
docker rm $(docker ps -aq --filter "ancestor=wechatopenai/weknora-app:latest" 2>/dev/null) 2>/dev/null || true
|
||||
docker rm $(docker ps -aq --filter "ancestor=wechatopenai/weknora-docreader:latest" 2>/dev/null) 2>/dev/null || true
|
||||
docker rm $(docker ps -aq --filter "ancestor=wechatopenai/weknora-ui:latest" 2>/dev/null) 2>/dev/null || true
|
||||
|
||||
# 删除镜像
|
||||
log_info "删除本地镜像..."
|
||||
docker rmi wechatopenai/weknora-app:latest 2>/dev/null || true
|
||||
docker rmi wechatopenai/weknora-docreader:latest 2>/dev/null || true
|
||||
docker rmi wechatopenai/weknora-ui:latest 2>/dev/null || true
|
||||
docker rmi wechatopenai/weknora-sandbox:latest 2>/dev/null || true
|
||||
docker rmi wechatopenai/weknora-sandbox:latest-cube 2>/dev/null || true
|
||||
docker rmi wechatopenai/weknora-sandbox:main 2>/dev/null || true
|
||||
docker rmi wechatopenai/weknora-sandbox:main-cube 2>/dev/null || true
|
||||
|
||||
docker image prune -f
|
||||
|
||||
log_success "镜像清理完成"
|
||||
return 0
|
||||
}
|
||||
|
||||
# 解析命令行参数
|
||||
BUILD_ALL=false
|
||||
BUILD_APP=false
|
||||
BUILD_DOCREADER=false
|
||||
BUILD_FRONTEND=false
|
||||
BUILD_SANDBOX=false
|
||||
CLEAN_IMAGES=false
|
||||
|
||||
# 没有参数时默认构建所有镜像
|
||||
if [ $# -eq 0 ]; then
|
||||
BUILD_ALL=true
|
||||
fi
|
||||
|
||||
while [ "$1" != "" ]; do
|
||||
case $1 in
|
||||
-h | --help ) show_help
|
||||
;;
|
||||
-a | --all ) BUILD_ALL=true
|
||||
;;
|
||||
-p | --app ) BUILD_APP=true
|
||||
;;
|
||||
-d | --docreader ) BUILD_DOCREADER=true
|
||||
;;
|
||||
-f | --frontend ) BUILD_FRONTEND=true
|
||||
;;
|
||||
-s | --sandbox ) BUILD_SANDBOX=true
|
||||
;;
|
||||
-c | --clean ) CLEAN_IMAGES=true
|
||||
;;
|
||||
-v | --version ) show_version
|
||||
;;
|
||||
* ) log_error "未知选项: $1"
|
||||
show_help
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# 检查Docker环境
|
||||
check_docker
|
||||
if [ $? -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 检测平台
|
||||
check_platform
|
||||
|
||||
# 执行清理操作
|
||||
if [ "$CLEAN_IMAGES" = true ]; then
|
||||
clean_images
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# 执行构建操作
|
||||
if [ "$BUILD_ALL" = true ]; then
|
||||
build_all_images
|
||||
exit $?
|
||||
fi
|
||||
|
||||
if [ "$BUILD_APP" = true ]; then
|
||||
build_app_image
|
||||
exit $?
|
||||
fi
|
||||
|
||||
if [ "$BUILD_DOCREADER" = true ]; then
|
||||
build_docreader_image
|
||||
exit $?
|
||||
fi
|
||||
|
||||
if [ "$BUILD_FRONTEND" = true ]; then
|
||||
build_frontend_image
|
||||
exit $?
|
||||
fi
|
||||
|
||||
if [ "$BUILD_SANDBOX" = true ]; then
|
||||
build_sandbox_image
|
||||
exit $?
|
||||
fi
|
||||
|
||||
exit 0
|
||||
Executable
+208
@@ -0,0 +1,208 @@
|
||||
#!/bin/bash
|
||||
# 检查开发环境配置
|
||||
|
||||
# 设置颜色
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # 无颜色
|
||||
|
||||
# 获取项目根目录
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
|
||||
|
||||
log_info() {
|
||||
printf "%b\n" "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
printf "%b\n" "${GREEN}[✓]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
printf "%b\n" "${RED}[✗]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
printf "%b\n" "${YELLOW}[!]${NC} $1"
|
||||
}
|
||||
|
||||
echo ""
|
||||
printf "%b\n" "${GREEN}========================================${NC}"
|
||||
printf "%b\n" "${GREEN} WeKnora 开发环境配置检查${NC}"
|
||||
printf "%b\n" "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
# 检查 .env 文件
|
||||
log_info "检查 .env 文件..."
|
||||
if [ -f ".env" ]; then
|
||||
log_success ".env 文件存在"
|
||||
else
|
||||
log_error ".env 文件不存在"
|
||||
echo ""
|
||||
log_info "解决方法:"
|
||||
echo " 1. 复制示例文件: cp .env.example .env"
|
||||
echo " 2. 编辑 .env 文件并配置必要的环境变量"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
log_info "检查必要的环境变量..."
|
||||
|
||||
# 加载 .env 文件
|
||||
set -a
|
||||
source .env
|
||||
set +a
|
||||
|
||||
# 检查必要的环境变量
|
||||
errors=0
|
||||
|
||||
check_var() {
|
||||
local var_name=$1
|
||||
local var_value="${!var_name}"
|
||||
|
||||
if [ -z "$var_value" ]; then
|
||||
log_error "$var_name 未设置"
|
||||
errors=$((errors + 1))
|
||||
else
|
||||
log_success "$var_name = $var_value"
|
||||
fi
|
||||
}
|
||||
|
||||
# 数据库配置
|
||||
log_info "数据库配置:"
|
||||
check_var "DB_DRIVER"
|
||||
check_var "DB_HOST"
|
||||
check_var "DB_PORT"
|
||||
check_var "DB_USER"
|
||||
check_var "DB_PASSWORD"
|
||||
check_var "DB_NAME"
|
||||
|
||||
echo ""
|
||||
log_info "存储配置:"
|
||||
check_var "STORAGE_TYPE"
|
||||
|
||||
if [ "$STORAGE_TYPE" = "minio" ]; then
|
||||
check_var "MINIO_BUCKET_NAME"
|
||||
fi
|
||||
|
||||
if [ "$STORAGE_TYPE" = "tos" ]; then
|
||||
check_var "TOS_ENDPOINT"
|
||||
check_var "TOS_REGION"
|
||||
check_var "TOS_ACCESS_KEY"
|
||||
check_var "TOS_SECRET_KEY"
|
||||
check_var "TOS_BUCKET_NAME"
|
||||
fi
|
||||
|
||||
if [ "$STORAGE_TYPE" = "s3" ]; then
|
||||
check_var "S3_REGION"
|
||||
check_var "S3_BUCKET_NAME"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
log_info "Redis 配置:"
|
||||
check_var "REDIS_ADDR"
|
||||
|
||||
echo ""
|
||||
log_info "Ollama 配置:"
|
||||
check_var "OLLAMA_BASE_URL"
|
||||
|
||||
echo ""
|
||||
log_info "模型配置:"
|
||||
if [ -n "$INIT_LLM_MODEL_NAME" ]; then
|
||||
log_success "INIT_LLM_MODEL_NAME = $INIT_LLM_MODEL_NAME"
|
||||
else
|
||||
log_warning "INIT_LLM_MODEL_NAME 未设置(可选)"
|
||||
fi
|
||||
|
||||
if [ -n "$INIT_EMBEDDING_MODEL_NAME" ]; then
|
||||
log_success "INIT_EMBEDDING_MODEL_NAME = $INIT_EMBEDDING_MODEL_NAME"
|
||||
else
|
||||
log_warning "INIT_EMBEDDING_MODEL_NAME 未设置(可选)"
|
||||
fi
|
||||
|
||||
# 检查 Go 环境
|
||||
echo ""
|
||||
log_info "检查 Go 环境..."
|
||||
if command -v go &> /dev/null; then
|
||||
go_version=$(go version)
|
||||
log_success "Go 已安装: $go_version"
|
||||
else
|
||||
log_error "Go 未安装"
|
||||
errors=$((errors + 1))
|
||||
fi
|
||||
|
||||
# 检查 Air
|
||||
if command -v air &> /dev/null; then
|
||||
log_success "Air 已安装(支持热重载)"
|
||||
else
|
||||
log_warning "Air 未安装(可选,用于热重载)"
|
||||
log_info "安装命令: go install github.com/air-verse/air@latest"
|
||||
fi
|
||||
|
||||
# 检查 npm
|
||||
echo ""
|
||||
log_info "检查 Node.js 环境..."
|
||||
if command -v npm &> /dev/null; then
|
||||
npm_version=$(npm --version)
|
||||
log_success "npm 已安装: $npm_version"
|
||||
else
|
||||
log_error "npm 未安装"
|
||||
errors=$((errors + 1))
|
||||
fi
|
||||
|
||||
# 检查 Docker
|
||||
echo ""
|
||||
log_info "检查 Docker 环境..."
|
||||
if command -v docker &> /dev/null; then
|
||||
docker_version=$(docker --version)
|
||||
log_success "Docker 已安装: $docker_version"
|
||||
|
||||
if docker info &> /dev/null; then
|
||||
log_success "Docker 服务正在运行"
|
||||
else
|
||||
log_error "Docker 服务未运行"
|
||||
errors=$((errors + 1))
|
||||
fi
|
||||
else
|
||||
log_error "Docker 未安装"
|
||||
errors=$((errors + 1))
|
||||
fi
|
||||
|
||||
# 检查 Docker Compose
|
||||
if docker compose version &> /dev/null; then
|
||||
compose_version=$(docker compose version)
|
||||
log_success "Docker Compose 已安装: $compose_version"
|
||||
elif command -v docker-compose &> /dev/null; then
|
||||
compose_version=$(docker-compose --version)
|
||||
log_success "docker-compose 已安装: $compose_version"
|
||||
else
|
||||
log_error "Docker Compose 未安装"
|
||||
errors=$((errors + 1))
|
||||
fi
|
||||
|
||||
# 总结
|
||||
echo ""
|
||||
printf "%b\n" "${GREEN}========================================${NC}"
|
||||
if [ $errors -eq 0 ]; then
|
||||
log_success "所有检查通过!环境配置正常"
|
||||
echo ""
|
||||
log_info "下一步:"
|
||||
echo " 1. 启动开发环境: make dev-start"
|
||||
echo " 2. 启动后端: make dev-app"
|
||||
echo " 3. 启动前端: make dev-frontend"
|
||||
else
|
||||
log_error "发现 $errors 个问题,请修复后再启动开发环境"
|
||||
echo ""
|
||||
log_info "常见问题:"
|
||||
echo " - 如果 .env 文件不存在,请复制 .env.example"
|
||||
echo " - 确保 DB_DRIVER 设置为 'postgres' 或 'mysql'"
|
||||
echo " - 确保 Docker 服务正在运行"
|
||||
fi
|
||||
printf "%b\n" "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
exit $errors
|
||||
@@ -0,0 +1,34 @@
|
||||
const fs = require('node:fs')
|
||||
const assert = require('node:assert/strict')
|
||||
const root = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
|
||||
const base = 'http://127.0.0.1:18080/api/v1'
|
||||
const fixtures = [
|
||||
['training-maps','c9fb3c98-a9f3-40f0-8ac0-2756c83288de'],
|
||||
['training-maps','013a398f-fe0e-4287-8f3f-6fe7d6f468f9'],
|
||||
['flows','1011c8bc-a0f4-4baf-abab-cd1c05a85889'],
|
||||
['flows','15fc92d5-d0f8-416f-97bc-1927ffe39a71'],
|
||||
['agents','6065fcd5-d827-40a1-8fe4-902906576b85'],
|
||||
['agents','e69f8f18-010d-43c7-be2c-4cd1409df2aa'],
|
||||
]
|
||||
async function main() {
|
||||
const c=JSON.parse(fs.readFileSync(`${root}/super-admin-login.json`,'utf8'))
|
||||
const login=await fetch(`${base}/auth/login`,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({email:c.email,password:c.password})})
|
||||
assert.equal(login.status,200)
|
||||
const auth=await login.json()
|
||||
const headers={Authorization:`Bearer ${auth.token}`,'X-Tenant-ID':'10059'}
|
||||
const archive=[]
|
||||
for(const [kind,id] of fixtures) {
|
||||
const path=kind==='agents'?`/agents/${id}`:`/workbench/resources/${kind}/${id}`
|
||||
const r=await fetch(base+path,{headers})
|
||||
if(r.status===404) continue
|
||||
assert.equal(r.status,200)
|
||||
const row=(await r.json()).data
|
||||
assert.ok(/^real-generation-178958(7501724|7865185)/.test(row.name),'Only known failed verification fixtures may be removed')
|
||||
archive.push({kind,row})
|
||||
fs.writeFileSync(`${root}/failed-generation-fixtures-archived.json`,JSON.stringify(archive,null,2),{mode:0o600})
|
||||
const deletion=await fetch(base+path,{method:'DELETE',headers})
|
||||
assert.ok([200,204].includes(deletion.status),`Fixture deletion ${deletion.status}`)
|
||||
}
|
||||
console.log(JSON.stringify({removed:archive.length,preserved:'Successful real-generation sample, usage ledger and test evidence'}))
|
||||
}
|
||||
main().catch(e=>{console.error(e.message);process.exitCode=1})
|
||||
@@ -0,0 +1,276 @@
|
||||
# WeKnora 云镜像打包脚本(Cloud-Agnostic)
|
||||
|
||||
> **本文档面向「想把 WeKnora 打包成云镜像(AMI / 自定义镜像 / Snapshot)分发给其他人」的用户。**
|
||||
> **如果你只是想自己用 WeKnora,请直接看主仓 [README](../../README.md),`docker compose up -d` 即可。**
|
||||
|
||||
## 这套脚本能做什么
|
||||
|
||||
帮你把任意一台「能跑 Docker 的 Linux 实例」变成一份**可分发的云镜像模板**:
|
||||
|
||||
- 别人基于这份镜像创建新实例后,**首次开机会自动**:
|
||||
- 生成全新的随机密钥(DB / Redis / JWT / AES)
|
||||
- 启动 WeKnora 全部默认容器
|
||||
- 把生成的凭证写到 `/root/weknora-credentials.txt`
|
||||
- 自删除一次性初始化脚本
|
||||
- 实现「**开机即用、零私密泄漏、每实例独立密钥**」
|
||||
|
||||
适用平台(任意 systemd + Docker 的 Linux 都行):
|
||||
|
||||
- 腾讯云轻量应用服务器(Lighthouse) / 云服务器 CVM
|
||||
- AWS EC2 AMI
|
||||
- 阿里云 ECS 自定义镜像
|
||||
- 火山引擎 / 华为云 / Vultr Snapshot
|
||||
- 本地 KVM / Proxmox 模板
|
||||
|
||||
各平台具体的「制作镜像 / 共享 / 上架」操作步骤,请参考 [`docs/cloud-image/`](../../docs/cloud-image/) 下对应文档。
|
||||
|
||||
---
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
scripts/cloud-image/
|
||||
├── README.md # 本文档
|
||||
├── prepare.sh # 步骤一: 装 Docker + 拉运行时 + 装 firstboot
|
||||
├── cleanup.sh # 步骤二: 制作镜像前清理(执行后将锁定 SSH)
|
||||
├── firstboot.sh # 新实例首次开机自动执行(用户无感)
|
||||
└── systemd/
|
||||
├── weknora.service # 开机自启 docker compose
|
||||
└── weknora-firstboot.service # 首次启动 init(执行后自删)
|
||||
```
|
||||
|
||||
## 不需要 clone 整个 WeKnora 仓库
|
||||
|
||||
WeKnora 所有容器都从 Docker Hub 拉镜像(`wechatopenai/weknora-*`),Go / Python / 前端源码都不需要带到宿主机。
|
||||
|
||||
`docker-compose.yml` 实际从宿主机挂载到容器的只有:
|
||||
|
||||
```
|
||||
- ./config/config.yaml (单文件)
|
||||
- ./skills/preloaded/ (目录)
|
||||
```
|
||||
|
||||
所以镜像里需要的运行时文件**总共不到 100KB**:
|
||||
|
||||
| 文件 | 大小 | 用途 |
|
||||
|---|---|---|
|
||||
| `docker-compose.yml` | 12K | 容器编排 |
|
||||
| `.env` | 12K | 环境变量 |
|
||||
| `config/config.yaml` | 8K | 后端业务配置 |
|
||||
| `skills/preloaded/` | 56K | Agent 预置技能 |
|
||||
|
||||
`prepare.sh` 用 `curl + tar` 只下载这 4 项,不 `git clone`。
|
||||
|
||||
## 镜像里启动哪些容器
|
||||
|
||||
WeKnora `docker-compose.yml` 大量服务是 **profile 限定**,本镜像只默认启动核心 5 个。
|
||||
|
||||
**默认启动(5 个常驻容器,开机自启):**
|
||||
|
||||
| 容器 | 角色 |
|
||||
|---|---|
|
||||
| `frontend` | Vue UI / NGINX 反代 |
|
||||
| `app` | WeKnora Go 后端 |
|
||||
| `docreader` | Python 文档解析 (gRPC) |
|
||||
| `postgres` (ParadeDB) | 主库 + pgvector 向量检索 + BM25 |
|
||||
| `redis` | 流式输出 / 缓存 / 异步队列 |
|
||||
|
||||
> ParadeDB 自带 pgvector,默认场景下不需额外起向量库。
|
||||
|
||||
**额外预拉但不常驻:**
|
||||
|
||||
- `sandbox` 镜像:Agent Skills 由 app 按需 `docker run`。提前 `pull` 避免新实例首次执行 Skill 卡在下载。
|
||||
|
||||
**Profile 限定,不预装(用户需要时自己 `pull`):**
|
||||
|
||||
| profile | 用途 |
|
||||
|---|---|
|
||||
| `minio` | 对象存储替代本地文件 |
|
||||
| `qdrant` / `milvus` / `weaviate` / `doris` | 替代 pgvector |
|
||||
| `neo4j` | GraphRAG 知识图谱 |
|
||||
| `langfuse` | 自建 Langfuse 可观测平台 |
|
||||
| `dex` | OIDC 登录 |
|
||||
| `odl-hybrid` | OpenDataLoader Docling hybrid(体积大,无预发布镜像,需 `--build`) |
|
||||
|
||||
启用方式:
|
||||
|
||||
```bash
|
||||
cd /opt/WeKnora
|
||||
docker compose --profile neo4j up -d # 启用 GraphRAG
|
||||
docker compose --profile langfuse up -d # 启用自建 Langfuse
|
||||
docker compose --profile qdrant up -d # 切换到 Qdrant
|
||||
docker compose --profile odl-hybrid up -d --build odl-hybrid # Docling hybrid(按需)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 完整流程(云无关)
|
||||
|
||||
```
|
||||
1) 在目标云上买/装一台干净 Linux 实例(建议 4C8G+,Ubuntu 22.04)
|
||||
2) SSH 进去, 拷入本目录, 执行 prepare.sh
|
||||
3) 浏览器验证功能
|
||||
4) 执行 cleanup.sh (清掉私密 + SSH key, 自动关机)
|
||||
5) 在云控制台「制作镜像 / 创建快照 / 创建 AMI」
|
||||
6) 用新镜像创建测试实例, 验证 firstboot 工作正常
|
||||
7) 共享 / 公开镜像(参考各平台文档)
|
||||
```
|
||||
|
||||
### 步骤一:在干净实例上部署
|
||||
|
||||
要求:systemd + 联网 + sudo 权限。推荐 Ubuntu 22.04 / Debian 12 / CentOS Stream 9。
|
||||
|
||||
**1. 拷入脚本(任选一种,都不用 clone 整个 WeKnora 仓库)。**
|
||||
|
||||
> 命令需要写入 `/opt/`,最省心的做法是先 `sudo -i` 切到 root 再粘贴。
|
||||
> 如果坚持每行加 `sudo`,注意 `>>` 重定向是在你当前 shell 执行的,必须改用 `sudo tee -a`。
|
||||
|
||||
> **中国大陆云主机请直接看方式 C (scp)**。实测腾讯云 / 阿里云轻量服务器经常连 `github.com`、`raw.githubusercontent.com`、`gh-proxy.com` 这类境外 / 公益代理都连不上 (TLS RST 或超时),方式 A/B 会全军覆没。本机已经有这份仓库,scp 上去最稳。
|
||||
|
||||
```bash
|
||||
sudo -i # 切到 root, 后续命令直接执行
|
||||
|
||||
# === 方式 A: sparse checkout (~60KB) ===
|
||||
# 不通时设 GH_PROXY=https://gh-proxy.com/ 或 https://ghfast.top/, 注意末尾斜杠。
|
||||
GH_PROXY="${GH_PROXY:-}"
|
||||
mkdir -p /opt/weknora-tools && cd /opt/weknora-tools
|
||||
git init -q && git remote add origin "${GH_PROXY}https://github.com/Tencent/WeKnora.git"
|
||||
git config core.sparseCheckout true
|
||||
echo "scripts/cloud-image/" >> .git/info/sparse-checkout
|
||||
git pull -q --depth=1 origin main
|
||||
|
||||
# === 方式 B: 直接 curl (无 git 时用这个) ===
|
||||
# 不通时设 GH_PROXY=https://gh-proxy.com/ 或 https://ghfast.top/, 注意末尾斜杠。
|
||||
GH_PROXY="${GH_PROXY:-}"
|
||||
mkdir -p /opt/weknora-tools/scripts/cloud-image/systemd && cd /opt/weknora-tools
|
||||
base="${GH_PROXY}https://raw.githubusercontent.com/Tencent/WeKnora/main/scripts/cloud-image"
|
||||
for f in prepare.sh cleanup.sh firstboot.sh README.md; do
|
||||
curl -fsSL "$base/$f" -o "scripts/cloud-image/$f"
|
||||
done
|
||||
for f in weknora.service weknora-firstboot.service; do
|
||||
curl -fsSL "$base/systemd/$f" -o "scripts/cloud-image/systemd/$f"
|
||||
done
|
||||
chmod +x scripts/cloud-image/*.sh
|
||||
|
||||
# === 方式 C: 从本地 scp 上来 (推荐: 中国大陆云主机直接走这条) ===
|
||||
# 在本机 (能正常访问 GitHub 的机器) 执行:
|
||||
# scp -r scripts/cloud-image root@<实例IP>:/opt/weknora-tools/scripts/
|
||||
```
|
||||
|
||||
> 不确定 VM 能不能访问代理时,先探一下:
|
||||
> `for h in gh-proxy.com ghfast.top mirror.ghproxy.com github.moeyy.xyz kkgithub.com; do printf '%-25s' "$h"; curl -sS -o /dev/null -m 5 -w 'http=%{http_code} t=%{time_total}s\n' "https://$h/" 2>&1 || echo FAIL; done`
|
||||
> 哪个返回 `http=200/301/302` 就把 `GH_PROXY` 设成它 (后面加 `/`)。一个都不通的话,认命走方式 C。
|
||||
|
||||
**2. 执行部署:**
|
||||
|
||||
```bash
|
||||
sudo bash /opt/weknora-tools/scripts/cloud-image/prepare.sh
|
||||
|
||||
# 想 pin 特定版本(推荐, 保证镜像可复现)
|
||||
sudo WEKNORA_REF=v0.5.0 bash /opt/weknora-tools/scripts/cloud-image/prepare.sh
|
||||
|
||||
# 中国大陆机器三件套: 同时绕开 GitHub / get.docker.com / Docker Hub 的境外 CDN
|
||||
# (以腾讯云为例, 阿里云 / 华为云换对应镜像即可)
|
||||
sudo \
|
||||
WEKNORA_REF=v0.5.0 \
|
||||
WEKNORA_GH_PROXY=https://gh-proxy.com/ \
|
||||
DOCKER_INSTALL_MIRROR=https://mirrors.tencent.com/docker-ce/linux/ubuntu \
|
||||
DOCKER_REGISTRY_MIRROR=https://mirror.ccs.tencentyun.com \
|
||||
bash /opt/weknora-tools/scripts/cloud-image/prepare.sh
|
||||
```
|
||||
|
||||
> 三个变量分别解决三个不同的境外 CDN 不可达问题:
|
||||
> - `WEKNORA_GH_PROXY`:加速 **GitHub tarball** 下载(`prepare.sh` 步骤 2,运行时文件)
|
||||
> - `DOCKER_INSTALL_MIRROR`:绕开 **`get.docker.com`**,改用 apt + docker-ce 镜像源装 Docker(步骤 1)
|
||||
> - `DOCKER_REGISTRY_MIRROR`:加速 **Docker Hub** 镜像拉取(步骤 4,`wechatopenai/weknora-*`)
|
||||
>
|
||||
> 不同云厂商对应地址(按需替换 ubuntu/debian 部分以匹配实际发行版):
|
||||
> | 厂商 | `DOCKER_INSTALL_MIRROR` | `DOCKER_REGISTRY_MIRROR` |
|
||||
> |---|---|---|
|
||||
> | 腾讯云 | `https://mirrors.tencent.com/docker-ce/linux/ubuntu` | `https://mirror.ccs.tencentyun.com` |
|
||||
> | 阿里云 | `https://mirrors.aliyun.com/docker-ce/linux/ubuntu` | `https://<your-id>.mirror.aliyuncs.com` |
|
||||
> | 华为云 | `https://mirrors.huaweicloud.com/docker-ce/linux/ubuntu` | `https://<id>.mirror.swr.myhuaweicloud.com` |
|
||||
>
|
||||
> `DOCKER_INSTALL_MIRROR` 目前仅支持 apt 系(Ubuntu / Debian / TencentOS-apt)。
|
||||
> CentOS / Rocky 等 yum 系发行版 `get.docker.com` 一般能直连,没碰到再说。
|
||||
|
||||
`prepare.sh` 会:
|
||||
|
||||
1. 安装 Docker / Docker Compose plugin(已装则跳过)
|
||||
2. 用 `curl + tar` 下载 4 个运行时文件到 `/opt/WeKnora`
|
||||
3. 拉取并启动默认 5 个容器 + 预拉 sandbox 镜像
|
||||
4. 安装 `weknora.service`(开机自启)+ `weknora-firstboot.service`(首启 init)
|
||||
|
||||
完成后访问 `http://<公网IP>`。
|
||||
|
||||
### 步骤二:验证
|
||||
|
||||
至少验证:
|
||||
|
||||
- 能注册管理员、能登录
|
||||
- 能创建一个知识库
|
||||
- 能上传一个文档并完成解析
|
||||
- 能进行一次问答
|
||||
|
||||
```bash
|
||||
sudo docker compose -f /opt/WeKnora/docker-compose.yml ps
|
||||
curl -f http://localhost:8080/health
|
||||
```
|
||||
|
||||
### 步骤三:清理并制作镜像
|
||||
|
||||
> **重要**:`cleanup.sh` 会删除所有 SSH 公钥、清空日志、清空数据库与 docker volume。执行后**不要再 SSH 进来**,直接去云控制台关机制作镜像。
|
||||
|
||||
```bash
|
||||
sudo bash /opt/weknora-tools/scripts/cloud-image/cleanup.sh
|
||||
```
|
||||
|
||||
执行完会自动 `poweroff`。然后到对应云控制台按其文档制作镜像。
|
||||
|
||||
### 新实例首次开机行为
|
||||
|
||||
用户用你的镜像创建实例后,第一次开机时 `weknora-firstboot.service` 会:
|
||||
|
||||
1. 生成随机的 `DB_PASSWORD` / `REDIS_PASSWORD` / `JWT_SECRET` / `SYSTEM_AES_KEY`
|
||||
2. 写回 `/opt/WeKnora/.env`
|
||||
3. `docker compose up -d` 启动全部服务
|
||||
4. 把生成的凭证写到 `/root/weknora-credentials.txt`(仅 root 可读)
|
||||
5. 把自己 disable + 删除自己(确保只跑一次)
|
||||
|
||||
之后每次开机都由 `weknora.service` 接管。
|
||||
|
||||
> **注意**:`firstboot.sh` 默认**不**禁用注册(`DISABLE_REGISTRATION=false`),第一个注册的人会成为管理员。
|
||||
> 凭证文件里有「请尽快注册以防被抢注」的醒目提示。需要更严格控制可在 `firstboot.sh` 的 `replace` 调用列表中追加一行 `replace DISABLE_REGISTRATION true` 后再重制镜像。
|
||||
|
||||
---
|
||||
|
||||
## 升级镜像版本
|
||||
|
||||
镜像里没有 git 仓库,升级直接重跑 `prepare.sh`(会覆盖 4 个运行时文件,**不动 `.env` 和 docker volume 数据**):
|
||||
|
||||
```bash
|
||||
sudo WEKNORA_REF=v0.6.0 bash /opt/weknora-tools/scripts/cloud-image/prepare.sh
|
||||
sudo bash /opt/weknora-tools/scripts/cloud-image/cleanup.sh # 制作新镜像前
|
||||
```
|
||||
|
||||
> **打新镜像时想顺便清掉旧版本镜像层**(每个旧 weknora-* tag 几百 MB,4 个镜像 ~2-4GB):
|
||||
> ```bash
|
||||
> sudo PRUNE_OLD_IMAGES=true WEKNORA_REF=v0.6.0 \
|
||||
> bash /opt/weknora-tools/scripts/cloud-image/prepare.sh
|
||||
> ```
|
||||
> 默认 `false` 是为了保留回滚路径。打镜像前确认新版本稳定后再开。
|
||||
|
||||
## 安全注意事项
|
||||
|
||||
- 镜像里**不要**预置任何 LLM API Key、Langfuse Key、个人 SSH key
|
||||
- 数据库 / Redis / MinIO 端口默认仅对 docker 网络可见,不要在云防火墙里对外开放
|
||||
- `/root/weknora-credentials.txt` 用 `umask 077` 创建,仅 root 可读
|
||||
- 每次重制镜像前必须执行 `cleanup.sh`,避免泄漏上一份测试数据 / SSH key / machine-id
|
||||
|
||||
## 各云平台具体操作
|
||||
|
||||
| 平台 | 文档 |
|
||||
|---|---|
|
||||
| 腾讯云轻量应用服务器 / CVM | [`docs/cloud-image/tencent-lighthouse.md`](../../docs/cloud-image/tencent-lighthouse.md) |
|
||||
| AWS EC2 AMI | (欢迎贡献) |
|
||||
| 阿里云 ECS | (欢迎贡献) |
|
||||
Executable
+94
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env bash
|
||||
# cleanup.sh - 在制作云镜像前清理私密数据。
|
||||
# 警告: 本脚本会删除 SSH 公钥、清空数据库与日志,最后会自动关机。
|
||||
# 执行后请直接在云控制台「制作镜像 / 创建快照 / 创建 AMI」,不要再 SSH 进来。
|
||||
set -euo pipefail
|
||||
|
||||
WEKNORA_DIR="${WEKNORA_DIR:-/opt/WeKnora}"
|
||||
|
||||
if [[ "${EUID}" -ne 0 ]]; then
|
||||
echo "[cleanup] 请使用 sudo 或 root 运行" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
read -r -p "[cleanup] 该操作不可逆,确认继续? 输入 YES 继续: " ans
|
||||
if [[ "${ans}" != "YES" ]]; then
|
||||
echo "[cleanup] 已取消"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[cleanup] 1/8 停止 WeKnora 容器"
|
||||
COMPOSE_PROJECT=""
|
||||
if [[ -d "${WEKNORA_DIR}" ]]; then
|
||||
cd "${WEKNORA_DIR}"
|
||||
# 优先用 compose ls 拿到真实 project 名 (默认是目录名小写, 如 weknora)
|
||||
COMPOSE_PROJECT="$(docker compose ls --format json 2>/dev/null \
|
||||
| grep -oE '"Name":"[^"]+"' | head -1 | cut -d'"' -f4 || true)"
|
||||
docker compose down -v --remove-orphans || true
|
||||
fi
|
||||
|
||||
echo "[cleanup] 2/8 清空 WeKnora 业务数据 + 首启 marker / 日志"
|
||||
if [[ -d "${WEKNORA_DIR}" ]]; then
|
||||
rm -rf "${WEKNORA_DIR}/data"/* "${WEKNORA_DIR}/logs"/* 2>/dev/null || true
|
||||
# 故意不在这里重建 .env: 让镜像里 .env 缺失, 任何在 firstboot 之前
|
||||
# 起来的 docker compose 都会因找不到 .env 而失败, 避免明文默认密码
|
||||
# (postgres123!@# 之类) 把 postgres 数据卷初始化坏。
|
||||
# firstboot.sh 自己会从 .env.example 拷贝并替换密钥。
|
||||
rm -f "${WEKNORA_DIR}/.env" "${WEKNORA_DIR}/.firstboot.done"
|
||||
fi
|
||||
rm -f /root/weknora-credentials.txt /var/log/weknora-firstboot.log
|
||||
|
||||
echo "[cleanup] 3/8 清理残留 docker 卷与构建缓存"
|
||||
# 严格按 compose project 名前缀匹配, 避免误伤同宿主上其它 postgres/redis 卷。
|
||||
if [[ -n "${COMPOSE_PROJECT}" ]]; then
|
||||
docker volume ls -q --filter "label=com.docker.compose.project=${COMPOSE_PROJECT}" \
|
||||
| xargs -r docker volume rm -f || true
|
||||
fi
|
||||
# 注意: 这里只清"卷 / 已停容器 / 构建缓存", 绝对不能清镜像。
|
||||
# 此前用过 `docker system prune -af --volumes`, 会把 prepare.sh 预拉的
|
||||
# wechatopenai/weknora-* 等镜像一并清掉, 导致基于镜像建出来的新实例
|
||||
# 在 firstboot 时还要重新从 Docker Hub 拉数 GB 镜像, 完全违背预装初衷。
|
||||
docker container prune -f || true
|
||||
docker builder prune -af || true
|
||||
# 只清未挂载的 dangling 卷 (此时 compose down -v 已清掉业务卷)
|
||||
docker volume prune -f || true
|
||||
|
||||
echo "[cleanup] 4/8 清空系统日志"
|
||||
journalctl --rotate || true
|
||||
journalctl --vacuum-time=1s || true
|
||||
find /var/log -type f \( -name '*.log' -o -name '*.gz' -o -name '*.[0-9]' \) -print0 \
|
||||
| xargs -0 -r truncate -s 0 || true
|
||||
find /var/log -type f \( -name '*.gz' -o -name '*.[0-9]' \) -print0 \
|
||||
| xargs -0 -r rm -f || true
|
||||
|
||||
echo "[cleanup] 5/8 清理 SSH 历史与授权 key(执行后将无法 SSH 进来)"
|
||||
rm -f /root/.ssh/authorized_keys /root/.ssh/known_hosts /root/.bash_history
|
||||
for d in /home/*; do
|
||||
[[ -d "$d" ]] || continue
|
||||
rm -f "$d/.ssh/authorized_keys" "$d/.ssh/known_hosts" "$d/.bash_history"
|
||||
done
|
||||
find / -xdev -type f \( -name 'id_rsa*' -o -name '*.pem' -o -name '*.key' \) \
|
||||
-not -path '/etc/ssl/*' -not -path '/usr/*' -not -path '/var/lib/docker/*' 2>/dev/null \
|
||||
| tee /tmp/cleanup-secrets-found.txt || true
|
||||
echo "[cleanup] ↑ 上面是疑似遗留的密钥文件,必要时人工再核对"
|
||||
|
||||
echo "[cleanup] 6/8 重置 cloud-init / machine-id(让新实例拿到新 ID)"
|
||||
cloud-init clean --logs --seed 2>/dev/null || true
|
||||
truncate -s 0 /etc/machine-id || true
|
||||
rm -f /var/lib/dbus/machine-id || true
|
||||
|
||||
echo "[cleanup] 7/8 清理 apt / tmp"
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get clean
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
fi
|
||||
rm -rf /tmp/* /var/tmp/* /root/.cache /home/*/.cache 2>/dev/null || true
|
||||
|
||||
echo "[cleanup] 8/8 同步磁盘并关机"
|
||||
history -c || true
|
||||
sync
|
||||
echo
|
||||
echo " 即将关机。关机完成后请到云控制台执行「制作镜像 / 创建快照 / 创建 AMI」。"
|
||||
echo
|
||||
sleep 3
|
||||
poweroff
|
||||
Executable
+139
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env bash
|
||||
# firstboot.sh - 由 weknora-firstboot.service 在新实例首次开机时自动执行。
|
||||
# 任务: 生成随机密钥写入 .env -> 启动容器 -> 输出凭证 -> 标记完成 + 自禁用。
|
||||
#
|
||||
# 幂等性策略:
|
||||
# 1) 一上来先写 ${MARKER} 标记 "已生成过密钥",
|
||||
# 之后即便 docker compose 失败导致脚本中断, 重启后由 unit 的
|
||||
# ConditionPathExists=!${MARKER} 拦住, 不会再生成新密钥覆盖 .env。
|
||||
# (旧密钥已经写进 postgres 数据卷, 二次生成会导致数据库永久无法登录)
|
||||
# 2) 失败时 unit 标记 failed, 用户可手动 docker compose up -d 恢复;
|
||||
# 凭证仍可在 ${ENV_FILE} 中查到。
|
||||
set -euo pipefail
|
||||
|
||||
WEKNORA_DIR="${WEKNORA_DIR:-/opt/WeKnora}"
|
||||
ENV_FILE="${WEKNORA_DIR}/.env"
|
||||
ENV_TEMPLATE="${WEKNORA_DIR}/.env.example"
|
||||
CRED_FILE="/root/weknora-credentials.txt"
|
||||
LOG_FILE="/var/log/weknora-firstboot.log"
|
||||
MARKER="${WEKNORA_DIR}/.firstboot.done"
|
||||
|
||||
# 提前打开 LOG_FILE, 同时把 stderr 也复制一份到 systemd journal 方便调试
|
||||
# (单纯 exec >> LOG_FILE 时, 脚本一开始的失败会因为 stderr 已被吞掉而看不到)
|
||||
mkdir -p "$(dirname "${LOG_FILE}")"
|
||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||||
echo "==== firstboot started at $(date -Iseconds) ===="
|
||||
|
||||
if [[ -f "${MARKER}" ]]; then
|
||||
echo "marker ${MARKER} exists, skip (already initialized)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# cleanup.sh 不再保留 .env, 这里从 .env.example 拷贝模板再做替换。
|
||||
# 这样保证 firstboot 之前不会有任何含明文默认密码的 .env 让 weknora.service
|
||||
# 抢先把 postgres 数据卷用错的密码初始化掉。
|
||||
if [[ ! -f "${ENV_FILE}" ]]; then
|
||||
if [[ -f "${ENV_TEMPLATE}" ]]; then
|
||||
echo "creating ${ENV_FILE} from ${ENV_TEMPLATE}"
|
||||
cp "${ENV_TEMPLATE}" "${ENV_FILE}"
|
||||
else
|
||||
echo "ERROR: neither ${ENV_FILE} nor ${ENV_TEMPLATE} found"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
DOCKER_BIN="$(command -v docker || true)"
|
||||
if [[ -z "${DOCKER_BIN}" ]]; then
|
||||
echo "ERROR: docker binary not found in PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 生成 32 字节强随机字符串(用于 AES-256 key, 必须刚好 32 字节)
|
||||
# 用 `() ... ()` 启子 shell 并关闭 pipefail: head 只读 N 字节后关闭 stdin,
|
||||
# tr 会收到 SIGPIPE (退出码 141), 在 `set -o pipefail` 下整条管道被判失败,
|
||||
# 触发顶层 `set -e` 把 firstboot.sh 8ms exit 1 干掉。
|
||||
gen32() ( set +o pipefail; LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c 32 )
|
||||
# 通用密码: 24 字符, 不含 / + = (避免出现在 URL / sed 替换时出问题)
|
||||
genpw() ( set +o pipefail; LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c 24 )
|
||||
|
||||
DB_PWD=$(genpw)
|
||||
REDIS_PWD=$(genpw)
|
||||
JWT=$(genpw)$(genpw)
|
||||
SYS_AES=$(gen32)
|
||||
|
||||
# 用 | 作 sed 分隔符避免冲突; 仅替换以 KEY= 开头的行
|
||||
replace() {
|
||||
local key="$1" val="$2"
|
||||
if grep -qE "^${key}=" "${ENV_FILE}"; then
|
||||
sed -i "s|^${key}=.*|${key}=${val}|" "${ENV_FILE}"
|
||||
else
|
||||
echo "${key}=${val}" >>"${ENV_FILE}"
|
||||
fi
|
||||
}
|
||||
|
||||
replace DB_PASSWORD "${DB_PWD}"
|
||||
replace REDIS_PASSWORD "${REDIS_PWD}"
|
||||
replace JWT_SECRET "${JWT}"
|
||||
replace SYSTEM_AES_KEY "${SYS_AES}"
|
||||
replace GIN_MODE "release"
|
||||
|
||||
# 把 prepare.sh 阶段记录在 .cloud-image-meta 里的 WEKNORA_REF 还原为 .env 的
|
||||
# WEKNORA_VERSION, 否则 docker compose 会落回 :latest 默认值, 导致镜像版本
|
||||
# 与 prepare 时拉取的版本不一致。
|
||||
META_FILE="${WEKNORA_DIR}/.cloud-image-meta"
|
||||
if [[ -f "${META_FILE}" ]]; then
|
||||
META_REF=$(grep -E '^WEKNORA_REF=' "${META_FILE}" | tail -1 | cut -d= -f2- || true)
|
||||
if [[ -n "${META_REF}" ]]; then
|
||||
replace WEKNORA_VERSION "${META_REF}"
|
||||
echo "restored WEKNORA_VERSION=${META_REF} from ${META_FILE}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 关键: .env 改完立刻写 marker。
|
||||
# 这之后即便 docker compose up 失败, 重启也不会再次重写 .env,
|
||||
# 防止与 postgres 已经持久化的密码不一致。
|
||||
umask 077
|
||||
touch "${MARKER}"
|
||||
chmod 0600 "${MARKER}"
|
||||
|
||||
echo "env updated, marker written, starting docker compose..."
|
||||
cd "${WEKNORA_DIR}"
|
||||
"${DOCKER_BIN}" compose up -d
|
||||
|
||||
# 尝试拿公网 IP, 失败就用内网
|
||||
PUB_IP=$(curl -fsS --max-time 5 https://ifconfig.me 2>/dev/null \
|
||||
|| curl -fsS --max-time 5 https://api.ipify.org 2>/dev/null \
|
||||
|| hostname -I | awk '{print $1}')
|
||||
|
||||
cat >"${CRED_FILE}" <<INFO
|
||||
========================================
|
||||
WeKnora 实例初始化完成
|
||||
生成时间: $(date -Iseconds)
|
||||
========================================
|
||||
|
||||
访问地址 : http://${PUB_IP}
|
||||
|
||||
注册后如需关闭后续注册, 编辑 ${ENV_FILE}:
|
||||
DISABLE_REGISTRATION=true
|
||||
然后执行: cd ${WEKNORA_DIR} && docker compose up -d
|
||||
|
||||
下列随机凭证已写入 ${ENV_FILE}, 请妥善保存(仅 root 可读):
|
||||
DB_PASSWORD = ${DB_PWD}
|
||||
REDIS_PASSWORD = ${REDIS_PWD}
|
||||
JWT_SECRET = ${JWT}
|
||||
SYSTEM_AES_KEY = ${SYS_AES}
|
||||
|
||||
注意:
|
||||
- 本文件只反映首启时刻的密钥, 后续以 ${ENV_FILE} 为准。
|
||||
- 切勿直接对外暴露 5432 / 6379 / 9000 等基础设施端口。
|
||||
- 仅用 80 / 443 对外服务, 必要时配置反向代理 + HTTPS。
|
||||
|
||||
INFO
|
||||
|
||||
echo "credentials written to ${CRED_FILE}"
|
||||
|
||||
# 仅停掉 unit, 不删 unit 文件 (否则当前正在运行的 oneshot 可能被 systemd 标记 failed)。
|
||||
# 下次重启时, weknora-firstboot.service 通过 ConditionPathExists=!${MARKER} 自动跳过。
|
||||
systemctl disable weknora-firstboot.service || true
|
||||
|
||||
echo "==== firstboot finished at $(date -Iseconds) ===="
|
||||
Executable
+237
@@ -0,0 +1,237 @@
|
||||
#!/usr/bin/env bash
|
||||
# prepare.sh - 在干净的 Linux 实例上部署 WeKnora 运行时, 用于制作云镜像模板。
|
||||
# 不需要 clone 整个 WeKnora 仓库, 只下载 4 个运行时文件 (~100KB)。
|
||||
# 兼容: Ubuntu / Debian / CentOS / Rocky / TencentOS 等带 systemd + Docker 的发行版。
|
||||
# 使用方式: sudo bash prepare.sh
|
||||
# 可调环境变量:
|
||||
# WEKNORA_REF 要拉取的 git ref (tag / branch / commit), 默认 main
|
||||
# WEKNORA_DIR 部署目录, 默认 /opt/WeKnora
|
||||
# WEKNORA_REPO 仓库地址, 默认 https://github.com/Tencent/WeKnora
|
||||
# WEKNORA_GH_PROXY GitHub 加速前缀, 默认空。中国大陆机器可设
|
||||
# https://gh-proxy.com/ 或 https://ghfast.top/
|
||||
# (实际下载地址变成 ${WEKNORA_GH_PROXY}${WEKNORA_REPO}/archive/...)
|
||||
# DOCKER_INSTALL_MIRROR Docker 安装包镜像源, 默认空 (走 get.docker.com)。
|
||||
# 中国大陆机器境外 CDN 不通时设为, 例如:
|
||||
# https://mirrors.tencent.com/docker-ce/linux/ubuntu
|
||||
# https://mirrors.aliyun.com/docker-ce/linux/ubuntu
|
||||
# 会改用 apt + docker-ce 官方仓库镜像安装,
|
||||
# 含 docker-ce / containerd.io / docker-compose-plugin,
|
||||
# 完全不访问 get.docker.com。仅支持 apt 系发行版。
|
||||
# DOCKER_REGISTRY_MIRROR Docker Hub 加速器, 默认空。腾讯云内网可设
|
||||
# https://mirror.ccs.tencentyun.com
|
||||
# (会写入 /etc/docker/daemon.json 并重启 docker)
|
||||
# PRUNE_OLD_IMAGES 升级场景下是否清理 dangling / 旧版本 tag 镜像,
|
||||
# 默认 false。设为 true 时在拉新镜像之后执行
|
||||
# `docker image prune -af`, 把没有容器引用的镜像
|
||||
# (含旧 WEKNORA_VERSION 的 wechatopenai/weknora-*)
|
||||
# 一次性删掉, 减少要打进云镜像的体积。
|
||||
set -euo pipefail
|
||||
|
||||
WEKNORA_REF="${WEKNORA_REF:-main}"
|
||||
WEKNORA_DIR="${WEKNORA_DIR:-/opt/WeKnora}"
|
||||
WEKNORA_REPO="${WEKNORA_REPO:-https://github.com/Tencent/WeKnora}"
|
||||
WEKNORA_GH_PROXY="${WEKNORA_GH_PROXY:-}"
|
||||
DOCKER_INSTALL_MIRROR="${DOCKER_INSTALL_MIRROR:-}"
|
||||
DOCKER_REGISTRY_MIRROR="${DOCKER_REGISTRY_MIRROR:-}"
|
||||
PRUNE_OLD_IMAGES="${PRUNE_OLD_IMAGES:-false}"
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
if [[ "${EUID}" -ne 0 ]]; then
|
||||
echo "[prepare] 请使用 sudo 或 root 运行" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 通过镜像源 apt 安装 docker-ce 全家桶 (含 compose-plugin)。
|
||||
# 用于中国大陆云主机直连 get.docker.com 被 RST 的场景。
|
||||
install_docker_via_apt_mirror() {
|
||||
local mirror="$1"
|
||||
if ! command -v apt-get >/dev/null 2>&1; then
|
||||
echo "[prepare] DOCKER_INSTALL_MIRROR 目前仅支持 apt 系发行版 (Ubuntu/Debian)" >&2
|
||||
return 1
|
||||
fi
|
||||
apt-get update -y
|
||||
apt-get install -y ca-certificates curl gnupg lsb-release
|
||||
install -m 0755 -d /etc/apt/keyrings
|
||||
curl -fsSL "${mirror%/}/gpg" | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg
|
||||
chmod a+r /etc/apt/keyrings/docker.gpg
|
||||
local arch codename
|
||||
arch="$(dpkg --print-architecture)"
|
||||
codename="$(lsb_release -cs)"
|
||||
echo "deb [arch=${arch} signed-by=/etc/apt/keyrings/docker.gpg] ${mirror%/} ${codename} stable" \
|
||||
> /etc/apt/sources.list.d/docker.list
|
||||
apt-get update -y
|
||||
apt-get install -y docker-ce docker-ce-cli containerd.io \
|
||||
docker-buildx-plugin docker-compose-plugin curl tar
|
||||
}
|
||||
|
||||
echo "[prepare] 1/6 安装 Docker 与依赖"
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
if [[ -n "${DOCKER_INSTALL_MIRROR}" ]]; then
|
||||
echo "[prepare] 通过 ${DOCKER_INSTALL_MIRROR} 走 apt 安装 docker-ce (跳过 get.docker.com)"
|
||||
install_docker_via_apt_mirror "${DOCKER_INSTALL_MIRROR}"
|
||||
else
|
||||
curl -fsSL https://get.docker.com | bash
|
||||
fi
|
||||
fi
|
||||
systemctl enable --now docker
|
||||
|
||||
if ! docker compose version >/dev/null 2>&1; then
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update -y
|
||||
apt-get install -y docker-compose-plugin curl tar
|
||||
elif command -v yum >/dev/null 2>&1; then
|
||||
yum install -y docker-compose-plugin curl tar
|
||||
fi
|
||||
fi
|
||||
|
||||
# 可选: 配置 Docker Hub 加速器, 解决直连 registry-1.docker.io 超时的场景
|
||||
# (典型: 中国大陆云主机 / 内网受限环境)。仅在用户显式传入时才动 daemon.json。
|
||||
if [[ -n "${DOCKER_REGISTRY_MIRROR}" ]]; then
|
||||
echo "[prepare] 1.5/6 配置 Docker Hub 加速器: ${DOCKER_REGISTRY_MIRROR}"
|
||||
mkdir -p /etc/docker
|
||||
# 已存在的 daemon.json 走 python 合并, 避免覆盖用户其它配置
|
||||
if [[ -s /etc/docker/daemon.json ]] && command -v python3 >/dev/null 2>&1; then
|
||||
python3 - "$DOCKER_REGISTRY_MIRROR" <<'PY'
|
||||
import json, sys, pathlib
|
||||
p = pathlib.Path("/etc/docker/daemon.json")
|
||||
mirror = sys.argv[1]
|
||||
try:
|
||||
cfg = json.loads(p.read_text())
|
||||
except Exception:
|
||||
cfg = {}
|
||||
mirrors = cfg.get("registry-mirrors") or []
|
||||
if mirror not in mirrors:
|
||||
mirrors.insert(0, mirror)
|
||||
cfg["registry-mirrors"] = mirrors
|
||||
p.write_text(json.dumps(cfg, indent=2) + "\n")
|
||||
PY
|
||||
else
|
||||
cat >/etc/docker/daemon.json <<EOF
|
||||
{
|
||||
"registry-mirrors": ["${DOCKER_REGISTRY_MIRROR}"]
|
||||
}
|
||||
EOF
|
||||
fi
|
||||
systemctl restart docker
|
||||
# 给 docker daemon 一点重启时间, 避免下面 docker compose pull 立刻拿到 EOF
|
||||
for _ in 1 2 3 4 5; do
|
||||
docker info >/dev/null 2>&1 && break
|
||||
sleep 1
|
||||
done
|
||||
fi
|
||||
|
||||
echo "[prepare] 2/6 拉取 WeKnora 运行时文件 (ref=${WEKNORA_REF})"
|
||||
# 只下载实际需要的 4 个文件, 不 clone 整个仓库 (~MB 级 -> ~KB 级)
|
||||
mkdir -p "${WEKNORA_DIR}/config" "${WEKNORA_DIR}/skills"
|
||||
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "${tmp}"' EXIT
|
||||
|
||||
tarball_url="${WEKNORA_GH_PROXY}${WEKNORA_REPO}/archive/${WEKNORA_REF}.tar.gz"
|
||||
echo "[prepare] tarball: ${tarball_url}"
|
||||
curl -fsSL "${tarball_url}" -o "${tmp}/repo.tar.gz"
|
||||
# 仅解压需要的路径, 显著加速且省空间
|
||||
tar -xzf "${tmp}/repo.tar.gz" -C "${tmp}" \
|
||||
--wildcards \
|
||||
'*/docker-compose.yml' \
|
||||
'*/.env.example' \
|
||||
'*/config/config.yaml' \
|
||||
'*/skills/preloaded'
|
||||
src=$(find "${tmp}" -maxdepth 1 -mindepth 1 -type d -name 'WeKnora-*' | head -1)
|
||||
if [[ -z "${src}" ]]; then
|
||||
echo "[prepare] 解压失败, 未找到 WeKnora-* 目录" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cp "${src}/docker-compose.yml" "${WEKNORA_DIR}/"
|
||||
cp "${src}/.env.example" "${WEKNORA_DIR}/"
|
||||
cp "${src}/config/config.yaml" "${WEKNORA_DIR}/config/"
|
||||
rm -rf "${WEKNORA_DIR}/skills/preloaded"
|
||||
cp -r "${src}/skills/preloaded" "${WEKNORA_DIR}/skills/"
|
||||
|
||||
# 记录元信息, 供 firstboot / 升级时参考
|
||||
cat >"${WEKNORA_DIR}/.cloud-image-meta" <<EOF
|
||||
WEKNORA_REF=${WEKNORA_REF}
|
||||
WEKNORA_REPO=${WEKNORA_REPO}
|
||||
PREPARED_AT=$(date -Iseconds)
|
||||
EOF
|
||||
|
||||
echo "[prepare] 3/6 准备 .env (默认值, firstboot 会替换为随机密钥)"
|
||||
cd "${WEKNORA_DIR}"
|
||||
[[ -f .env ]] || cp .env.example .env
|
||||
sed -i 's/^GIN_MODE=.*/GIN_MODE=release/' .env || true
|
||||
|
||||
# 把 WEKNORA_VERSION 与 WEKNORA_REF 对齐, 让 docker compose 拉取与 ref 一致的
|
||||
# 镜像 tag。无条件覆盖, 避免 .env 残留上一次 prepare 留下的旧版本号。
|
||||
# Docker Hub 上 wechatopenai/weknora-* 的 tag 命名约定:
|
||||
# - 浮动 tag:main(持续指向最新构建)
|
||||
# - 固定 release tag:v 前缀 + semver(如 v0.7.2、v0.5.2)
|
||||
# 因此这里不剥 v、也不映射到 latest。
|
||||
WEKNORA_VERSION_VAL="${WEKNORA_REF}"
|
||||
if grep -qE '^WEKNORA_VERSION=' .env; then
|
||||
sed -i "s|^WEKNORA_VERSION=.*|WEKNORA_VERSION=${WEKNORA_VERSION_VAL}|" .env
|
||||
else
|
||||
echo "WEKNORA_VERSION=${WEKNORA_VERSION_VAL}" >>.env
|
||||
fi
|
||||
echo "[prepare] -> WEKNORA_VERSION=${WEKNORA_VERSION_VAL}"
|
||||
|
||||
echo "[prepare] 4/6 拉取并启动默认 5 个常驻容器 (frontend/app/docreader/postgres/redis)"
|
||||
docker compose pull
|
||||
docker compose up -d
|
||||
|
||||
# 提前 pull sandbox 镜像 (Agent Skills 运行时由 app 按需 docker run, 非常驻)
|
||||
# 不预拉的话, 用户首次跑 Skill 会卡在下载
|
||||
echo "[prepare] 4.5/6 预拉 sandbox 镜像 (Agent Skills 用, 非常驻)"
|
||||
docker compose --profile full pull sandbox || true
|
||||
|
||||
# 其他向量库 / 可观测组件 (qdrant, milvus, weaviate, doris, neo4j, langfuse-*, minio, dex)
|
||||
# 不预拉, 体积可省 5-15GB. 用户如需启用:
|
||||
# cd /opt/WeKnora && docker compose --profile <name> up -d
|
||||
|
||||
# 升级场景: 清理旧版本 tag 的 wechatopenai/weknora-* 镜像。
|
||||
# 默认关闭, 保留回滚路径; 制作镜像前显式打开以减小体积。
|
||||
#
|
||||
# 注意: 不用 `docker image prune -af`!
|
||||
# sandbox 镜像在 compose 里只 pull 不 up (Agent Skills 由 app 按需 docker run),
|
||||
# 没有任何容器引用它, 一旦 `prune -a` 会把当前版本的 sandbox 一起删掉,
|
||||
# 反而违背 prepare.sh 4.5 步预拉 sandbox 的目的。
|
||||
# 这里精确按 tag 比对, 只删 wechatopenai/weknora-* 仓库下、tag 不等于当前
|
||||
# WEKNORA_VERSION 的镜像, 基础设施镜像 (paradedb / redis) 不动。
|
||||
if [[ "${PRUNE_OLD_IMAGES,,}" == "true" || "${PRUNE_OLD_IMAGES}" == "1" ]]; then
|
||||
echo "[prepare] 4.6/6 清理 wechatopenai/weknora-* 仓库下旧版本镜像 (PRUNE_OLD_IMAGES=true, keep=${WEKNORA_VERSION_VAL})"
|
||||
docker image ls --format '{{.Repository}}:{{.Tag}}' \
|
||||
| grep -E '^wechatopenai/weknora-' \
|
||||
| grep -vE ":${WEKNORA_VERSION_VAL}\$" \
|
||||
| xargs -r docker rmi -f 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "[prepare] 5/6 安装 systemd 单元"
|
||||
# 探测 docker 二进制路径, 不同发行版可能在 /usr/bin 或 /usr/local/bin
|
||||
DOCKER_BIN="$(command -v docker)"
|
||||
if [[ -z "${DOCKER_BIN}" ]]; then
|
||||
echo "[prepare] 未找到 docker 二进制" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[prepare] docker binary: ${DOCKER_BIN}"
|
||||
|
||||
install -m 0644 "${SCRIPT_DIR}/systemd/weknora.service" /etc/systemd/system/weknora.service
|
||||
install -m 0644 "${SCRIPT_DIR}/systemd/weknora-firstboot.service" /etc/systemd/system/weknora-firstboot.service
|
||||
install -m 0755 "${SCRIPT_DIR}/firstboot.sh" /usr/local/sbin/weknora-firstboot.sh
|
||||
|
||||
# 把 systemd 单元里的 docker 路径模板替换为实际路径
|
||||
sed -i "s|@DOCKER_BIN@|${DOCKER_BIN}|g" /etc/systemd/system/weknora.service
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable weknora.service
|
||||
systemctl enable weknora-firstboot.service
|
||||
|
||||
echo "[prepare] 6/6 完成"
|
||||
echo
|
||||
echo " WeKnora 运行时已部署到 ${WEKNORA_DIR}"
|
||||
echo " docker-compose.yml / config/config.yaml / skills/preloaded / .env"
|
||||
echo " 版本: ${WEKNORA_REF} (见 ${WEKNORA_DIR}/.cloud-image-meta)"
|
||||
echo
|
||||
echo " 打开浏览器访问 http://<本机公网IP> 验证功能"
|
||||
echo
|
||||
echo " 验证通过后执行清理并制作镜像:"
|
||||
echo " sudo bash ${SCRIPT_DIR}/cleanup.sh"
|
||||
@@ -0,0 +1,21 @@
|
||||
[Unit]
|
||||
Description=WeKnora (docker compose)
|
||||
Requires=docker.service weknora-firstboot.service
|
||||
After=docker.service network-online.target weknora-firstboot.service
|
||||
Wants=network-online.target
|
||||
# 关键: Requires=weknora-firstboot.service 让首启 init 失败时本服务也不启动,
|
||||
# 避免 firstboot 没成功生成随机密钥就用 .env.example 默认明文密码把 postgres
|
||||
# 数据卷初始化掉。weknora-firstboot.service 在已初始化的实例上由
|
||||
# ConditionPathExists=!/opt/WeKnora/.firstboot.done 跳过, 跳过 = 成功,
|
||||
# 不会阻塞本服务后续重启。
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
WorkingDirectory=/opt/WeKnora
|
||||
ExecStart=@DOCKER_BIN@ compose up -d
|
||||
ExecStop=@DOCKER_BIN@ compose down
|
||||
TimeoutStartSec=600
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,52 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const assert = require('node:assert/strict')
|
||||
const root = path.resolve(__dirname, '../..')
|
||||
const model = '404e005b-83ae-5e2e-a3b3-999f9d5cd47e'
|
||||
const tenant = '10059'
|
||||
let token
|
||||
async function api(route, method = 'GET', body) {
|
||||
const response = await fetch(`http://127.0.0.1:18080/api/v1${route}`, { method,
|
||||
headers: { 'Content-Type': 'application/json', ...(token ? { Authorization: `Bearer ${token}`, 'X-Tenant-ID': tenant } : {}) },
|
||||
body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(30000) })
|
||||
const result = await response.json()
|
||||
assert.ok(response.ok && result.success !== false, `${method} ${route}: HTTP ${response.status}`)
|
||||
return result
|
||||
}
|
||||
async function main() {
|
||||
const credentials = JSON.parse(fs.readFileSync(path.join(root, 'acceptance/super-admin-login.json'), 'utf8'))
|
||||
const login = await api('/auth/login', 'POST', { username: credentials.username, password: credentials.password })
|
||||
token = (login.data || login).token
|
||||
const before = (await api('/knowledge-bases')).data
|
||||
const targets = before.filter(kb => !kb.summary_model_id && kb.embedding_model_id)
|
||||
if (!process.argv.includes('--apply')) { console.log(JSON.stringify({ missing_models: targets.map(kb => kb.name) })); return }
|
||||
fs.writeFileSync(path.join(root, 'acceptance', `kb-model-config-before-${Date.now()}.json`), JSON.stringify(targets, null, 2), { mode: 0o600, flag: 'wx' })
|
||||
for (const kb of targets) {
|
||||
const c = kb.chunking_config || {}, e = kb.extract_config || {}, q = kb.question_generation_config || {}
|
||||
const request = {
|
||||
llmModelId: model, embeddingModelId: kb.embedding_model_id,
|
||||
vlm_config: kb.vlm_config, asr_config: kb.asr_config,
|
||||
multimodal: { enabled: !!kb.vlm_config?.enabled },
|
||||
storageBackendId: kb.storage_backend_id, storageProvider: kb.storage_config?.provider || 'local',
|
||||
documentSplitting: {
|
||||
chunkSize: c.chunk_size || 512, chunkOverlap: c.chunk_size ? c.chunk_overlap : 80,
|
||||
separators: c.separators?.length ? c.separators : ['\n\n', '\n', '。', '!', '?', ';', ';'],
|
||||
parserEngineRules: c.parser_engine_rules, enableParentChild: !!c.enable_parent_child,
|
||||
parentChunkSize: c.parent_chunk_size || 4096, childChunkSize: c.child_chunk_size || 384,
|
||||
strategy: c.strategy, tokenLimit: c.token_limit, languages: c.languages, tableMetadataInstructions: c.table_metadata_instructions,
|
||||
},
|
||||
nodeExtract: { ...e, customInstructions: e.custom_instructions },
|
||||
questionGeneration: { enabled: !!q.enabled, questionCount: q.question_count, customInstructions: q.custom_instructions },
|
||||
}
|
||||
await api(`/initialization/config/${kb.id}`, 'PUT', request)
|
||||
const after = (await api(`/knowledge-bases/${kb.id}`)).data
|
||||
assert.equal(after.summary_model_id, model)
|
||||
assert.equal(after.embedding_model_id, kb.embedding_model_id)
|
||||
assert.deepEqual(after.indexing_strategy, kb.indexing_strategy)
|
||||
assert.deepEqual(after.wiki_config, kb.wiki_config)
|
||||
assert.equal(after.storage_backend_id, kb.storage_backend_id)
|
||||
}
|
||||
const after = (await api('/knowledge-bases')).data
|
||||
console.log(JSON.stringify({ updated: targets.map(kb => kb.name), initialized: after.filter(kb => kb.summary_model_id && kb.embedding_model_id).length, total: after.length, existing_indexes_preserved: true }))
|
||||
}
|
||||
main().catch(error => { console.error(error.message); process.exitCode = 1 })
|
||||
Executable
+601
@@ -0,0 +1,601 @@
|
||||
#!/bin/bash
|
||||
# 开发环境启动脚本 - 只启动基础设施,app 和 frontend 需要手动在本地运行
|
||||
|
||||
# 设置颜色
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # 无颜色
|
||||
|
||||
# 获取项目根目录
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
|
||||
|
||||
# 日志函数
|
||||
log_info() {
|
||||
printf "%b\n" "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
printf "%b\n" "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
printf "%b\n" "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
printf "%b\n" "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
# 选择可用的 Docker Compose 命令
|
||||
DOCKER_COMPOSE_BIN=""
|
||||
DOCKER_COMPOSE_SUBCMD=""
|
||||
|
||||
detect_compose_cmd() {
|
||||
if docker compose version &> /dev/null; then
|
||||
DOCKER_COMPOSE_BIN="docker"
|
||||
DOCKER_COMPOSE_SUBCMD="compose"
|
||||
return 0
|
||||
fi
|
||||
if command -v docker-compose &> /dev/null; then
|
||||
if docker-compose version &> /dev/null; then
|
||||
DOCKER_COMPOSE_BIN="docker-compose"
|
||||
DOCKER_COMPOSE_SUBCMD=""
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# 显示帮助信息
|
||||
show_help() {
|
||||
printf "%b\n" "${GREEN}WeKnora 开发环境脚本${NC}"
|
||||
echo "用法: $0 [命令] [选项]"
|
||||
echo ""
|
||||
echo "命令:"
|
||||
echo " start 启动基础设施服务(postgres, redis, docreader, langfuse)"
|
||||
echo " stop 停止所有服务"
|
||||
echo " restart 重启所有服务"
|
||||
echo " logs 查看服务日志"
|
||||
echo " status 查看服务状态"
|
||||
echo " app 启动后端应用(本地运行)"
|
||||
echo " frontend 启动前端开发服务器(本地运行)"
|
||||
echo " help 显示此帮助信息"
|
||||
echo ""
|
||||
echo "可选 Profile(用于 start 命令):"
|
||||
echo " --minio 启动 MinIO 对象存储"
|
||||
echo " --qdrant 启动 Qdrant 向量数据库"
|
||||
echo " --neo4j 启动 Neo4j 图数据库"
|
||||
echo " --dex 启动 Dex(OIDC 身份认证)"
|
||||
echo " --langfuse 启动 Langfuse(默认已开启)"
|
||||
echo " --no-langfuse 不启动 Langfuse"
|
||||
echo " --odl-hybrid 启动 OpenDataLoader hybrid(Docling,镜像较大,按需启用)"
|
||||
echo " --full 启动所有可选服务(不含 odl-hybrid,需另加 --odl-hybrid)"
|
||||
echo ""
|
||||
echo "示例:"
|
||||
echo " $0 start # 启动基础服务"
|
||||
echo " $0 start --qdrant # 启动基础服务 + Qdrant"
|
||||
echo " $0 start --dex # 启动基础服务 + Dex"
|
||||
echo " $0 start --odl-hybrid # 启动基础服务 + OpenDataLoader hybrid"
|
||||
echo " $0 start --full # 启动所有服务"
|
||||
echo " make dev-start DEV_ARGS=--odl-hybrid # 同上(Makefile 传参)"
|
||||
echo " $0 app # 在另一个终端启动后端"
|
||||
echo " $0 frontend # 在另一个终端启动前端"
|
||||
}
|
||||
|
||||
# 加载 .env 与可选的 .env.local(后者覆盖前者)
|
||||
# 读取时去掉行尾 \r,兼容 Windows 风格(CRLF)换行符,
|
||||
# 否则 bash source 会把残留的 \r 当成命令导致 "...: $'\r': command not found"。
|
||||
# 注意:不能用 source <(sed ...)——macOS 自带 Bash 3.2 对 process substitution
|
||||
# 的 source 不会把变量导入当前 shell;必须落到可 seek 的临时文件再 source。
|
||||
_source_env_file() {
|
||||
local src="$1"
|
||||
local tmp
|
||||
tmp="$(mktemp)" || return 1
|
||||
sed -e 's/\r$//' "$src" > "$tmp"
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source "$tmp"
|
||||
set +a
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
load_env_files() {
|
||||
if [ -f ".env" ]; then
|
||||
_source_env_file .env || return 1
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ -f ".env.local" ]; then
|
||||
log_info "加载 .env.local 覆盖配置..."
|
||||
_source_env_file .env.local || return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# 检查 Docker
|
||||
check_docker() {
|
||||
if ! command -v docker &> /dev/null; then
|
||||
log_error "未安装Docker,请先安装Docker"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! detect_compose_cmd; then
|
||||
log_error "未检测到 Docker Compose"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! docker info &> /dev/null; then
|
||||
log_error "Docker服务未运行"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# 检查 .env 是否启用了 hybrid 模式(用于 --odl-hybrid 启动后重建 docreader)
|
||||
_should_enable_odl_hybrid_from_env() {
|
||||
local hybrid="${DOCREADER_ODL_HYBRID:-off}"
|
||||
hybrid=$(printf '%s' "$hybrid" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')
|
||||
case "$hybrid" in
|
||||
off|"") return 1 ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
_enable_odl_hybrid_profile() {
|
||||
PROFILES="$PROFILES --profile odl-hybrid"
|
||||
ENABLED_SERVICES="$ENABLED_SERVICES odl-hybrid"
|
||||
}
|
||||
|
||||
# 等待 odl-hybrid HTTP 健康检查通过(compose 启动后服务可能仍在拉依赖)
|
||||
_wait_odl_hybrid_ready() {
|
||||
local port="${ODL_HYBRID_PORT:-5002}"
|
||||
local max_wait="${ODL_HYBRID_STARTUP_WAIT_SEC:-180}"
|
||||
local waited=0
|
||||
local interval=5
|
||||
|
||||
if ! command -v curl &> /dev/null; then
|
||||
log_warning "未安装 curl,跳过 odl-hybrid 就绪等待;请手动检查 http://localhost:${port}/health"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "等待 odl-hybrid 就绪(最多 ${max_wait}s,首次需构建镜像: docker compose ... build odl-hybrid)..."
|
||||
while [ "$waited" -lt "$max_wait" ]; do
|
||||
if curl -sf "http://127.0.0.1:${port}/health" >/dev/null 2>&1; then
|
||||
log_success "odl-hybrid 已就绪 (http://localhost:${port}/health)"
|
||||
return 0
|
||||
fi
|
||||
sleep "$interval"
|
||||
waited=$((waited + interval))
|
||||
done
|
||||
log_warning "odl-hybrid 在 ${max_wait}s 内未就绪,请查看: docker logs WeKnora-odl-hybrid"
|
||||
return 1
|
||||
}
|
||||
|
||||
# 启动基础设施服务
|
||||
start_services() {
|
||||
log_info "启动开发环境基础设施服务..."
|
||||
|
||||
check_docker
|
||||
if [ $? -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
# 检查 .env 文件
|
||||
if [ ! -f ".env" ]; then
|
||||
log_error ".env 文件不存在,请先创建"
|
||||
return 1
|
||||
fi
|
||||
|
||||
load_env_files
|
||||
if [ $? -ne 0 ]; then
|
||||
log_error ".env 文件不存在,请先创建"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ -n "${DEV_REMOTE_HOST:-}" ]; then
|
||||
log_warning "已配置 DEV_REMOTE_HOST=${DEV_REMOTE_HOST},跳过本地 Docker 基础设施启动"
|
||||
log_info "远程服务: PostgreSQL/Redis/DocReader/Langfuse → ${DEV_REMOTE_HOST}"
|
||||
log_info "接下来: make dev-app(本地后端)或 make dev-frontend(前端)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 解析 profile 参数
|
||||
shift # 移除 "start" 命令本身
|
||||
# 默认启动基础设施(postgres / redis / docreader)+ langfuse,
|
||||
# 其余可选服务通过 --minio / --qdrant / --neo4j / --dex / --full 按需开启。
|
||||
PROFILES="--profile langfuse"
|
||||
ENABLED_SERVICES="langfuse"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--minio)
|
||||
PROFILES="$PROFILES --profile minio"
|
||||
ENABLED_SERVICES="$ENABLED_SERVICES minio"
|
||||
;;
|
||||
--qdrant)
|
||||
PROFILES="$PROFILES --profile qdrant"
|
||||
ENABLED_SERVICES="$ENABLED_SERVICES qdrant"
|
||||
;;
|
||||
--neo4j)
|
||||
PROFILES="$PROFILES --profile neo4j"
|
||||
ENABLED_SERVICES="$ENABLED_SERVICES neo4j"
|
||||
;;
|
||||
--dex)
|
||||
PROFILES="$PROFILES --profile dex"
|
||||
ENABLED_SERVICES="$ENABLED_SERVICES dex"
|
||||
;;
|
||||
--langfuse)
|
||||
PROFILES="$PROFILES --profile langfuse"
|
||||
ENABLED_SERVICES="$ENABLED_SERVICES langfuse"
|
||||
;;
|
||||
--no-langfuse)
|
||||
PROFILES="${PROFILES//--profile langfuse/}"
|
||||
ENABLED_SERVICES="${ENABLED_SERVICES//langfuse/}"
|
||||
;;
|
||||
--odl-hybrid)
|
||||
if [[ "$ENABLED_SERVICES" != *"odl-hybrid"* ]]; then
|
||||
_enable_odl_hybrid_profile
|
||||
fi
|
||||
;;
|
||||
--full)
|
||||
PROFILES="--profile full"
|
||||
ENABLED_SERVICES="minio qdrant neo4j dex"
|
||||
break
|
||||
;;
|
||||
*)
|
||||
log_warning "未知参数: $1"
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# 启动服务(odl-hybrid 单独 --build,避免每次重建 docreader)
|
||||
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml $PROFILES up -d
|
||||
local compose_rc=$?
|
||||
if [ "$compose_rc" -eq 0 ] && [[ "$ENABLED_SERVICES" == *"odl-hybrid"* ]]; then
|
||||
log_info "构建/更新 odl-hybrid 镜像..."
|
||||
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml $PROFILES up -d --build odl-hybrid
|
||||
_wait_odl_hybrid_ready || true
|
||||
# docreader 需读取 DOCREADER_ODL_HYBRID;若刚改 .env,强制重建以注入环境变量
|
||||
if _should_enable_odl_hybrid_from_env; then
|
||||
log_info "重建 docreader 以应用 DOCREADER_ODL_HYBRID=${DOCREADER_ODL_HYBRID} ..."
|
||||
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml up -d --force-recreate docreader
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$compose_rc" -eq 0 ]; then
|
||||
log_success "基础设施服务已启动"
|
||||
echo ""
|
||||
log_info "服务访问地址:"
|
||||
echo " - PostgreSQL: localhost:5432"
|
||||
echo " - Redis: localhost:6379"
|
||||
echo " - DocReader: localhost:50051"
|
||||
|
||||
# 根据启用的 profile 显示额外服务
|
||||
if [[ "$ENABLED_SERVICES" == *"minio"* ]]; then
|
||||
echo " - MinIO: localhost:9000 (Console: localhost:9001)"
|
||||
fi
|
||||
if [[ "$ENABLED_SERVICES" == *"qdrant"* ]]; then
|
||||
echo " - Qdrant: localhost:6333 (gRPC: localhost:6334)"
|
||||
fi
|
||||
if [[ "$ENABLED_SERVICES" == *"neo4j"* ]]; then
|
||||
echo " - Neo4j: localhost:7474 (Bolt: localhost:7687)"
|
||||
fi
|
||||
if [[ "$ENABLED_SERVICES" == *"dex"* ]]; then
|
||||
echo " - Dex: localhost:5556"
|
||||
fi
|
||||
if [[ "$ENABLED_SERVICES" == *"langfuse"* ]]; then
|
||||
echo " - Langfuse: http://localhost:${LANGFUSE_WEB_PORT:-3000}"
|
||||
fi
|
||||
if [[ "$ENABLED_SERVICES" == *"odl-hybrid"* ]]; then
|
||||
echo " - ODL Hybrid: http://localhost:${ODL_HYBRID_PORT:-5002} (health: /health)"
|
||||
echo " docreader 需 DOCREADER_ODL_HYBRID=docling-fast"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
log_info "接下来的步骤:"
|
||||
printf "%b\n" "${YELLOW}1. 在新终端运行后端:${NC} make dev-app"
|
||||
printf "%b\n" "${YELLOW}2. 在新终端运行前端:${NC} make dev-frontend"
|
||||
return 0
|
||||
else
|
||||
log_error "服务启动失败"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 停止服务
|
||||
stop_services() {
|
||||
log_info "停止开发环境服务..."
|
||||
|
||||
check_docker
|
||||
if [ $? -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml down
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_success "所有服务已停止"
|
||||
return 0
|
||||
else
|
||||
log_error "服务停止失败"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 重启服务
|
||||
restart_services() {
|
||||
stop_services
|
||||
sleep 2
|
||||
start_services
|
||||
}
|
||||
|
||||
# 查看日志
|
||||
show_logs() {
|
||||
check_docker
|
||||
if [ $? -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml logs -f
|
||||
}
|
||||
|
||||
# 查看状态
|
||||
show_status() {
|
||||
check_docker
|
||||
if [ $? -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml ps
|
||||
}
|
||||
|
||||
# 远程开发模式下检查基础设施端口是否可达
|
||||
check_remote_dev_connectivity() {
|
||||
local host="${DEV_REMOTE_HOST:-}"
|
||||
if [ -z "$host" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local db_port="${DB_PORT:-5432}"
|
||||
local redis_port
|
||||
redis_port="${REDIS_ADDR#*:}"
|
||||
if [ "$redis_port" = "$REDIS_ADDR" ]; then
|
||||
redis_port=6379
|
||||
fi
|
||||
local docreader_port="${DOCREADER_PORT:-50051}"
|
||||
|
||||
log_info "检查远程基础设施连通性 (${host})..."
|
||||
local failed=0
|
||||
for spec in "PostgreSQL:${host}:${db_port}" "Redis:${host}:${redis_port}" "DocReader:${host}:${docreader_port}"; do
|
||||
local name="${spec%%:*}"
|
||||
local rest="${spec#*:}"
|
||||
local h="${rest%%:*}"
|
||||
local p="${rest##*:}"
|
||||
if command -v nc &> /dev/null; then
|
||||
if nc -z -w 3 "$h" "$p" 2>/dev/null; then
|
||||
log_success "${name} ${h}:${p} 可达"
|
||||
else
|
||||
log_error "${name} ${h}:${p} 不可达 (no route / connection refused)"
|
||||
failed=1
|
||||
fi
|
||||
else
|
||||
log_warning "未安装 nc,跳过 ${name} 连通性检查"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$failed" -ne 0 ]; then
|
||||
echo ""
|
||||
log_error "无法连接远程开发环境 ${host}"
|
||||
log_info "排查建议:"
|
||||
echo " 1. 确认远程机器 Docker 容器在运行 (postgres/redis/docreader)"
|
||||
echo " 2. 确认本机与 ${host} 在同一局域网 (本机: $(ipconfig getifaddr en0 2>/dev/null || echo '未知'))"
|
||||
echo " 3. 在远程检查端口映射: docker ps --format 'table {{.Names}}\t{{.Ports}}'"
|
||||
echo " 4. 检查远程防火墙是否放行 5432/6379/50051"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Host-platform path of the anydoc static archive (built by `make anydoc-lib`).
|
||||
anydoc_host_archive() {
|
||||
case "$(uname -s)-$(uname -m)" in
|
||||
Darwin-arm64) echo "$PROJECT_ROOT/third_party/anydoc-go/lib/darwin_arm64/libanydoc_go.a" ;;
|
||||
Darwin-x86_64) echo "$PROJECT_ROOT/third_party/anydoc-go/lib/darwin_amd64/libanydoc_go.a" ;;
|
||||
Linux-x86_64)
|
||||
if [ -f "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_amd64_gnu/libanydoc_go.a" ]; then
|
||||
echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_amd64_gnu/libanydoc_go.a"
|
||||
else
|
||||
echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_amd64_musl/libanydoc_go.a"
|
||||
fi
|
||||
;;
|
||||
Linux-aarch64)
|
||||
if [ -f "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_arm64_gnu/libanydoc_go.a" ]; then
|
||||
echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_arm64_gnu/libanydoc_go.a"
|
||||
else
|
||||
echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_arm64_musl/libanydoc_go.a"
|
||||
fi
|
||||
;;
|
||||
*) echo "" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Enable the in-process anydoc engine when the archive is present, unless the
|
||||
# caller already set GO_BUILD_TAGS (including empty, which opts out).
|
||||
enable_anydoc_build_tag() {
|
||||
if [ -n "${GO_BUILD_TAGS+x}" ]; then
|
||||
export GO_BUILD_TAGS
|
||||
return
|
||||
fi
|
||||
local archive
|
||||
archive="$(anydoc_host_archive)"
|
||||
if [ -n "$archive" ] && [ -f "$archive" ]; then
|
||||
export GO_BUILD_TAGS=anydoc
|
||||
log_info "检测到 anydoc 静态库,已启用 -tags anydoc"
|
||||
else
|
||||
log_info "未检测到 anydoc 静态库,解析引擎不可用。需要时先运行: make anydoc-lib"
|
||||
fi
|
||||
}
|
||||
|
||||
# 启动后端应用(本地)
|
||||
start_app() {
|
||||
log_info "启动后端应用(本地开发模式)..."
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
# 检查 Go 是否安装
|
||||
if ! command -v go &> /dev/null; then
|
||||
log_error "Go 未安装"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "加载环境配置..."
|
||||
if ! load_env_files; then
|
||||
log_error ".env 文件不存在,请先创建配置文件"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 本地 docker-compose.dev 模式:把容器服务名映射到宿主机回环地址
|
||||
# 远程开发模式(DEV_REMOTE_HOST 或 .env.local 已设地址)则保留 .env/.env.local 中的值
|
||||
if [ -n "${DEV_REMOTE_HOST:-}" ]; then
|
||||
log_info "远程开发模式: 基础设施 → ${DEV_REMOTE_HOST}"
|
||||
export DB_HOST="${DB_HOST:-$DEV_REMOTE_HOST}"
|
||||
export REDIS_ADDR="${REDIS_ADDR:-$DEV_REMOTE_HOST:6379}"
|
||||
export DOCREADER_ADDR="${DOCREADER_ADDR:-$DEV_REMOTE_HOST:50051}"
|
||||
export MINIO_ENDPOINT="${MINIO_ENDPOINT:-$DEV_REMOTE_HOST:9000}"
|
||||
export MILVUS_ADDRESS="${MILVUS_ADDRESS:-$DEV_REMOTE_HOST:19530}"
|
||||
export NEO4J_URI="${NEO4J_URI:-bolt://$DEV_REMOTE_HOST:7687}"
|
||||
export QDRANT_HOST="${QDRANT_HOST:-$DEV_REMOTE_HOST}"
|
||||
if [ -z "${LANGFUSE_HOST:-}" ] || [ "$LANGFUSE_HOST" = "http://langfuse-web:3000" ]; then
|
||||
export LANGFUSE_HOST="http://${DEV_REMOTE_HOST}:3000"
|
||||
fi
|
||||
else
|
||||
export DB_HOST=127.0.0.1
|
||||
export DOCREADER_ADDR=127.0.0.1:50051
|
||||
export MINIO_ENDPOINT=127.0.0.1:9000
|
||||
export REDIS_ADDR=127.0.0.1:6379
|
||||
export MILVUS_ADDRESS=127.0.0.1:19530
|
||||
export NEO4J_URI=bolt://127.0.0.1:7687
|
||||
export QDRANT_HOST=127.0.0.1
|
||||
fi
|
||||
export DOCREADER_TRANSPORT="${DOCREADER_TRANSPORT:-grpc}"
|
||||
|
||||
if ! check_remote_dev_connectivity; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# .env.example uses /data/files for the Docker app container, where a
|
||||
# volume is mounted at that path. When the backend runs directly on the
|
||||
# host via dev-app, /data is often read-only or missing, so use a repo-local
|
||||
# writable directory unless the developer explicitly configured another
|
||||
# local storage path.
|
||||
if [ -z "${LOCAL_STORAGE_BASE_DIR:-}" ] || [ "$LOCAL_STORAGE_BASE_DIR" = "/data/files" ]; then
|
||||
export LOCAL_STORAGE_BASE_DIR="$PROJECT_ROOT/.local-data/files"
|
||||
fi
|
||||
mkdir -p "$LOCAL_STORAGE_BASE_DIR"
|
||||
|
||||
# 确保必要的环境变量已设置
|
||||
if [ -z "$DB_DRIVER" ]; then
|
||||
log_error "DB_DRIVER 环境变量未设置,请检查 .env 文件"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "环境变量已设置,启动应用..."
|
||||
log_info "数据库地址: $DB_HOST:${DB_PORT:-5432}"
|
||||
|
||||
export CGO_CFLAGS="-Wno-deprecated-declarations -Wno-gnu-folding-constant"
|
||||
if [[ "$(uname)" == "Darwin" ]]; then
|
||||
export CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries"
|
||||
fi
|
||||
|
||||
enable_anydoc_build_tag
|
||||
|
||||
# 检查是否安装了 Air(热重载工具)
|
||||
if command -v air &> /dev/null; then
|
||||
log_success "检测到 Air,使用热重载模式启动..."
|
||||
log_info "修改 Go 代码后将自动重新编译和重启"
|
||||
air
|
||||
else
|
||||
log_info "未检测到 Air,使用普通模式启动"
|
||||
log_warning "提示: 安装 Air 可以实现代码修改后自动重启"
|
||||
log_info "安装命令: go install github.com/air-verse/air@latest"
|
||||
LDFLAGS="$(./scripts/get_version.sh ldflags) -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"
|
||||
go run -tags "${GO_BUILD_TAGS:-}" -ldflags="$LDFLAGS" ./cmd/server
|
||||
fi
|
||||
}
|
||||
|
||||
# 启动前端(本地)
|
||||
start_frontend() {
|
||||
log_info "启动前端开发服务器..."
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
if [ -f ".env" ] || [ -f ".env.local" ]; then
|
||||
load_env_files >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
cd "$PROJECT_ROOT/frontend"
|
||||
|
||||
# 检查 npm 是否安装
|
||||
if ! command -v npm &> /dev/null; then
|
||||
log_error "npm 未安装"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 检查依赖是否已安装
|
||||
if [ ! -d "node_modules" ]; then
|
||||
log_warning "node_modules 不存在,正在安装依赖..."
|
||||
npm install
|
||||
fi
|
||||
|
||||
log_info "启动 Vite 开发服务器..."
|
||||
log_info "前端将运行在 http://localhost:5173"
|
||||
log_info "前端 API 代理目标: ${VITE_DEV_PROXY_TARGET:-${FRONTEND_BACKEND_URL:-http://localhost:8080}}"
|
||||
|
||||
# 运行开发服务器
|
||||
npm run dev
|
||||
}
|
||||
|
||||
# 解析命令
|
||||
CMD="${1:-help}"
|
||||
case "$CMD" in
|
||||
start)
|
||||
start_services "$@"
|
||||
;;
|
||||
stop)
|
||||
stop_services
|
||||
;;
|
||||
restart)
|
||||
restart_services
|
||||
;;
|
||||
logs)
|
||||
show_logs
|
||||
;;
|
||||
status)
|
||||
show_status
|
||||
;;
|
||||
app)
|
||||
start_app
|
||||
;;
|
||||
frontend)
|
||||
start_frontend
|
||||
;;
|
||||
help|--help|-h)
|
||||
show_help
|
||||
;;
|
||||
*)
|
||||
log_error "未知命令: $CMD"
|
||||
show_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,101 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const { execFile } = require('node:child_process')
|
||||
const { promisify } = require('node:util')
|
||||
|
||||
const execute = promisify(execFile)
|
||||
const directory = path.resolve(__dirname, '../../acceptance')
|
||||
const startedAt = new Date().toISOString()
|
||||
const reportPath = path.join(directory, `public-latency-${startedAt.replace(/[:.]/g, '-')}.json`)
|
||||
const report = { started_at: startedAt, samples: [], summaries: [] }
|
||||
const headerNames = ['cache-control', 'cf-cache-status', 'cf-ray', 'age', 'etag', 'last-modified', 'content-type', 'content-encoding', 'content-length', 'vary', 'server']
|
||||
const measurementFormat = '\n__BEGIN__%{json}__HEADERS__%{header_json}__END__\n'
|
||||
|
||||
function decode(output, label, url) {
|
||||
return [...output.matchAll(/__BEGIN__([\s\S]*?)__HEADERS__([\s\S]*?)__END__/g)].map((match, index) => {
|
||||
const measurement = JSON.parse(match[1])
|
||||
const headers = JSON.parse(match[2])
|
||||
const ms = (value) => Math.round(value * 1000000) / 1000
|
||||
return {
|
||||
label, url, connection_sequence: index + 1,
|
||||
status: measurement.http_code, exit_code: measurement.exitcode,
|
||||
http_version: measurement.http_version, new_connections: measurement.num_connects,
|
||||
remote_ip: measurement.remote_ip, downloaded_bytes: measurement.size_download,
|
||||
dns_ms: ms(measurement.time_namelookup),
|
||||
connect_ms: ms(measurement.time_connect), tls_ms: ms(measurement.time_appconnect),
|
||||
ttfb_ms: ms(measurement.time_starttransfer), total_ms: ms(measurement.time_total),
|
||||
wait_after_request_ready_ms: ms(measurement.time_starttransfer - measurement.time_pretransfer),
|
||||
headers: Object.fromEntries(headerNames.filter(name => headers[name]).map(name => [name, headers[name]])),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function measure(label, url, family, repetitions = 1) {
|
||||
const args = ['--silent', '--show-error', '--compressed', '--connect-timeout', '8', '--max-time', '15', '--write-out', measurementFormat]
|
||||
if (family) args.push(family)
|
||||
for (let index = 0; index < repetitions; index++) args.push('--output', '/dev/null', url)
|
||||
let result
|
||||
try { result = await execute('curl', args, { maxBuffer: 1024 * 1024 }) }
|
||||
catch (error) { result = error }
|
||||
const samples = decode(result.stdout || '', label, url)
|
||||
if (!samples.length) samples.push({ label, url, failed: true, exit_code: result.code || 1 })
|
||||
report.samples.push(...samples)
|
||||
console.log(JSON.stringify({ label, samples: samples.map(sample => ({ status: sample.status, ttfb_ms: sample.ttfb_ms, total_ms: sample.total_ms, exit_code: sample.exit_code })) }))
|
||||
}
|
||||
|
||||
async function tunnelMetrics() {
|
||||
try {
|
||||
const response = await fetch('http://127.0.0.1:20241/metrics', { signal: AbortSignal.timeout(3000) })
|
||||
const lines = (await response.text()).split('\n')
|
||||
return lines.filter(line => /^cloudflared_tunnel_(?:ha_connections|request_errors|total_requests|server_locations)/.test(line))
|
||||
} catch { return ['metrics_unavailable'] }
|
||||
}
|
||||
|
||||
function summarize() {
|
||||
for (const label of new Set(report.samples.map(sample => sample.label))) {
|
||||
const samples = report.samples.filter(sample => sample.label === label && sample.exit_code === 0)
|
||||
const median = key => {
|
||||
const values = samples.map(sample => sample[key]).sort((a, b) => a - b)
|
||||
const middle = Math.floor(values.length / 2)
|
||||
return values.length ? (values.length % 2 ? values[middle] : (values[middle - 1] + values[middle]) / 2) : null
|
||||
}
|
||||
report.summaries.push({ label, successful_samples: samples.length,
|
||||
dns_median_ms: median('dns_ms'), connect_median_ms: median('connect_ms'), tls_median_ms: median('tls_ms'),
|
||||
ttfb_median_ms: median('ttfb_ms'), total_median_ms: median('total_ms'), wait_median_ms: median('wait_after_request_ready_ms'),
|
||||
cache_states: [...new Set(samples.flatMap(sample => sample.headers?.['cf-cache-status'] || []))],
|
||||
edge_locations: [...new Set(samples.flatMap(sample => sample.headers?.['cf-ray']?.map(ray => ray.split('-').at(-1)) || []))],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
report.tunnel_before = await tunnelMetrics()
|
||||
const targets = [
|
||||
['html', '/login'],
|
||||
['small_static', '/assets/favicon-bEsoBimf.ico'],
|
||||
['api_guard', '/api/v1/auth/me'],
|
||||
]
|
||||
for (let round = 0; round < 3; round++) {
|
||||
for (const [name, resource] of targets) {
|
||||
await measure(`origin_${name}`, `http://127.0.0.1:18088${resource}`)
|
||||
await measure(`public_ipv4_${name}`, `https://agent.pducv.cn${resource}`, '--ipv4')
|
||||
await measure(`public_ipv6_${name}`, `https://agent.pducv.cn${resource}`, '--ipv6')
|
||||
}
|
||||
await measure('backend_health', 'http://127.0.0.1:18080/health')
|
||||
}
|
||||
for (const [label, family] of [['ipv4', '--ipv4'], ['ipv6', '--ipv6']]) {
|
||||
await measure(`public_${label}_warm_html`, 'https://agent.pducv.cn/login', family, 3)
|
||||
await measure(`public_${label}_config`, 'https://agent.pducv.cn/config.js', family)
|
||||
await measure(`public_${label}_icons`, 'https://agent.pducv.cn/tdesign-icons/0.4.1/fonts/index.js', family)
|
||||
}
|
||||
await measure('origin_config', 'http://127.0.0.1:18088/config.js')
|
||||
await measure('origin_icons', 'http://127.0.0.1:18088/tdesign-icons/0.4.1/fonts/index.js')
|
||||
report.tunnel_after = await tunnelMetrics()
|
||||
}
|
||||
|
||||
main().catch(error => { report.error = error.name; process.exitCode = 1 }).finally(() => {
|
||||
report.completed_at = new Date().toISOString()
|
||||
summarize()
|
||||
fs.writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n', { flag: 'wx' })
|
||||
console.log(JSON.stringify({ report: reportPath, summaries: report.summaries, error: report.error }))
|
||||
})
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# ─── Fix ownership of bind-mounted directories ───
|
||||
# When users bind-mount host directories (e.g. ./skills/preloaded),
|
||||
# the mount inherits the host UID/GID which may differ from the
|
||||
# container's appuser. This entrypoint runs as root, fixes ownership,
|
||||
# then drops privileges to appuser via gosu — the same pattern used
|
||||
# by official postgres/redis images.
|
||||
|
||||
# Directories that may be bind-mounted and need appuser access
|
||||
MOUNT_DIRS=(
|
||||
/app/skills/preloaded
|
||||
/data/files
|
||||
)
|
||||
|
||||
for dir in "${MOUNT_DIRS[@]}"; do
|
||||
if [ -d "$dir" ]; then
|
||||
chown -R appuser:appuser "$dir" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
|
||||
# ─── Merge built-in skills into preloaded ───
|
||||
# Built-in skills are backed up at /app/skills/_builtin during image build.
|
||||
# After a bind-mount replaces /app/skills/preloaded, copy back any
|
||||
# missing built-in skills (without overwriting user-provided ones).
|
||||
BUILTIN_DIR="/app/skills/_builtin"
|
||||
PRELOADED_DIR="/app/skills/preloaded"
|
||||
|
||||
if [ -d "$BUILTIN_DIR" ]; then
|
||||
mkdir -p "$PRELOADED_DIR"
|
||||
for skill_dir in "$BUILTIN_DIR"/*/; do
|
||||
[ -d "$skill_dir" ] || continue
|
||||
skill_name="$(basename "$skill_dir")"
|
||||
if [ ! -d "$PRELOADED_DIR/$skill_name" ]; then
|
||||
cp -r "$skill_dir" "$PRELOADED_DIR/$skill_name"
|
||||
fi
|
||||
done
|
||||
chown -R appuser:appuser "$PRELOADED_DIR"
|
||||
fi
|
||||
|
||||
# ─── Docker socket access for the sandbox backend ───
|
||||
# The Engine API socket is typically root:docker 0660. This process then
|
||||
# drops to appuser via gosu, which calls initgroups and therefore drops
|
||||
# compose group_add. Match the socket's GID in /etc/group before gosu.
|
||||
# Never chmod the host socket: that would weaken daemon access on the host.
|
||||
grant_docker_sock_to_appuser() {
|
||||
local sock="$1"
|
||||
local gid grp
|
||||
[ -S "$sock" ] || return 0
|
||||
if gosu appuser sh -c "test -r \"$sock\" && test -w \"$sock\"" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
gid="$(stat -c '%g' "$sock" 2>/dev/null || true)"
|
||||
if [ -z "$gid" ]; then
|
||||
echo "weknora: cannot stat $sock; Docker sandbox may be unable to reach the daemon" >&2
|
||||
return 0
|
||||
fi
|
||||
if [ "$gid" = "0" ]; then
|
||||
echo "weknora: $sock is not writable by appuser and owned by GID 0; Docker sandbox needs a group-writable socket with a non-root GID" >&2
|
||||
return 0
|
||||
fi
|
||||
if ! getent group "$gid" >/dev/null 2>&1; then
|
||||
if ! groupadd -g "$gid" dockersock >/dev/null 2>&1; then
|
||||
echo "weknora: failed to create group for $sock GID $gid; Docker sandbox may be unable to reach the daemon" >&2
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
grp="$(getent group "$gid" | cut -d: -f1)"
|
||||
if [ -z "$grp" ]; then
|
||||
echo "weknora: no group name for GID $gid on $sock" >&2
|
||||
return 0
|
||||
fi
|
||||
if ! usermod -aG "$grp" appuser >/dev/null 2>&1; then
|
||||
echo "weknora: failed to add appuser to $grp for $sock; Docker sandbox may be unable to reach the daemon" >&2
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
grant_docker_sock_to_appuser /var/run/docker.sock
|
||||
case "${DOCKER_HOST:-}" in
|
||||
unix://*)
|
||||
grant_docker_sock_to_appuser "${DOCKER_HOST#unix://}"
|
||||
;;
|
||||
esac
|
||||
|
||||
# ─── Drop privileges and exec the main process ───
|
||||
exec gosu appuser "$@"
|
||||
@@ -0,0 +1,57 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const assert = require('node:assert/strict')
|
||||
const root = path.resolve(__dirname, '../..')
|
||||
const ids = ['16c3e59c-4bb6-41cf-ae0a-6d0941b3d032', '20b9280e-ed73-4700-8fa5-944d6ae6dd5f', 'bb86b44a-576b-4bf8-a6bf-8f1baca3a845', 'badd084b-f229-4005-8fb3-196fbe139fe0', '5d28a7e5-e00e-44a7-a0e1-c72342cdcf68', '3ae9257d-928a-4886-8101-7cfd0ed8c27c']
|
||||
let token
|
||||
async function api(route, method = 'GET', body) {
|
||||
const response = await fetch(`http://127.0.0.1:18080/api/v1${route}`, { method,
|
||||
headers: { 'Content-Type': 'application/json', ...(token ? { Authorization: `Bearer ${token}`, 'X-Tenant-ID': '10059' } : {}) },
|
||||
body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(30000) })
|
||||
const result = await response.json()
|
||||
assert.ok(response.ok && result.success !== false, `${method} ${route}: HTTP ${response.status}`)
|
||||
return result
|
||||
}
|
||||
async function main() {
|
||||
const credentials = JSON.parse(fs.readFileSync(path.join(root, 'acceptance/super-admin-login.json'), 'utf8'))
|
||||
const login = await api('/auth/login', 'POST', { username: credentials.username, password: credentials.password })
|
||||
token = (login.data || login).token
|
||||
const selected = process.argv.includes('--product-only') ? ids.slice(0, 1) : ids
|
||||
const before = []
|
||||
for (const id of selected) before.push((await api(`/knowledge-bases/${id}`)).data)
|
||||
if (!process.argv.includes('--apply')) {
|
||||
console.log(JSON.stringify(before.map(kb => ({ name: kb.name, enabled: !!kb.extract_config?.enabled, initialized: !!(kb.embedding_model_id && kb.summary_model_id) }))))
|
||||
return
|
||||
}
|
||||
fs.writeFileSync(path.join(root, 'acceptance', `kb-graph-config-before-${Date.now()}.json`), JSON.stringify(before, null, 2), { mode: 0o600, flag: 'wx' })
|
||||
for (const kb of before) {
|
||||
assert.ok(kb.embedding_model_id && kb.summary_model_id, 'Models must be configured first')
|
||||
const config = (await api(`/initialization/config/${kb.id}`)).data
|
||||
const request = { ...config,
|
||||
llmModelId: kb.summary_model_id, embeddingModelId: kb.embedding_model_id,
|
||||
nodeExtract: {
|
||||
enabled: true,
|
||||
text: '基础概念是实践练习的先修条件。',
|
||||
tags: ['先修条件', '包含', '适用于', '条件适用于', '禁用于', '慎用于', '不适用于', '不能用于', '关联', '导致', '不导致', '约束', '对比', '支持', '不支持', '抑制'],
|
||||
nodes: [{ name: '基础概念', attributes: ['概念'] }, { name: '实践练习', attributes: ['方法'] }],
|
||||
relations: [{ node1: '基础概念', node2: '实践练习', type: '先修条件', evidence: '基础概念是实践练习的先修条件。' }],
|
||||
customInstructions: '按当前原文的实际主题提取有依据的实体和关系,不根据知识库名称预设领域。每条关系的 evidence 必须逐字引用连续原文,保留主体、时间、否定、条件、观点归属和不确定性。叙事和个人观点不能改写成通用因果;先修条件按前提指向后续目标;约束须保留完整动作。产品使用标签只适用于明确的产品使用说明,不同产品分节不得串用关系。不要把格式示例写入实际图谱,不要补造原文没有的关系;确实没有关系时如实返回,交由覆盖复核。',
|
||||
},
|
||||
}
|
||||
await api(`/initialization/config/${kb.id}`, 'PUT', request)
|
||||
await api(`/knowledge-bases/${kb.id}`, 'PUT', { name: kb.name, description: kb.description,
|
||||
config: { chunking_config: kb.chunking_config, image_processing_config: kb.image_processing_config,
|
||||
faq_config: kb.faq_config, auto_tag_config: kb.auto_tag_config, wiki_config: kb.wiki_config,
|
||||
indexing_strategy: { ...kb.indexing_strategy, graph_enabled: true } } })
|
||||
const after = (await api(`/knowledge-bases/${kb.id}`)).data
|
||||
assert.equal(after.extract_config.enabled, true)
|
||||
assert.equal(after.indexing_strategy.graph_enabled, true)
|
||||
assert.equal(after.embedding_model_id, kb.embedding_model_id)
|
||||
assert.equal(after.summary_model_id, kb.summary_model_id)
|
||||
assert.equal(after.storage_backend_id, kb.storage_backend_id)
|
||||
assert.deepEqual(after.wiki_config, kb.wiki_config)
|
||||
assert.deepEqual(after.chunking_config, kb.chunking_config)
|
||||
}
|
||||
console.log(JSON.stringify({ configured: before.map(kb => kb.name), existing_indexes_preserved: true, graph_build_complete: false }))
|
||||
}
|
||||
main().catch(error => { console.error(error.message); process.exitCode = 1 })
|
||||
@@ -0,0 +1,96 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const assert = require('node:assert/strict')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
|
||||
const root = path.resolve(__dirname, '../..')
|
||||
const directory = path.join(root, 'acceptance')
|
||||
const tenant = 10059
|
||||
const businessIDs = ['16c3e59c-4bb6-41cf-ae0a-6d0941b3d032', '20b9280e-ed73-4700-8fa5-944d6ae6dd5f', 'bb86b44a-576b-4bf8-a6bf-8f1baca3a845', 'badd084b-f229-4005-8fb3-196fbe139fe0', '5d28a7e5-e00e-44a7-a0e1-c72342cdcf68', '3ae9257d-928a-4886-8101-7cfd0ed8c27c']
|
||||
const model = '404e005b-83ae-5e2e-a3b3-999f9d5cd47e'
|
||||
const origin = 'http://127.0.0.1:18080/api/v1'
|
||||
const statePath = path.join(directory, 'business-wiki-backfill.json')
|
||||
const literal = s => `'${String(s).replaceAll("'", "''")}'`
|
||||
const docker = (args, input) => execFileSync('docker', ['--context', 'colima-yawan', ...args], {input, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], maxBuffer: 8 * 1024 * 1024})
|
||||
const sql = statement => docker(['exec', '-i', 'WeKnora-postgres', 'psql', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-At', '-v', 'ON_ERROR_STOP=1'], statement).trim()
|
||||
let auth
|
||||
async function api(route, method = 'GET', body) {
|
||||
const response = await fetch(origin + route, {method, headers: {'Content-Type': 'application/json', ...(auth ? {Authorization: `Bearer ${auth.token}`, 'X-Tenant-ID': String(tenant)} : {})}, body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(30000)})
|
||||
const value = await response.json()
|
||||
assert.ok(response.ok && value.success !== false, `${method} ${route}: HTTP ${response.status}`)
|
||||
return value
|
||||
}
|
||||
function inventory() {
|
||||
return JSON.parse(sql(`SELECT coalesce(json_agg(row_to_json(t)),'[]'::json) FROM (
|
||||
SELECT k.id,k.name,k.type,k.indexing_strategy,k.wiki_config,k.embedding_model_id,
|
||||
(SELECT count(*) FROM knowledges n WHERE n.knowledge_base_id=k.id AND n.deleted_at IS NULL) AS documents,
|
||||
(SELECT count(*) FROM chunks c WHERE c.knowledge_base_id=k.id AND c.deleted_at IS NULL) AS chunks,
|
||||
(SELECT count(*) FROM wiki_pages p WHERE p.knowledge_base_id=k.id AND p.deleted_at IS NULL) AS wiki_pages,
|
||||
(SELECT count(*) FROM task_pending_ops o WHERE o.scope_id=k.id AND o.task_type='wiki:ingest') AS pending_documents,
|
||||
(SELECT count(*) FROM task_pending_ops o WHERE o.scope_id=k.id AND o.task_type='wiki:finalize') AS pending_finalization,
|
||||
(SELECT count(*) FROM task_dead_letters d WHERE d.scope_id=k.id AND d.task_type='wiki:ingest') AS failed_documents
|
||||
FROM knowledge_bases k WHERE k.tenant_id=${tenant} AND k.id IN (${businessIDs.map(literal).join(',')}) AND k.deleted_at IS NULL AND k.is_temporary=false ORDER BY k.name
|
||||
) t;`))
|
||||
}
|
||||
async function main() {
|
||||
const credentials = JSON.parse(fs.readFileSync(path.join(directory, 'super-admin-login.json'), 'utf8'))
|
||||
auth = await api('/auth/login', 'POST', {username: credentials.username, password: credentials.password})
|
||||
assert.equal(auth.user.is_system_admin, true)
|
||||
const before = inventory()
|
||||
assert.equal(before.length, businessIDs.length, 'All six business knowledge bases must exist')
|
||||
let state = fs.existsSync(statePath) ? JSON.parse(fs.readFileSync(statePath, 'utf8')) : {started_at: new Date().toISOString(), tenant_id: tenant, before}
|
||||
if (process.argv.includes('--apply')) {
|
||||
const modelRow = sql(`SELECT count(*) FROM models WHERE id=${literal(model)} AND tenant_id=${tenant} AND deleted_at IS NULL AND type='KnowledgeQA';`)
|
||||
assert.equal(modelRow, '1')
|
||||
fs.writeFileSync(path.join(directory, `business-wiki-config-before-${Date.now()}.json`), JSON.stringify(before, null, 2), {mode: 0o600, flag: 'wx'})
|
||||
for (const row of before) {
|
||||
assert.equal(row.type, 'document', 'Only document knowledge bases are supported')
|
||||
assert.equal(row.indexing_strategy.vector_enabled, true, 'Existing vector index is required')
|
||||
assert.equal(row.indexing_strategy.keyword_enabled, true, 'Existing keyword index is required')
|
||||
const kb = (await api(`/knowledge-bases/${row.id}`)).data
|
||||
const updated = await api(`/knowledge-bases/${row.id}`, 'PUT', {name: kb.name, description: kb.description, config: {
|
||||
chunking_config: kb.chunking_config, image_processing_config: kb.image_processing_config,
|
||||
faq_config: kb.faq_config, auto_tag_config: kb.auto_tag_config,
|
||||
indexing_strategy: {...kb.indexing_strategy, wiki_enabled: true},
|
||||
wiki_config: {...kb.wiki_config, synthesis_model_id: model, extraction_granularity: 'standard', max_pages_per_ingest: 0,
|
||||
ingest_batch_size: 1, ingest_map_parallel: 1, ingest_reduce_parallel: 1, ingest_max_inflight: 1},
|
||||
}})
|
||||
assert.equal(updated.data.indexing_strategy.wiki_enabled, true)
|
||||
assert.equal(updated.data.embedding_model_id, row.embedding_model_id)
|
||||
}
|
||||
const ids = before.map(k => literal(k.id)).join(',')
|
||||
// Backfill the Wiki queue without resetting document status or rebuilding RAG indexes.
|
||||
state.enqueued = Number(sql(`WITH added AS (
|
||||
INSERT INTO task_pending_ops (tenant_id,task_type,scope,scope_id,op,dedup_key,payload,fail_count,enqueued_at)
|
||||
SELECT n.tenant_id,'wiki:ingest','knowledge_base',n.knowledge_base_id,'ingest',n.id,
|
||||
jsonb_build_object('op','ingest','knowledge_id',n.id,'language','zh-CN'),0,now()
|
||||
FROM knowledges n JOIN knowledge_bases k ON k.id=n.knowledge_base_id AND k.tenant_id=n.tenant_id
|
||||
WHERE n.tenant_id=${tenant} AND n.knowledge_base_id IN (${ids}) AND n.deleted_at IS NULL
|
||||
AND k.deleted_at IS NULL AND k.indexing_strategy->>'wiki_enabled'='true'
|
||||
AND n.parse_status='completed' AND n.pending_subtasks_count=0
|
||||
AND EXISTS (SELECT 1 FROM chunks c WHERE c.knowledge_id=n.id AND c.deleted_at IS NULL)
|
||||
AND NOT EXISTS (SELECT 1 FROM task_pending_ops o WHERE o.tenant_id=n.tenant_id AND o.scope_id=n.knowledge_base_id AND o.task_type='wiki:ingest' AND o.dedup_key=n.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM wiki_pages p WHERE p.knowledge_base_id=n.knowledge_base_id AND p.deleted_at IS NULL AND p.source_refs @> jsonb_build_array(n.id))
|
||||
RETURNING id
|
||||
) SELECT count(*) FROM added;`))
|
||||
state.configured_at = new Date().toISOString()
|
||||
fs.writeFileSync(statePath, JSON.stringify(state, null, 2), {mode: 0o600})
|
||||
const scopes = before.map(k => ({tenant_id: tenant, knowledge_base_id: k.id, language: 'zh-CN'}))
|
||||
docker(['cp', path.join(directory, 'wiki-backfill-trigger'), 'yawan-native-acceptance:/tmp/wiki-backfill-trigger'])
|
||||
state.trigger_result = JSON.parse(docker(['exec', '-i', 'yawan-native-acceptance', '/tmp/wiki-backfill-trigger'], JSON.stringify(scopes)))
|
||||
}
|
||||
state.checked_at = new Date().toISOString()
|
||||
state.current = inventory()
|
||||
for (const prior of state.before) {
|
||||
const current = state.current.find(k => k.id === prior.id)
|
||||
assert.equal(current.documents, prior.documents)
|
||||
assert.equal(current.chunks, prior.chunks)
|
||||
assert.equal(current.embedding_model_id, prior.embedding_model_id)
|
||||
}
|
||||
state.rag_preserved = true
|
||||
state.all_wiki_enabled = state.current.every(k => k.indexing_strategy.wiki_enabled && k.indexing_strategy.vector_enabled && k.indexing_strategy.keyword_enabled)
|
||||
if (process.argv.includes('--apply')) assert.equal(state.all_wiki_enabled, true)
|
||||
fs.writeFileSync(statePath, JSON.stringify(state, null, 2), {mode: 0o600})
|
||||
console.log(JSON.stringify({checked_at: state.checked_at, all_wiki_enabled: state.all_wiki_enabled, rag_preserved: true, enqueued: state.enqueued, knowledge_bases: state.current.map(k => ({name: k.name,documents: k.documents,chunks: k.chunks,wiki_pages: k.wiki_pages,pending_documents:k.pending_documents,pending_finalization:k.pending_finalization,failed_documents:k.failed_documents}))}))
|
||||
}
|
||||
main().catch(error => {console.error(error.message); process.exitCode = 1})
|
||||
@@ -0,0 +1,91 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
const { idle } = require('./publish_public_runtime.cjs')
|
||||
|
||||
const root = path.resolve(__dirname, '../..')
|
||||
const modelID = '404e005b-83ae-5e2e-a3b3-999f9d5cd47e'
|
||||
const tenantID = 10059
|
||||
const report = { checked_at: new Date().toISOString(), model_id: modelID, tenant_id: tenantID, model_calls: 0, changed: false }
|
||||
let token, stage = 'snapshot'
|
||||
|
||||
function snapshot() {
|
||||
const sql = `BEGIN READ ONLY;
|
||||
SELECT json_build_object('model_id',id,'provider',parameters->>'provider',
|
||||
'thinking_control',parameters->'extra_config'->>'thinking_control',
|
||||
'unchanged_fields_sha256',encode(sha256(convert_to((to_jsonb(models) - 'updated_at' - 'parameters' ||
|
||||
jsonb_build_object('parameters',parameters #- '{extra_config,thinking_control}'))::text,'UTF8')),'hex'))
|
||||
FROM models WHERE id='${modelID}' AND tenant_id=${tenantID} AND deleted_at IS NULL;
|
||||
COMMIT;`
|
||||
const result = execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres',
|
||||
'psql', '-X', '-qAt', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915'],
|
||||
{ input: sql, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 15000 })
|
||||
return JSON.parse(result)
|
||||
}
|
||||
|
||||
async function api(route, method = 'GET', body) {
|
||||
const response = await fetch(`http://127.0.0.1:18080/api/v1${route}`, {
|
||||
method, headers: { 'Content-Type': 'application/json', ...(token ? { Authorization: `Bearer ${token}`, 'X-Tenant-ID': String(tenantID) } : {}) },
|
||||
body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(30000),
|
||||
})
|
||||
assert.equal(response.status, 200)
|
||||
const result = await response.json()
|
||||
assert.notEqual(result.success, false)
|
||||
return result.data || result
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const before = snapshot()
|
||||
report.before = before
|
||||
assert.equal(before.provider, 'deepseek')
|
||||
assert.equal(before.thinking_control, 'none')
|
||||
if (!process.argv.includes('--apply')) {
|
||||
console.log(JSON.stringify({ ...report, proposed: 'thinking_type', dry_run: true }))
|
||||
return
|
||||
}
|
||||
stage = 'idle_check'
|
||||
idle()
|
||||
stage = 'login'
|
||||
const credentials = JSON.parse(fs.readFileSync(path.join(root, 'acceptance/super-admin-login.json'), 'utf8'))
|
||||
token = (await api('/auth/login', 'POST', credentials)).token
|
||||
assert.ok(token)
|
||||
stage = 'model_read'
|
||||
const model = await api(`/models/${modelID}`)
|
||||
assert.equal(model.tenant_id, tenantID)
|
||||
assert.equal(model.name, 'deepseek-flash')
|
||||
assert.equal(model.type, 'KnowledgeQA')
|
||||
assert.equal(model.source, 'remote')
|
||||
assert.equal(model.parameters.provider, 'deepseek')
|
||||
assert.equal(new URL(model.parameters.base_url).hostname, 'api.deepseek.com')
|
||||
assert.equal(model.parameters.extra_config?.thinking_control, 'none')
|
||||
const receiptPath = path.join(root, 'acceptance', `graph-model-thinking-fix-${Date.now()}.json`)
|
||||
fs.writeFileSync(receiptPath, JSON.stringify(report, null, 2) + '\n', { flag: 'wx', mode: 0o600 })
|
||||
stage = 'compare_before_update'
|
||||
assert.deepEqual(snapshot(), before)
|
||||
stage = 'model_update'
|
||||
await api(`/models/${modelID}`, 'PUT', {
|
||||
name: model.name, display_name: model.display_name, description: model.description,
|
||||
type: model.type, source: model.source,
|
||||
parameters: { ...model.parameters, extra_config: { ...model.parameters.extra_config, thinking_control: 'thinking_type' } },
|
||||
})
|
||||
report.changed = true
|
||||
stage = 'verify'
|
||||
report.after = snapshot()
|
||||
fs.writeFileSync(receiptPath, JSON.stringify(report, null, 2) + '\n', { mode: 0o600 })
|
||||
assert.equal(report.after.thinking_control, 'thinking_type')
|
||||
assert.equal(report.after.unchanged_fields_sha256, before.unchanged_fields_sha256)
|
||||
const after = await api(`/models/${modelID}`)
|
||||
assert.equal(after.parameters.extra_config.thinking_control, 'thinking_type')
|
||||
report.passed = true
|
||||
report.verified_at = new Date().toISOString()
|
||||
fs.writeFileSync(receiptPath, JSON.stringify(report, null, 2) + '\n', { mode: 0o600 })
|
||||
console.log(JSON.stringify({ passed: true, changed: true, before: 'none', after: 'thinking_type',
|
||||
unrelated_fields_and_credentials_preserved: true, model_calls: 0, receipt_path: receiptPath }))
|
||||
}
|
||||
|
||||
main().catch(() => {
|
||||
console.error(JSON.stringify({ passed: false, stage, changed: report.changed, model_calls: 0,
|
||||
error: 'Guarded model configuration update stopped; inspect the private receipt without printing credentials.' }))
|
||||
process.exitCode = 1
|
||||
})
|
||||
@@ -0,0 +1,121 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
const fs = require('node:fs')
|
||||
const os = require('node:os')
|
||||
const path = require('node:path')
|
||||
const test = require('node:test')
|
||||
const { setTimeout: delay } = require('node:timers/promises')
|
||||
|
||||
const frontend = path.resolve(__dirname, '../frontend')
|
||||
const dockerContext = process.env.NGINX_TEST_DOCKER_CONTEXT
|
||||
const docker = (...args) => execFileSync('docker', [
|
||||
...(dockerContext ? ['--context', dockerContext] : []), ...args,
|
||||
], { encoding: 'utf8', timeout: 30000, stdio: ['ignore', 'pipe', 'pipe'] }).trim()
|
||||
|
||||
// Runs only a disposable nginx with static fixtures, never the application stack.
|
||||
test('nginx serves runtime config and versioned assets with distinct cache policies', { timeout: 90000 }, async t => {
|
||||
const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'weknora-cache-test-'))
|
||||
const container = `weknora-cache-test-${process.pid}-${Date.now()}`
|
||||
const image = process.env.NGINX_TEST_IMAGE || fs.readFileSync(path.join(frontend, 'Dockerfile'), 'utf8').match(/^FROM (\S+)/m)[1]
|
||||
let created = false
|
||||
try {
|
||||
const site = path.join(temp, 'html')
|
||||
const fixtures = {
|
||||
'index.html': '<html>main cache fixture</html>',
|
||||
'embed.html': '<html>embed cache fixture</html>',
|
||||
'config.js': 'window.__RUNTIME_CONFIG__ = { MAX_FILE_SIZE_MB: 50 };',
|
||||
'tdesign-icons/0.4.1/fonts/index.js': 'window.cacheTestIcon = true;',
|
||||
'assets/cache-test-012345.js': 'window.cacheTestAsset = true;',
|
||||
}
|
||||
for (const [name, body] of Object.entries(fixtures)) {
|
||||
const filename = path.join(site, name)
|
||||
fs.mkdirSync(path.dirname(filename), { recursive: true })
|
||||
fs.writeFileSync(filename, body)
|
||||
}
|
||||
const replacements = { APP_SCHEME: 'http', APP_HOST: '127.0.0.1', APP_PORT: '9', MAX_FILE_SIZE: '50M', MAX_SKILL_BUNDLE_SIZE: '256M' }
|
||||
const config = fs.readFileSync(path.join(frontend, 'nginx.conf'), 'utf8')
|
||||
.replace(/\$\{(APP_SCHEME|APP_HOST|APP_PORT|MAX_FILE_SIZE|MAX_SKILL_BUNDLE_SIZE)\}/g, (_, key) => replacements[key])
|
||||
fs.writeFileSync(path.join(temp, 'default.conf'), config)
|
||||
docker('create', '--rm', '--pull', 'never', '--name', container,
|
||||
'--publish', '127.0.0.1::80',
|
||||
'--entrypoint', 'nginx', image, '-g', 'daemon off;')
|
||||
created = true
|
||||
// Copying also supports remote Docker contexts without host-directory mounts.
|
||||
docker('cp', `${site}/.`, `${container}:/usr/share/nginx/html`)
|
||||
docker('cp', path.join(temp, 'default.conf'), `${container}:/etc/nginx/conf.d/default.conf`)
|
||||
docker('cp', path.join(frontend, 'nginx-api-proxy.conf'), `${container}:/etc/nginx/api-proxy.conf`)
|
||||
docker('start', container)
|
||||
docker('exec', container, 'nginx', '-t')
|
||||
const address = docker('port', container, '80/tcp').split('\n')[0]
|
||||
assert.match(address, /^127\.0\.0\.1:\d+$/)
|
||||
const base = `http://${address}`
|
||||
const request = async (url, options) => {
|
||||
const response = await fetch(`${base}${url}`, { ...options, signal: AbortSignal.timeout(3000) })
|
||||
return { status: response.status, headers: response.headers, body: await response.text() }
|
||||
}
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
try { await request('/login'); break } catch (error) {
|
||||
if (attempt >= 19) throw error
|
||||
await delay(250)
|
||||
}
|
||||
}
|
||||
|
||||
await t.test('runtime config is never stored, including conditional responses', async () => {
|
||||
const response = await request('/config.js?deployment=next')
|
||||
assert.equal(response.status, 200)
|
||||
assert.equal(response.body, fixtures['config.js'])
|
||||
assert.equal(response.headers.get('cache-control'), 'no-store')
|
||||
assert.equal(response.headers.get('x-content-type-options'), 'nosniff')
|
||||
assert.ok(response.headers.get('etag'))
|
||||
const conditional = await request('/config.js', { headers: { 'If-None-Match': response.headers.get('etag') } })
|
||||
assert.equal(conditional.status, 304)
|
||||
assert.equal(conditional.headers.get('cache-control'), 'no-store')
|
||||
})
|
||||
|
||||
await t.test('versioned icons and hashed build assets retain immutable caching', async () => {
|
||||
for (const name of ['tdesign-icons/0.4.1/fonts/index.js', 'assets/cache-test-012345.js']) {
|
||||
const response = await request(`/${name}`)
|
||||
assert.equal(response.status, 200)
|
||||
assert.equal(response.body, fixtures[name])
|
||||
assert.equal(response.headers.get('cache-control'), 'public, max-age=31536000, immutable')
|
||||
assert.equal(response.headers.get('x-content-type-options'), 'nosniff')
|
||||
assert.ok(response.headers.get('etag'))
|
||||
const conditional = await request(`/${name}`, { headers: { 'If-None-Match': response.headers.get('etag') } })
|
||||
assert.equal(conditional.status, 304)
|
||||
assert.equal(conditional.headers.get('cache-control'), 'public, max-age=31536000, immutable')
|
||||
}
|
||||
})
|
||||
|
||||
await t.test('missing versioned assets return an uncacheable 404, never the SPA', async () => {
|
||||
for (const url of ['/tdesign-icons/0.4.1/fonts/missing.js', '/tdesign-icons/99.0.0/fonts/index.js', '/assets/missing.js']) {
|
||||
const response = await request(url)
|
||||
assert.equal(response.status, 404)
|
||||
assert.equal(response.headers.get('cache-control'), 'no-store')
|
||||
assert.ok(!response.body.includes('main cache fixture'))
|
||||
}
|
||||
})
|
||||
|
||||
await t.test('SPA and embed entry points still require revalidation', async () => {
|
||||
for (const [url, fixture] of [['/login', 'index.html'], ['/embed/cache-test', 'embed.html']]) {
|
||||
const response = await request(url)
|
||||
assert.equal(response.status, 200)
|
||||
assert.equal(response.body, fixtures[fixture])
|
||||
assert.equal(response.headers.get('cache-control'), 'no-cache, must-revalidate')
|
||||
}
|
||||
})
|
||||
|
||||
await t.test('missing runtime configuration is an uncacheable 404', async () => {
|
||||
docker('exec', container, 'rm', '/usr/share/nginx/html/config.js')
|
||||
const response = await request('/config.js')
|
||||
assert.equal(response.status, 404)
|
||||
assert.equal(response.headers.get('cache-control'), 'no-store')
|
||||
assert.ok(!response.body.includes('main cache fixture'))
|
||||
})
|
||||
} finally {
|
||||
try {
|
||||
if (created) docker('rm', '--force', container)
|
||||
} finally {
|
||||
fs.rmSync(temp, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,146 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
const fs = require('node:fs')
|
||||
const os = require('node:os')
|
||||
const path = require('node:path')
|
||||
const test = require('node:test')
|
||||
const { setTimeout: delay } = require('node:timers/promises')
|
||||
|
||||
const frontend = path.resolve(__dirname, '../frontend')
|
||||
const source = fs.readFileSync(path.join(frontend, 'nginx.conf'), 'utf8')
|
||||
|
||||
test('ingress access format exposes only approved fields and preserves nginx variables', () => {
|
||||
const format = source.match(/log_format ingress_timing escape=json\s+([\s\S]*?);/)
|
||||
assert.ok(format, 'escaped timing format exists')
|
||||
const variables = [...new Set(format[1].match(/\$[a-z][a-z0-9_]*/g))].sort()
|
||||
assert.deepEqual(variables, [
|
||||
'$ingress_cf_ray', '$ingress_route_class', '$ingress_upstream_connect_time',
|
||||
'$ingress_upstream_header_time', '$ingress_upstream_response_time', '$request_time', '$status',
|
||||
'$time_iso8601',
|
||||
].sort())
|
||||
const active = source.split('\n').filter(line => !line.trim().startsWith('#')).join('\n')
|
||||
assert.deepEqual([...active.matchAll(/access_log\s+([^;]+);/g)].map(match => match[1]), [
|
||||
'/var/log/nginx/access.log ingress_timing',
|
||||
])
|
||||
const entrypoint = fs.readFileSync(path.join(frontend, 'docker-entrypoint.sh'), 'utf8')
|
||||
const substituted = entrypoint.match(/envsubst '([^']+)'/)
|
||||
assert.ok(substituted)
|
||||
for (const variable of variables) assert.ok(!substituted[1].includes(variable))
|
||||
})
|
||||
|
||||
const dockerContext = process.env.NGINX_TEST_DOCKER_CONTEXT
|
||||
const docker = (...args) => execFileSync('docker', [
|
||||
...(dockerContext ? ['--context', dockerContext] : []), ...args,
|
||||
], { encoding: 'utf8', timeout: 30000, stdio: ['ignore', 'pipe', 'pipe'] }).trim()
|
||||
|
||||
// Enable only after the coordinated runtime freeze. No production network is used.
|
||||
test('isolated nginx classifies requests and records timings without request secrets', {
|
||||
skip: process.env.NGINX_TEST_RUN_INGRESS !== '1', timeout: 90000,
|
||||
}, async () => {
|
||||
const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'weknora-ingress-test-'))
|
||||
const container = `weknora-ingress-test-${process.pid}-${Date.now()}`
|
||||
const seedContainer = `${container}-seed`
|
||||
const volume = `${container}-fixtures`
|
||||
const image = process.env.NGINX_TEST_IMAGE || fs.readFileSync(path.join(frontend, 'Dockerfile'), 'utf8').match(/^FROM (\S+)/m)[1]
|
||||
let created = false
|
||||
let seedCreated = false
|
||||
let volumeCreated = false
|
||||
try {
|
||||
const site = path.join(temp, 'html')
|
||||
fs.mkdirSync(site)
|
||||
for (const [filename, body] of Object.entries({
|
||||
'index.html': '<html>timing fixture</html>',
|
||||
'embed.html': '<html>embed fixture</html>',
|
||||
'config.js': 'window.fixture = true;',
|
||||
'release.json': '{"release":"fixture"}',
|
||||
})) fs.writeFileSync(path.join(site, filename), body)
|
||||
|
||||
const replacements = { APP_SCHEME: 'http', APP_HOST: '127.0.0.1', APP_PORT: '81', MAX_FILE_SIZE: '50M', MAX_SKILL_BUNDLE_SIZE: '256M' }
|
||||
const config = source.replace(/\$\{(APP_SCHEME|APP_HOST|APP_PORT|MAX_FILE_SIZE|MAX_SKILL_BUNDLE_SIZE)\}/g, (_, key) => replacements[key])
|
||||
.replaceAll('/usr/share/nginx/html', '/fixtures/html')
|
||||
.replaceAll('/etc/nginx/api-proxy.conf', '/fixtures/api-proxy.conf')
|
||||
.replaceAll('/var/log/nginx/access.log ingress_timing', '/proc/1/fd/1 ingress_timing')
|
||||
.replaceAll('/var/log/nginx/error.log warn', '/proc/1/fd/2 warn')
|
||||
fs.writeFileSync(path.join(temp, 'nginx.conf'),
|
||||
'worker_processes 1; error_log /dev/stderr warn; pid /var/run/nginx.pid;\n'
|
||||
+ 'events { worker_connections 128; }\nhttp { include /etc/nginx/mime.types;\n'
|
||||
+ config + '\nserver { listen 127.0.0.1:81; access_log off; location / { return 204; } }\n}\n')
|
||||
fs.copyFileSync(path.join(frontend, 'nginx-api-proxy.conf'), path.join(temp, 'api-proxy.conf'))
|
||||
// Stage through Docker copy so a remote context needs no host-path sharing.
|
||||
docker('volume', 'create', volume)
|
||||
volumeCreated = true
|
||||
docker('create', '--pull', 'never', '--read-only', '--network', 'none', '--name', seedContainer,
|
||||
'--mount', `type=volume,src=${volume},dst=/fixtures`, '--entrypoint', 'true', image)
|
||||
seedCreated = true
|
||||
docker('cp', `${temp}/.`, `${seedContainer}:/fixtures`)
|
||||
docker('rm', seedContainer)
|
||||
seedCreated = false
|
||||
docker('create', '--rm', '--pull', 'never', '--read-only', '--network', 'none', '--name', container,
|
||||
'--tmpfs', '/var/cache/nginx:rw,noexec,nosuid,size=16m',
|
||||
'--tmpfs', '/var/run:rw,noexec,nosuid,size=1m',
|
||||
'--mount', `type=volume,src=${volume},dst=/fixtures,readonly`,
|
||||
'--entrypoint', 'nginx', image, '-c', '/fixtures/nginx.conf', '-g', 'daemon off;')
|
||||
created = true
|
||||
docker('start', container)
|
||||
docker('exec', container, 'nginx', '-t', '-c', '/fixtures/nginx.conf')
|
||||
const fetchFixture = (url, extra = []) => docker('exec', container, 'wget', '-q', '-O', '/dev/null', ...extra, `http://127.0.0.1${url}`)
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
try { fetchFixture('/release.json'); break } catch (error) {
|
||||
if (attempt >= 19) throw error
|
||||
await delay(250)
|
||||
}
|
||||
}
|
||||
|
||||
const secret = 'ingress-secret-sentinel'
|
||||
const cases = [
|
||||
['/admin', 'shell', 200], ['/platform/workbench/content', 'shell', 200],
|
||||
['/api/v1/auth/me', 'auth_me', 204], ['/api/v1/access/me', 'access_me', 204],
|
||||
['/api/v1/system/capabilities', 'capabilities', 204],
|
||||
['/api/v1/workbench/resources/training-maps', 'resource_list', 204],
|
||||
[`/api/v1/workbench/resources/training-maps/${secret}`, 'resource_detail', 204],
|
||||
[`/api/v1/workbench/resources/training-maps/${secret}/jobs`, 'api', 204],
|
||||
[`/r/${secret}`, 'resource_link', 204], [`/files/${secret}`, 'file', 200],
|
||||
[`/embed/${secret}`, 'embed', 200], ['/config.js', 'config', 200],
|
||||
['/release.json', 'release', 200], [`/unknown/${secret}`, 'other', 200],
|
||||
]
|
||||
const rays = cases.map((_, index) => `${(index + 1).toString(16).padStart(16, '0')}-AMS`)
|
||||
for (const [index, [url]] of cases.entries()) fetchFixture(`${url}?token=${secret}`, [
|
||||
`--header=CF-Ray: ${rays[index]}`, `--header=Authorization: Bearer ${secret}`,
|
||||
`--header=Cookie: session=${secret}`, `--header=Referer: https://example.test/${secret}`,
|
||||
`--header=User-Agent: ${secret}`,
|
||||
])
|
||||
fetchFixture('/api/body', [`--header=CF-Ray: ${secret}`, `--post-data=${secret}`])
|
||||
const lines = docker('logs', container).split('\n').filter(line => line.startsWith('{"time":'))
|
||||
assert.ok(lines.length >= cases.length + 2)
|
||||
assert.ok(!lines.join('\n').includes(secret), 'no path, query, header, or body secret appears in access records')
|
||||
const records = lines.map(line => JSON.parse(line))
|
||||
const fields = ['time', 'route', 'status', 'request_time', 'upstream_connect_time', 'upstream_header_time', 'upstream_response_time', 'cf_ray'].sort()
|
||||
for (const [index, [, route, status]] of cases.entries()) {
|
||||
const matches = records.filter(record => record.cf_ray === rays[index])
|
||||
assert.equal(matches.length, 1)
|
||||
const record = matches[0]
|
||||
assert.deepEqual(Object.keys(record).sort(), fields)
|
||||
assert.equal(record.route, route)
|
||||
assert.equal(record.status, status, `${route} ${record.cf_ray}`)
|
||||
assert.match(record.request_time, /^\d+\.\d{3}$/)
|
||||
for (const field of ['upstream_connect_time', 'upstream_header_time', 'upstream_response_time']) {
|
||||
assert.match(record[field], status === 204 ? /^\d+\.\d{3}$/ : /^-$/)
|
||||
}
|
||||
}
|
||||
assert.ok(records.some(record => record.route === 'api' && record.cf_ray === '-' && record.status === 204))
|
||||
} finally {
|
||||
try {
|
||||
if (created) docker('rm', '--force', container)
|
||||
} finally {
|
||||
try {
|
||||
if (seedCreated) docker('rm', '--force', seedContainer)
|
||||
} finally {
|
||||
try {
|
||||
if (volumeCreated) docker('volume', 'rm', volume)
|
||||
} finally {
|
||||
fs.rmSync(temp, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,62 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
const fs = require('node:fs')
|
||||
const os = require('node:os')
|
||||
const path = require('node:path')
|
||||
const vm = require('node:vm')
|
||||
const test = require('node:test')
|
||||
|
||||
const frontend = path.resolve(__dirname, '../frontend')
|
||||
const docker = (...args) => execFileSync('docker', [
|
||||
...(process.env.NGINX_TEST_DOCKER_CONTEXT ? ['--context', process.env.NGINX_TEST_DOCKER_CONTEXT] : []), ...args,
|
||||
], { encoding: 'utf8', timeout: 30000, stdio: ['ignore', 'pipe', 'pipe'] }).trim()
|
||||
|
||||
test('startup embeds current validated config in both entry points on every run', { timeout: 90000 }, async () => {
|
||||
const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'weknora-runtime-config-'))
|
||||
const name = `weknora-runtime-config-${process.pid}-${Date.now()}`
|
||||
const image = process.env.NGINX_TEST_IMAGE || fs.readFileSync(path.join(frontend, 'Dockerfile'), 'utf8').match(/^FROM (\S+)/m)[1]
|
||||
let created = false
|
||||
try {
|
||||
// The stand-in nginx exits, so the real startup script can be run repeatedly.
|
||||
fs.writeFileSync(path.join(temp, 'nginx'), '#!/bin/sh\nexit 0\n', { mode: 0o755 })
|
||||
docker('create', '--rm', '--pull', 'never', '--network', 'none', '--name', name,
|
||||
'--entrypoint', 'sh', image, '-c', 'while :; do sleep 30; done')
|
||||
created = true
|
||||
docker('start', name)
|
||||
docker('exec', name, 'mkdir', '-p', '/usr/share/nginx/runtime-templates', '/etc/nginx/templates', '/fixture-bin')
|
||||
for (const page of ['index.html', 'embed.html']) docker('cp', path.join(frontend, page), `${name}:/usr/share/nginx/runtime-templates/${page}`)
|
||||
docker('cp', path.join(frontend, 'nginx.conf'), `${name}:/etc/nginx/templates/default.conf.template`)
|
||||
docker('cp', path.join(frontend, 'docker-entrypoint.sh'), `${name}:/fixture-entrypoint.sh`)
|
||||
docker('cp', path.join(temp, 'nginx'), `${name}:/fixture-bin/nginx`)
|
||||
const run = env => docker('exec', '-e', 'PATH=/fixture-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin',
|
||||
...Object.entries(env).flatMap(([key, value]) => ['-e', `${key}=${value}`]), name, 'sh', '/fixture-entrypoint.sh')
|
||||
const readConfig = (source) => {
|
||||
const matches = [...source.matchAll(/<script data-runtime-config>([\s\S]*?)<\/script>/g)]
|
||||
assert.equal(matches.length, 1)
|
||||
assert.ok(!source.includes('src="/config.js"'))
|
||||
const context = { window: {} }
|
||||
vm.runInNewContext(matches[0][1], context, { timeout: 500 })
|
||||
return JSON.parse(JSON.stringify(context.window.__RUNTIME_CONFIG__))
|
||||
}
|
||||
for (const [env, expected] of [
|
||||
[{ DEFAULT_LOCALE: 'zh-CN', MAX_FILE_SIZE_MB: '50', MAX_SKILL_BUNDLE_SIZE_MB: '300' },
|
||||
{ DEFAULT_LOCALE: 'zh-CN', MAX_FILE_SIZE_MB: 50, MAX_SKILL_BUNDLE_SIZE_MB: 300 }],
|
||||
[{ DEFAULT_LOCALE: 'en-US', MAX_FILE_SIZE_MB: '99999', MAX_SKILL_BUNDLE_SIZE_MB: '600' },
|
||||
{ DEFAULT_LOCALE: 'en-US', MAX_FILE_SIZE_MB: 20480, MAX_SKILL_BUNDLE_SIZE_MB: 512 }],
|
||||
[{ DEFAULT_LOCALE: '</script><script>alert(1)</script>', MAX_FILE_SIZE_MB: 'invalid', MAX_SKILL_BUNDLE_SIZE_MB: '1;alert(1)' },
|
||||
{ DEFAULT_LOCALE: '', MAX_FILE_SIZE_MB: 50, MAX_SKILL_BUNDLE_SIZE_MB: 256 }],
|
||||
]) {
|
||||
run(env)
|
||||
for (const page of ['index.html', 'embed.html']) {
|
||||
const body = docker('exec', name, 'cat', `/usr/share/nginx/html/${page}`)
|
||||
assert.deepEqual(readConfig(body), expected)
|
||||
assert.ok(body.includes('type="module"'))
|
||||
}
|
||||
const compatibility = { window: {} }
|
||||
vm.runInNewContext(docker('exec', name, 'cat', '/usr/share/nginx/html/config.js'), compatibility, { timeout: 500 })
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(compatibility.window.__RUNTIME_CONFIG__)), expected)
|
||||
}
|
||||
} finally {
|
||||
try { if (created) docker('rm', '--force', name) } finally { fs.rmSync(temp, { recursive: true, force: true }) }
|
||||
}
|
||||
})
|
||||
Executable
+90
@@ -0,0 +1,90 @@
|
||||
#!/bin/bash
|
||||
# 统一的版本信息获取脚本
|
||||
# 支持本地构建和CI构建环境
|
||||
|
||||
# 设置默认值
|
||||
VERSION="unknown"
|
||||
EDITION="${EDITION:-standard}"
|
||||
COMMIT_ID="unknown"
|
||||
BUILD_TIME="unknown"
|
||||
GO_VERSION="unknown"
|
||||
|
||||
# 获取版本号
|
||||
if [ -f "VERSION" ]; then
|
||||
VERSION=$(cat VERSION | tr -d '\n\r')
|
||||
fi
|
||||
|
||||
# 获取commit ID
|
||||
if [ -n "$GITHUB_SHA" ]; then
|
||||
# GitHub Actions环境
|
||||
COMMIT_ID="${GITHUB_SHA:0:7}"
|
||||
elif command -v git >/dev/null 2>&1; then
|
||||
# 本地环境
|
||||
COMMIT_ID=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||||
fi
|
||||
|
||||
# 获取构建时间
|
||||
if [ -n "$GITHUB_ACTIONS" ]; then
|
||||
# GitHub Actions环境,使用标准时间格式
|
||||
BUILD_TIME=$(date -u '+%Y-%m-%d %H:%M:%S UTC')
|
||||
else
|
||||
# 本地环境
|
||||
BUILD_TIME=$(date -u '+%Y-%m-%d %H:%M:%S UTC')
|
||||
fi
|
||||
|
||||
# 获取Go版本
|
||||
if command -v go >/dev/null 2>&1; then
|
||||
GO_VERSION=$(go version 2>/dev/null || echo "unknown")
|
||||
fi
|
||||
|
||||
# 根据参数输出不同格式
|
||||
case "${1:-env}" in
|
||||
"env")
|
||||
# 输出环境变量格式,对包含空格的值进行转义
|
||||
echo "VERSION=$VERSION"
|
||||
echo "EDITION=$EDITION"
|
||||
echo "COMMIT_ID=$COMMIT_ID"
|
||||
echo "BUILD_TIME=\"$BUILD_TIME\""
|
||||
echo "GO_VERSION=\"$GO_VERSION\""
|
||||
;;
|
||||
"json")
|
||||
# 输出JSON格式
|
||||
cat << EOF
|
||||
{
|
||||
"version": "$VERSION",
|
||||
"edition": "$EDITION",
|
||||
"commit_id": "$COMMIT_ID",
|
||||
"build_time": "$BUILD_TIME",
|
||||
"go_version": "$GO_VERSION"
|
||||
}
|
||||
EOF
|
||||
;;
|
||||
"docker-args")
|
||||
# 输出Docker构建参数格式
|
||||
echo "--build-arg VERSION_ARG=$VERSION"
|
||||
echo "--build-arg COMMIT_ID_ARG=$COMMIT_ID"
|
||||
echo "--build-arg BUILD_TIME_ARG=$BUILD_TIME"
|
||||
echo "--build-arg GO_VERSION_ARG=$GO_VERSION"
|
||||
;;
|
||||
"ldflags")
|
||||
# 输出Go ldflags格式
|
||||
echo "-X 'github.com/Tencent/WeKnora/internal/handler.Version=$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.Edition=$EDITION' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$GO_VERSION'"
|
||||
;;
|
||||
"info")
|
||||
# 输出信息格式
|
||||
echo "版本信息: $VERSION"
|
||||
echo "版本类型: $EDITION"
|
||||
echo "Commit ID: $COMMIT_ID"
|
||||
echo "构建时间: $BUILD_TIME"
|
||||
echo "Go版本: $GO_VERSION"
|
||||
;;
|
||||
*)
|
||||
echo "用法: $0 [env|json|docker-args|ldflags|info]"
|
||||
echo " env - 输出环境变量格式 (默认)"
|
||||
echo " json - 输出JSON格式"
|
||||
echo " docker-args - 输出Docker构建参数格式"
|
||||
echo " ldflags - 输出Go ldflags格式"
|
||||
echo " info - 输出信息格式"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Executable
+234
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared helpers for WeKnora git hooks (mirrors .github/workflows and PR checklist).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
|
||||
hook_skip() {
|
||||
[[ "${SKIP_HOOKS:-}" == "1" ]]
|
||||
}
|
||||
|
||||
log_step() {
|
||||
printf '→ %s\n' "$*"
|
||||
}
|
||||
|
||||
log_ok() {
|
||||
printf '✓ %s\n' "$*"
|
||||
}
|
||||
|
||||
log_fail() {
|
||||
printf '✗ %s\n' "$*" >&2
|
||||
}
|
||||
|
||||
# File types we expect hand-edited in PRs (skip sample-data / generated blobs).
|
||||
WHITESPACE_PATHSPECS=(
|
||||
'*.go'
|
||||
'*.ts' '*.tsx' '*.vue' '*.js' '*.mjs'
|
||||
'*.yaml' '*.yml' '*.json'
|
||||
'*.sh' '*.sql'
|
||||
'Makefile'
|
||||
'frontend/package.json'
|
||||
'frontend/package-lock.json'
|
||||
)
|
||||
|
||||
# Best-effort fetch so merge-base matches CI (origin/main).
|
||||
ensure_origin_main() {
|
||||
git -C "$ROOT" fetch origin main --quiet 2>/dev/null || true
|
||||
}
|
||||
|
||||
merge_base() {
|
||||
ensure_origin_main
|
||||
local base
|
||||
base="$(git -C "$ROOT" merge-base HEAD origin/main 2>/dev/null || true)"
|
||||
if [[ -n "$base" ]]; then
|
||||
printf '%s\n' "$base"
|
||||
return
|
||||
fi
|
||||
base="$(git -C "$ROOT" merge-base HEAD main 2>/dev/null || true)"
|
||||
if [[ -n "$base" ]]; then
|
||||
printf '%s\n' "$base"
|
||||
return
|
||||
fi
|
||||
git -C "$ROOT" rev-parse HEAD~1
|
||||
}
|
||||
|
||||
# Prefer the open PR's base SHA (matches GitHub CI diff) when gh is available.
|
||||
pr_diff_base_ref() {
|
||||
ensure_origin_main
|
||||
if command -v gh >/dev/null 2>&1; then
|
||||
local base_sha
|
||||
base_sha="$(gh pr view --json baseRefOid --jq .baseRefOid 2>/dev/null || true)"
|
||||
if [[ -n "$base_sha" && "$base_sha" != "null" ]]; then
|
||||
printf '%s\n' "$base_sha"
|
||||
return
|
||||
fi
|
||||
fi
|
||||
if git -C "$ROOT" rev-parse --verify origin/main >/dev/null 2>&1; then
|
||||
printf '%s\n' "origin/main"
|
||||
return
|
||||
fi
|
||||
printf '%s\n' "main"
|
||||
}
|
||||
|
||||
check_whitespace() {
|
||||
log_step "git diff --check (source files)"
|
||||
if [[ "${1:-}" == "--cached" ]]; then
|
||||
if git -C "$ROOT" diff --cached --quiet -- "${WHITESPACE_PATHSPECS[@]}"; then
|
||||
log_ok "no staged source changes to check"
|
||||
return 0
|
||||
fi
|
||||
git -C "$ROOT" diff --check --cached -- "${WHITESPACE_PATHSPECS[@]}"
|
||||
else
|
||||
if git -C "$ROOT" diff --quiet "$@" -- "${WHITESPACE_PATHSPECS[@]}"; then
|
||||
log_ok "no changed source files to check"
|
||||
return 0
|
||||
fi
|
||||
git -C "$ROOT" diff --check "$@" -- "${WHITESPACE_PATHSPECS[@]}"
|
||||
fi
|
||||
log_ok "no whitespace errors"
|
||||
}
|
||||
|
||||
check_gofmt_files() {
|
||||
local mode="$1" # check | write
|
||||
shift
|
||||
local -a files=("$@")
|
||||
if [[ ${#files[@]} -gt 0 ]]; then
|
||||
local -a existing=()
|
||||
local f rel
|
||||
for f in "${files[@]}"; do
|
||||
if [[ -f "$f" ]]; then
|
||||
existing+=("$f")
|
||||
continue
|
||||
fi
|
||||
rel="${f#"$ROOT"/}"
|
||||
if [[ -f "$ROOT/$rel" ]]; then
|
||||
existing+=("$ROOT/$rel")
|
||||
fi
|
||||
done
|
||||
files=("${existing[@]}")
|
||||
fi
|
||||
[[ ${#files[@]} -gt 0 ]] || return 0
|
||||
|
||||
local unformatted
|
||||
if [[ "$mode" == "write" ]]; then
|
||||
log_step "gofmt (auto-format staged Go files)"
|
||||
gofmt -w "${files[@]}"
|
||||
# Re-stage formatted files.
|
||||
git -C "$ROOT" add -- "${files[@]}"
|
||||
log_ok "gofmt applied"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_step "gofmt (check)"
|
||||
unformatted="$(gofmt -l "${files[@]}")"
|
||||
if [[ -n "$unformatted" ]]; then
|
||||
log_fail "the following Go files are not gofmt-formatted:"
|
||||
printf '%s\n' "$unformatted" >&2
|
||||
log_fail "run: gofmt -w <files> (or commit again after pre-commit auto-format)"
|
||||
return 1
|
||||
fi
|
||||
log_ok "gofmt"
|
||||
}
|
||||
|
||||
collect_go_packages_from_files() {
|
||||
local -a files=("$@")
|
||||
local f dir pkg module root_module
|
||||
local -a pkgs=()
|
||||
# Nested modules (docs/poc, cli, client) are not testable from the root
|
||||
# module, so only keep packages that belong to it.
|
||||
root_module="$(cd "$ROOT" && go list -m 2>/dev/null || true)"
|
||||
for f in "${files[@]}"; do
|
||||
[[ "$f" == *.go ]] || continue
|
||||
dir="$(dirname "$f")"
|
||||
module="$(cd "$ROOT/$dir" && go list -m 2>/dev/null)" || continue
|
||||
[[ "$module" == "$root_module" ]] || continue
|
||||
pkg="$(cd "$ROOT/$dir" && go list . 2>/dev/null)" || continue
|
||||
pkgs+=("$pkg")
|
||||
done
|
||||
if [[ ${#pkgs[@]} -eq 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
printf '%s\n' "${pkgs[@]}" | sort -u
|
||||
}
|
||||
|
||||
run_golangci_if_available() {
|
||||
local from_rev="$1"
|
||||
if ! command -v golangci-lint >/dev/null 2>&1; then
|
||||
log_step "golangci-lint not installed; skip (install: https://golangci-lint.run)"
|
||||
return 0
|
||||
fi
|
||||
log_step "golangci-lint run --new-from-rev=$from_rev ./..."
|
||||
(
|
||||
cd "$ROOT"
|
||||
golangci-lint run --new-from-rev="$from_rev" ./...
|
||||
)
|
||||
log_ok "golangci-lint"
|
||||
}
|
||||
|
||||
run_full_app_vet() {
|
||||
# Mirrors .github/workflows/app.yml: vet every app package (excl. docreader).
|
||||
log_step "go vet (all app packages, like CI)"
|
||||
(
|
||||
cd "$ROOT"
|
||||
local -a pkgs=()
|
||||
while IFS= read -r line; do
|
||||
[[ -n "$line" ]] && pkgs+=("$line")
|
||||
done < <(go list ./... | grep -v '/docreader/' || true)
|
||||
go vet "${pkgs[@]}"
|
||||
)
|
||||
log_ok "go vet"
|
||||
}
|
||||
|
||||
run_go_test_packages() {
|
||||
local -a pkgs=("$@")
|
||||
if [[ ${#pkgs[@]} -eq 0 ]]; then
|
||||
log_step "no changed Go packages; skip go test"
|
||||
return 0
|
||||
fi
|
||||
if [[ "${HOOK_SKIP_TEST:-}" == "1" ]]; then
|
||||
log_step "HOOK_SKIP_TEST=1; skip go test"
|
||||
return 0
|
||||
fi
|
||||
log_step "go test (${#pkgs[@]} changed package(s))"
|
||||
(
|
||||
cd "$ROOT"
|
||||
go test -count=1 "${pkgs[@]}"
|
||||
)
|
||||
log_ok "go test"
|
||||
}
|
||||
|
||||
run_app_vet_test() {
|
||||
run_full_app_vet
|
||||
run_go_test_packages "$@"
|
||||
}
|
||||
|
||||
run_cli_checks() {
|
||||
log_step "cli: go vet && go test"
|
||||
(
|
||||
cd "$ROOT/cli"
|
||||
go vet ./...
|
||||
if [[ "${HOOK_SKIP_TEST:-}" == "1" ]]; then
|
||||
log_step "HOOK_SKIP_TEST=1; skip cli go test"
|
||||
else
|
||||
go test -count=1 ./...
|
||||
fi
|
||||
)
|
||||
log_ok "cli checks"
|
||||
}
|
||||
|
||||
run_frontend_checks() {
|
||||
log_step "frontend: verify (test, type-check, build)"
|
||||
"$ROOT/scripts/verify_frontend_pr.sh"
|
||||
log_ok "frontend checks"
|
||||
}
|
||||
|
||||
paths_match_prefix() {
|
||||
local prefix="$1"
|
||||
shift
|
||||
local p
|
||||
for p in "$@"; do
|
||||
[[ "$p" == "$prefix"* ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fast checks on staged files (PR checklist + gofmt). Mirrors app.yml formatting gate.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
# shellcheck source=scripts/git-hooks/common.sh
|
||||
source "$ROOT/scripts/git-hooks/common.sh"
|
||||
|
||||
if hook_skip; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
|
||||
log_step "pre-commit"
|
||||
|
||||
check_whitespace --cached
|
||||
|
||||
staged_go=()
|
||||
while IFS= read -r line; do
|
||||
[[ -n "$line" ]] && staged_go+=("$line")
|
||||
done < <(git diff --cached --name-only --diff-filter=ACMR -- '*.go' ':!cli/**' || true)
|
||||
|
||||
if [[ ${#staged_go[@]} -gt 0 ]]; then
|
||||
go_files=()
|
||||
for f in "${staged_go[@]}"; do
|
||||
go_files+=("$ROOT/$f")
|
||||
done
|
||||
check_gofmt_files write "${go_files[@]}"
|
||||
|
||||
if command -v golangci-lint >/dev/null 2>&1; then
|
||||
# Lint only what differs from HEAD (staged + unstaged in those packages).
|
||||
run_golangci_if_available HEAD
|
||||
fi
|
||||
fi
|
||||
|
||||
log_ok "pre-commit passed"
|
||||
Executable
+98
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env bash
|
||||
# Pre-push checks aligned with CI (.github/workflows/app.yml, frontend.yml, cli.yml).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
# shellcheck source=scripts/git-hooks/common.sh
|
||||
source "$ROOT/scripts/git-hooks/common.sh"
|
||||
|
||||
if hook_skip; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
|
||||
remote="${1:-origin}"
|
||||
while read -r _local_ref local_sha remote_ref remote_sha; do
|
||||
if [[ "$local_sha" =~ ^0+$ ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
log_step "pre-push ($remote ${remote_ref#refs/heads/})"
|
||||
|
||||
ensure_origin_main
|
||||
pr_ref="$(pr_diff_base_ref)"
|
||||
pr_tip="$local_sha"
|
||||
pr_diff="${pr_ref}...${pr_tip}"
|
||||
lint_from_rev="$pr_ref"
|
||||
|
||||
if [[ "$remote_sha" =~ ^0+$ ]]; then
|
||||
push_start="$(git -C "$ROOT" merge-base "$pr_ref" "$local_sha")"
|
||||
push_range="${push_start}..${local_sha}"
|
||||
log_step "new branch; PR diff ${pr_ref}...${local_sha:0:12}"
|
||||
else
|
||||
if [[ "$remote_sha" == "$local_sha" ]]; then
|
||||
log_ok "pre-push: nothing new to push"
|
||||
continue
|
||||
fi
|
||||
push_range="${remote_sha}..${local_sha}"
|
||||
log_step "pushing ${remote_sha:0:12}..${local_sha:0:12}; PR diff ${pr_ref}...${local_sha:0:12}"
|
||||
fi
|
||||
|
||||
check_whitespace "$push_range"
|
||||
|
||||
pr_changed=()
|
||||
while IFS= read -r line; do
|
||||
[[ -n "$line" ]] && pr_changed+=("$line")
|
||||
done < <(git diff --name-only --diff-filter=ACMR "$pr_diff" || true)
|
||||
|
||||
if [[ ${#pr_changed[@]} -eq 0 ]]; then
|
||||
log_ok "pre-push: no PR diff changes"
|
||||
continue
|
||||
fi
|
||||
|
||||
pr_changed_go=()
|
||||
while IFS= read -r line; do
|
||||
[[ -n "$line" ]] && pr_changed_go+=("$line")
|
||||
done < <(git diff --name-only --diff-filter=ACMR "$pr_diff" -- '*.go' ':!cli/**' || true)
|
||||
|
||||
if [[ ${#pr_changed_go[@]} -gt 0 ]]; then
|
||||
go_files=()
|
||||
for f in "${pr_changed_go[@]}"; do
|
||||
go_files+=("$ROOT/$f")
|
||||
done
|
||||
check_gofmt_files check "${go_files[@]}"
|
||||
|
||||
run_golangci_if_available "$lint_from_rev"
|
||||
|
||||
pkgs=()
|
||||
while IFS= read -r line; do
|
||||
[[ -n "$line" ]] && pkgs+=("$line")
|
||||
done < <(collect_go_packages_from_files "${pr_changed_go[@]}")
|
||||
|
||||
filtered=()
|
||||
for pkg in "${pkgs[@]}"; do
|
||||
[[ "$pkg" == *"/docreader/"* ]] && continue
|
||||
filtered+=("$pkg")
|
||||
done
|
||||
|
||||
run_full_app_vet
|
||||
run_go_test_packages "${filtered[@]}"
|
||||
|
||||
log_step "go build ./cmd/server"
|
||||
go build ./cmd/server
|
||||
log_ok "go build ./cmd/server"
|
||||
fi
|
||||
|
||||
if paths_match_prefix "cli/" "${pr_changed[@]}"; then
|
||||
run_cli_checks
|
||||
fi
|
||||
if paths_match_prefix "frontend/" "${pr_changed[@]}"; then
|
||||
run_frontend_checks
|
||||
fi
|
||||
|
||||
log_ok "pre-push passed"
|
||||
done
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/requestbudget"
|
||||
)
|
||||
|
||||
func probeSendLimit(requested *int, environment string, envSet bool) (int, error) {
|
||||
limit := requestbudget.GraphMaxSends
|
||||
if requested != nil {
|
||||
limit = *requested
|
||||
}
|
||||
if limit < 1 || limit > requestbudget.GraphMaxSends {
|
||||
return 0, fmt.Errorf("invalid probe HTTP send cap")
|
||||
}
|
||||
if envSet && environment != strconv.Itoa(limit) {
|
||||
return 0, fmt.Errorf("probe HTTP send cap differs between request and environment")
|
||||
}
|
||||
return limit, nil
|
||||
}
|
||||
|
||||
// This tighter probe allowance delegates model, phase and deadline enforcement
|
||||
// to the existing graph guard; worker construction and defaults are untouched.
|
||||
type probeExecutionBudget struct {
|
||||
*requestbudget.GraphExecution
|
||||
mu sync.Mutex
|
||||
limit int
|
||||
stopped error
|
||||
cancel context.CancelCauseFunc
|
||||
}
|
||||
|
||||
func bindProbeExecution(ctx context.Context, key, model string, limit int) (context.Context, *probeExecutionBudget, context.CancelFunc, error) {
|
||||
if _, err := probeSendLimit(&limit, "", false); err != nil {
|
||||
return ctx, nil, func() {}, err
|
||||
}
|
||||
ctx, base, baseCancel, err := requestbudget.BindGraphExecution(ctx, key, model)
|
||||
if err != nil {
|
||||
return ctx, nil, baseCancel, err
|
||||
}
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
guard := &probeExecutionBudget{GraphExecution: base, limit: limit, cancel: cancel}
|
||||
return requestbudget.WithGuard(ctx, guard), guard, func() { cancel(nil); baseCancel() }, nil
|
||||
}
|
||||
|
||||
func (g *probeExecutionBudget) Check(ctx context.Context) error {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.stopped != nil {
|
||||
return g.stopped
|
||||
}
|
||||
return g.GraphExecution.Check(ctx)
|
||||
}
|
||||
|
||||
func (g *probeExecutionBudget) Reserve(ctx context.Context, model, phase string) error {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.stopped != nil {
|
||||
return g.stopped
|
||||
}
|
||||
if err := g.GraphExecution.Check(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if g.Reservations() >= g.limit {
|
||||
g.stopped = fmt.Errorf("%w: probe HTTP send cap reached", requestbudget.ErrStopped)
|
||||
g.cancel(g.stopped)
|
||||
return g.stopped
|
||||
}
|
||||
return g.GraphExecution.Reserve(ctx, model, phase)
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/requestbudget"
|
||||
)
|
||||
|
||||
func TestProbeSendLimit(t *testing.T) {
|
||||
one, two, three, zero, four := 1, 2, 3, 0, 4
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
requested *int
|
||||
environment string
|
||||
envSet bool
|
||||
want int
|
||||
}{
|
||||
{name: "default", want: 3},
|
||||
{name: "minimum", requested: &one, want: 1},
|
||||
{name: "two", requested: &two, environment: "2", envSet: true, want: 2},
|
||||
{name: "explicit default", requested: &three, environment: "3", envSet: true, want: 3},
|
||||
{name: "default environment", environment: "3", envSet: true, want: 3},
|
||||
{name: "zero", requested: &zero},
|
||||
{name: "excess", requested: &four},
|
||||
{name: "environment mismatch", requested: &two, environment: "3", envSet: true},
|
||||
{name: "environment without request", environment: "2", envSet: true},
|
||||
{name: "empty environment", requested: &two, envSet: true},
|
||||
{name: "padded environment", requested: &two, environment: "02", envSet: true},
|
||||
{name: "decimal environment", requested: &two, environment: "2.0", envSet: true},
|
||||
{name: "whitespace environment", requested: &two, environment: " 2", envSet: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := probeSendLimit(tc.requested, tc.environment, tc.envSet)
|
||||
if got != tc.want || (err != nil) != (tc.want == 0) {
|
||||
t.Fatalf("limit = %d, error = %v; want %d", got, err, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type probeRoundTripper func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f probeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
|
||||
|
||||
func TestProbeTransportHonorsActualSendCap(t *testing.T) {
|
||||
for _, limit := range []int{1, 2, requestbudget.GraphMaxSends} {
|
||||
t.Run(fmt.Sprint(limit), func(t *testing.T) {
|
||||
ctx, guard, cancel, err := bindProbeExecution(context.Background(), "offline", "model", limit)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cancel()
|
||||
ctx = requestbudget.WithModel(ctx, "model")
|
||||
sends := 0
|
||||
transport := requestbudget.Transport{Base: probeRoundTripper(func(req *http.Request) (*http.Response, error) {
|
||||
sends++
|
||||
if req.Context().Err() != nil {
|
||||
t.Fatal("allowed response was canceled")
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("usable response"))}, nil
|
||||
})}
|
||||
for i := 0; i < limit; i++ {
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, "https://offline.invalid", nil)
|
||||
response, err := transport.RoundTrip(req)
|
||||
if err != nil {
|
||||
t.Fatalf("send %d: %v", i+1, err)
|
||||
}
|
||||
body, readErr := io.ReadAll(response.Body)
|
||||
response.Body.Close()
|
||||
if readErr != nil || string(body) != "usable response" || requestbudget.Check(ctx) != nil {
|
||||
t.Fatal("last allowed response must remain usable")
|
||||
}
|
||||
}
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, "https://offline.invalid", nil)
|
||||
if _, err := transport.RoundTrip(req); !errors.Is(err, requestbudget.ErrStopped) {
|
||||
t.Fatalf("excess send error = %v", err)
|
||||
}
|
||||
if sends != limit || guard.Reservations() != limit || !errors.Is(context.Cause(ctx), requestbudget.ErrStopped) {
|
||||
t.Fatalf("sends = %d, reservations = %d, cause = %v", sends, guard.Reservations(), context.Cause(ctx))
|
||||
}
|
||||
if err := guard.Reserve(context.Background(), "model", requestbudget.GraphPhase); !errors.Is(err, requestbudget.ErrStopped) {
|
||||
t.Fatal("stopped execution was revived")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeBudgetPreservesScopeAndDeadline(t *testing.T) {
|
||||
for _, tc := range []struct{ name, model, phase string }{
|
||||
{"model", "other", requestbudget.GraphPhase},
|
||||
{"phase", "model", "other"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ctx, guard, cancel, err := bindProbeExecution(context.Background(), "offline", "model", 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cancel()
|
||||
if err := guard.Reserve(ctx, tc.model, tc.phase); !errors.Is(err, requestbudget.ErrStopped) {
|
||||
t.Fatalf("scope mismatch error = %v", err)
|
||||
}
|
||||
if guard.Reservations() != 0 || !errors.Is(context.Cause(ctx), requestbudget.ErrStopped) {
|
||||
t.Fatal("scope mismatch consumed a send or did not stop execution")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("inherited deadline", func(t *testing.T) {
|
||||
deadline := time.Now().Add(time.Minute)
|
||||
parent, parentCancel := context.WithDeadline(context.Background(), deadline)
|
||||
defer parentCancel()
|
||||
ctx, guard, cancel, err := bindProbeExecution(parent, "offline", "model", 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cancel()
|
||||
if got, ok := ctx.Deadline(); !ok || !got.Equal(deadline) {
|
||||
t.Fatal("inherited deadline was reset")
|
||||
}
|
||||
if err := guard.Reserve(ctx, "model", requestbudget.GraphPhase); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := ctx.Deadline(); !got.Equal(deadline) {
|
||||
t.Fatal("reservation reset the deadline")
|
||||
}
|
||||
parentCancel()
|
||||
if err := guard.Reserve(ctx, "model", requestbudget.GraphPhase); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancellation error = %v", err)
|
||||
}
|
||||
if guard.Reservations() != 1 {
|
||||
t.Fatal("canceled send consumed a reservation")
|
||||
}
|
||||
})
|
||||
t.Run("default deadline", func(t *testing.T) {
|
||||
started := time.Now()
|
||||
ctx, _, cancel, err := bindProbeExecution(context.Background(), "offline", "model", 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cancel()
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok || deadline.Before(started.Add(requestbudget.GraphTimeout)) || deadline.After(time.Now().Add(requestbudget.GraphTimeout)) {
|
||||
t.Fatal("probe did not retain the shared absolute timeout")
|
||||
}
|
||||
})
|
||||
t.Run("already canceled", func(t *testing.T) {
|
||||
parent, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, _, cleanup, err := bindProbeExecution(parent, "offline", "model", 2); err == nil {
|
||||
cleanup()
|
||||
t.Fatal("canceled execution was rebound")
|
||||
}
|
||||
})
|
||||
t.Run("nested execution", func(t *testing.T) {
|
||||
ctx, _, cancel, err := bindProbeExecution(context.Background(), "offline", "model", 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cancel()
|
||||
if _, _, cleanup, err := bindProbeExecution(ctx, "retry", "model", 2); err == nil {
|
||||
cleanup()
|
||||
t.Fatal("nested execution reset the allowance")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestProbeBudgetConcurrentReservationsStayBounded(t *testing.T) {
|
||||
ctx, guard, cancel, err := bindProbeExecution(context.Background(), "offline", "model", 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cancel()
|
||||
var successful atomic.Int32
|
||||
var group sync.WaitGroup
|
||||
for i := 0; i < 16; i++ {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
if guard.Reserve(ctx, "model", requestbudget.GraphPhase) == nil {
|
||||
successful.Add(1)
|
||||
}
|
||||
}()
|
||||
}
|
||||
group.Wait()
|
||||
if successful.Load() != 2 || guard.Reservations() != 2 {
|
||||
t.Fatalf("successful = %d; reservations = %d", successful.Load(), guard.Reservations())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
// graph_extraction_probe calls the production extractor without writing graph data.
|
||||
// The model configuration and AES key arrive on stdin and are never reported.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
chatpipeline "github.com/Tencent/WeKnora/internal/application/service/chat_pipeline"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type request struct {
|
||||
ExecutionID string `json:"execution_id"`
|
||||
MaxHTTPSends *int `json:"max_http_sends"`
|
||||
AESKey string `json:"aes_key"`
|
||||
Model types.Model `json:"model"`
|
||||
Chunk types.Chunk `json:"chunk"`
|
||||
ExtractConfig types.ExtractConfig `json:"extract_config"`
|
||||
}
|
||||
|
||||
type chatDelegate interface{ chat.Chat }
|
||||
|
||||
type probeCall struct {
|
||||
Number int `json:"number"`
|
||||
ElapsedMS int64 `json:"elapsed_ms"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
Thinking *bool `json:"thinking_requested"`
|
||||
ResponseReceived bool `json:"response_received"`
|
||||
Failed bool `json:"failed"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
ContentRunes int `json:"content_runes"`
|
||||
ReasoningRunes int `json:"reasoning_runes"`
|
||||
Usage types.TokenUsage `json:"usage"`
|
||||
}
|
||||
|
||||
type recordingChat struct {
|
||||
chatDelegate
|
||||
response *types.ChatResponse
|
||||
calls int
|
||||
records []probeCall
|
||||
}
|
||||
|
||||
func (c *recordingChat) Chat(ctx context.Context, messages []chat.Message, options *chat.ChatOptions) (*types.ChatResponse, error) {
|
||||
started := time.Now()
|
||||
response, err := c.chatDelegate.Chat(ctx, messages, options)
|
||||
c.response = response
|
||||
c.calls++
|
||||
record := probeCall{
|
||||
Number: c.calls, ElapsedMS: time.Since(started).Milliseconds(),
|
||||
ResponseReceived: response != nil, Failed: err != nil,
|
||||
}
|
||||
if options != nil {
|
||||
record.MaxTokens = options.MaxTokens
|
||||
if options.Thinking != nil {
|
||||
thinking := *options.Thinking
|
||||
record.Thinking = &thinking
|
||||
}
|
||||
}
|
||||
if response != nil {
|
||||
record.FinishReason = response.FinishReason
|
||||
record.ContentRunes = utf8.RuneCountInString(response.Content)
|
||||
record.ReasoningRunes = utf8.RuneCountInString(response.ReasoningContent)
|
||||
record.Usage = response.Usage
|
||||
}
|
||||
c.records = append(c.records, record)
|
||||
return response, err
|
||||
}
|
||||
|
||||
func run() error {
|
||||
var input request
|
||||
if err := json.NewDecoder(os.Stdin).Decode(&input); err != nil {
|
||||
return fmt.Errorf("invalid probe input")
|
||||
}
|
||||
capValue, capSet := os.LookupEnv("GRAPH_PROBE_MAX_SENDS")
|
||||
maxSends, err := probeSendLimit(input.MaxHTTPSends, capValue, capSet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if input.ExecutionID == "" || input.Chunk.ID == "" || input.Chunk.Content == "" || input.Model.ID == "" || input.Model.Name == "" {
|
||||
return fmt.Errorf("probe requires a source chunk and model")
|
||||
}
|
||||
if err := os.Setenv("SYSTEM_AES_KEY", input.AESKey); err != nil {
|
||||
return err
|
||||
}
|
||||
input.AESKey = ""
|
||||
key, err := utils.DecryptStoredSecret(input.Model.Parameters.APIKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot decrypt configured model credential")
|
||||
}
|
||||
input.Model.Parameters.APIKey = key
|
||||
defer os.Unsetenv("SYSTEM_AES_KEY")
|
||||
chatConfig := chat.ConfigFromModel(&input.Model, "", "")
|
||||
model, err := chat.NewChat(chatConfig, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot initialize configured remote model")
|
||||
}
|
||||
base, err := os.ReadFile("config/config.yaml")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var config struct {
|
||||
Extract struct {
|
||||
Graph struct {
|
||||
Description string `yaml:"description"`
|
||||
} `yaml:"extract_graph"`
|
||||
} `yaml:"extract"`
|
||||
}
|
||||
if err := yaml.Unmarshal(base, &config); err != nil {
|
||||
return err
|
||||
}
|
||||
recorder := &recordingChat{chatDelegate: model}
|
||||
extractor := chatpipeline.NewGraphExtractor(recorder, &types.PromptTemplateStructured{
|
||||
Description: types.AppendCustomPromptInstructions(config.Extract.Graph.Description, input.ExtractConfig.CustomInstructions, "graph_extraction"),
|
||||
Tags: input.ExtractConfig.Tags,
|
||||
})
|
||||
ctx, guard, cancel, err := bindProbeExecution(context.Background(), input.ExecutionID, input.Model.ID, maxSends)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cancel()
|
||||
started := time.Now()
|
||||
graph, extractionErr := extractor.Extract(ctx, input.Chunk.Content)
|
||||
hash := sha256.Sum256([]byte(input.Chunk.Content))
|
||||
report := map[string]any{
|
||||
"checked_at": started.UTC().Format(time.RFC3339), "elapsed_ms": time.Since(started).Milliseconds(),
|
||||
"chunk_id": input.Chunk.ID, "source_sha256": hex.EncodeToString(hash[:]),
|
||||
"model_name": input.Model.Name, "allowed_relation_types": input.ExtractConfig.Tags,
|
||||
"passed": extractionErr == nil, "graph": graph, "production_write": false,
|
||||
"model_calls": recorder.calls,
|
||||
"execution_id": input.ExecutionID,
|
||||
"http_send_reservations": guard.Reservations(),
|
||||
"http_send_limit": maxSends,
|
||||
"call_diagnostics": recorder.records,
|
||||
"effective_thinking_control": chat.EffectiveThinkingControl(chatConfig),
|
||||
"diagnostics_version": 2,
|
||||
}
|
||||
if recorder.response != nil {
|
||||
report["finish_reason"] = recorder.response.FinishReason
|
||||
report["usage"] = recorder.response.Usage
|
||||
if extractionErr != nil {
|
||||
report["rejected_model_content"] = recorder.response.Content
|
||||
}
|
||||
}
|
||||
if extractionErr != nil {
|
||||
report["error"] = extractionErr.Error()
|
||||
}
|
||||
encoded, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("GRAPH_EXTRACTION_PROBE_RESULT=%s\n", encoded)
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
type fakeProbeChat struct {
|
||||
chatDelegate
|
||||
response *types.ChatResponse
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeProbeChat) Chat(context.Context, []chat.Message, *chat.ChatOptions) (*types.ChatResponse, error) {
|
||||
return f.response, f.err
|
||||
}
|
||||
|
||||
func TestRecordingChatPreservesPerCallDiagnosticsWithoutContent(t *testing.T) {
|
||||
fake := &fakeProbeChat{response: &types.ChatResponse{
|
||||
Content: "answer", ReasoningContent: "private reasoning", FinishReason: "length",
|
||||
Usage: types.TokenUsage{PromptTokens: 10, CompletionTokens: 4, TotalTokens: 14},
|
||||
}}
|
||||
recorder := &recordingChat{chatDelegate: fake}
|
||||
thinking := false
|
||||
options := &chat.ChatOptions{MaxTokens: 4096, Thinking: &thinking}
|
||||
first, err := recorder.Chat(context.Background(), nil, options)
|
||||
if err != nil || first != fake.response {
|
||||
t.Fatal("recorder changed the response")
|
||||
}
|
||||
thinking = true
|
||||
fake.response = &types.ChatResponse{Content: "\u4e2d\u6587", FinishReason: "stop"}
|
||||
if _, err := recorder.Chat(context.Background(), nil, options); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if recorder.calls != 2 || len(recorder.records) != 2 {
|
||||
t.Fatal("missing call diagnostics")
|
||||
}
|
||||
a, b := recorder.records[0], recorder.records[1]
|
||||
if a.Number != 1 || a.MaxTokens != 4096 || a.Thinking == nil || *a.Thinking ||
|
||||
!a.ResponseReceived || a.Failed || a.FinishReason != "length" ||
|
||||
a.ContentRunes != 6 || a.ReasoningRunes != 17 || a.Usage.TotalTokens != 14 {
|
||||
t.Fatalf("unexpected first call metadata: %+v", a)
|
||||
}
|
||||
if b.Number != 2 || b.Thinking == nil || !*b.Thinking || b.ContentRunes != 2 || b.ReasoningRunes != 0 {
|
||||
t.Fatalf("unexpected second call metadata: %+v", b)
|
||||
}
|
||||
encoded, err := json.Marshal(recorder.records)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(encoded), "private reasoning") || strings.Contains(string(encoded), "answer") {
|
||||
t.Fatal("diagnostics exposed response content")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordingChatRecordsNoResponseWithoutLeakingError(t *testing.T) {
|
||||
want := errors.New("private upstream error")
|
||||
recorder := &recordingChat{chatDelegate: &fakeProbeChat{err: want}}
|
||||
response, err := recorder.Chat(context.Background(), nil, nil)
|
||||
if response != nil || err != want || len(recorder.records) != 1 {
|
||||
t.Fatal("recorder changed the failure")
|
||||
}
|
||||
record := recorder.records[0]
|
||||
if record.ResponseReceived || !record.Failed || record.Thinking != nil || record.ContentRunes != 0 || record.ReasoningRunes != 0 {
|
||||
t.Fatalf("unexpected failure metadata: %+v", record)
|
||||
}
|
||||
encoded, _ := json.Marshal(record)
|
||||
if strings.Contains(string(encoded), want.Error()) {
|
||||
t.Fatal("diagnostics exposed upstream error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const {execFileSync} = require('node:child_process')
|
||||
|
||||
const tenantID = 10059
|
||||
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
|
||||
const literal = value => `'${String(value).replaceAll("'", "''")}'`
|
||||
const arraySQL = values => `ARRAY[${values.map(literal).join(',')}]::text[]`
|
||||
|
||||
function validateReport(report, now = Date.now()) {
|
||||
assert.ok(report && typeof report === 'object', 'Missing indexed fixture report')
|
||||
assert.equal(report.base, 'https://agent.pducv.cn/api/v1', 'Unexpected indexed API origin')
|
||||
// Older completed reports omit tenant_id; every database identity is still checked below.
|
||||
assert.ok(report.tenant_id === undefined || report.tenant_id === tenantID, 'Unexpected indexed tenant')
|
||||
assert.match(report.prefix || '', /^indexed-isolation-[0-9]+$/, 'Invalid indexed fixture prefix')
|
||||
for (const key of ['started_at', 'time']) {
|
||||
assert.ok(typeof report[key] === 'string' && Number.isFinite(Date.parse(report[key])), `Invalid report ${key}`)
|
||||
assert.equal(new Date(report[key]).toISOString(), report[key], `Noncanonical report ${key}`)
|
||||
}
|
||||
const start = Date.parse(report.started_at), end = Date.parse(report.time)
|
||||
const prefixTime = Number(report.prefix.slice('indexed-isolation-'.length))
|
||||
assert.ok(start <= end && end <= now && end - start <= 86400000, 'Invalid indexed report window')
|
||||
assert.ok(Number.isSafeInteger(prefixTime) && prefixTime >= start && prefixTime <= end, 'Prefix outside report window')
|
||||
const fixtures = report.fixtures
|
||||
assert.ok(fixtures && typeof fixtures === 'object', 'Missing indexed fixture IDs')
|
||||
const allIDs = []
|
||||
for (const [key, maximum] of Object.entries({users: 2, roles: 1, kbs: 1, docs: 1, tags: 2})) {
|
||||
assert.ok(Array.isArray(fixtures[key]) && fixtures[key].length <= maximum, `Invalid indexed ${key} count`)
|
||||
for (const id of fixtures[key]) {
|
||||
assert.ok(typeof id === 'string' && uuid.test(id), `Invalid indexed ${key} UUID`)
|
||||
allIDs.push(id.toLowerCase())
|
||||
}
|
||||
}
|
||||
assert.equal(new Set(allIDs).size, allIDs.length, 'Reused indexed fixture UUID')
|
||||
assert.ok(!fixtures.kbs.length || fixtures.users.length > 0, 'Knowledge fixture has no owner')
|
||||
assert.ok(!(fixtures.tags.length || fixtures.docs.length) || fixtures.kbs.length === 1, 'Child fixture has no knowledge base')
|
||||
for (const id of fixtures.kbs) {
|
||||
const proof = report.cleanup?.filter(item => item?.name === `knowledge ${id}`)
|
||||
assert.ok(proof?.length === 1 && proof[0].passed === true && !proof[0].error, 'Missing successful knowledge deletion proof')
|
||||
}
|
||||
return fixtures
|
||||
}
|
||||
|
||||
function buildCleanupSQL(report, now = Date.now()) {
|
||||
const fixtures = validateReport(report, now)
|
||||
const ids = Object.fromEntries(Object.entries(fixtures).filter(([, value]) => Array.isArray(value)).map(([key, value]) => [key, arraySQL(value)]))
|
||||
const window = alias => `${alias}.created_at BETWEEN ${literal(report.started_at)}::timestamptz AND ${literal(report.time)}::timestamptz`
|
||||
const deleted = alias => `${alias}.deleted_at BETWEEN ${alias}.created_at AND ${literal(report.time)}::timestamptz`
|
||||
const guards = []
|
||||
const requireSQL = (predicate, message) => guards.push(` IF (${predicate}) IS DISTINCT FROM TRUE THEN RAISE EXCEPTION ${literal(message)}; END IF;`)
|
||||
fixtures.users.forEach((id, index) => {
|
||||
const name = `${report.prefix}-${['owner', 'reader'][index]}`
|
||||
requireSQL(`SELECT count(*)=1 FROM users u WHERE u.id=${literal(id)} AND u.tenant_id=${tenantID} AND u.username=${literal(name)} AND u.email=${literal(name + '@example.test')} AND ${window('u')}`, 'Indexed fixture user identity mismatch')
|
||||
})
|
||||
fixtures.kbs.forEach(id => {
|
||||
requireSQL(`SELECT count(*)=1 FROM knowledge_bases k WHERE k.id=${literal(id)} AND k.tenant_id=${tenantID} AND k.name=${literal(report.prefix)} AND k.creator_id=${literal(fixtures.users[0])} AND ${window('k')} AND ${deleted('k')}`, 'Indexed fixture knowledge base must match its deleted owner scope')
|
||||
})
|
||||
fixtures.docs.forEach(id => {
|
||||
requireSQL(`SELECT count(*)=1 FROM knowledges d WHERE d.id=${literal(id)} AND d.tenant_id=${tenantID} AND d.knowledge_base_id=ANY(${ids.kbs}) AND d.type='file' AND d.title=${literal(report.prefix + '.txt')} AND d.file_name=${literal(report.prefix + '.txt')} AND ${window('d')} AND ${deleted('d')}`, 'Indexed fixture document must match its deleted knowledge scope')
|
||||
})
|
||||
requireSQL(`NOT EXISTS (SELECT 1 FROM knowledges d WHERE d.knowledge_base_id=ANY(${ids.kbs}) AND NOT (d.id=ANY(${ids.docs})))`, 'Unreported document in indexed fixture knowledge base')
|
||||
requireSQL(`NOT EXISTS (SELECT 1 FROM knowledge_tags t WHERE t.knowledge_base_id=ANY(${ids.kbs}) AND NOT (t.id=ANY(${ids.tags})))`, 'Unreported tag in indexed fixture knowledge base')
|
||||
fixtures.tags.forEach((id, index) => {
|
||||
const name = `${report.prefix}-${['category', 'empty-category'][index]}`
|
||||
// Missing tags are allowed only for an already-cleaned (or partially cleaned) fixture.
|
||||
requireSQL(`NOT EXISTS (SELECT 1 FROM knowledge_tags t WHERE t.id=${literal(id)} AND (t.tenant_id=${tenantID} AND t.knowledge_base_id=ANY(${ids.kbs}) AND t.name=${literal(name)} AND ${window('t')}) IS DISTINCT FROM TRUE)`, 'Indexed fixture tag identity mismatch')
|
||||
})
|
||||
requireSQL(`NOT EXISTS (SELECT 1 FROM knowledge_tag_relations r WHERE (r.tag_id=ANY(${ids.tags}) OR r.knowledge_id=ANY(${ids.docs})) AND (r.tag_id=ANY(${ids.tags}) AND r.knowledge_id=ANY(${ids.docs}) AND EXISTS (SELECT 1 FROM knowledge_tags t WHERE t.id=r.tag_id) AND ${window('r')}) IS DISTINCT FROM TRUE)`, 'Indexed fixture has an unexpected tag relation')
|
||||
requireSQL(`NOT EXISTS (SELECT 1 FROM chunks c WHERE (c.tag_id=ANY(${ids.tags}) OR c.knowledge_base_id=ANY(${ids.kbs}) OR c.knowledge_id=ANY(${ids.docs})) AND (c.tenant_id=${tenantID} AND c.knowledge_base_id=ANY(${ids.kbs}) AND c.knowledge_id=ANY(${ids.docs}) AND (c.tag_id IS NULL OR c.tag_id='' OR c.tag_id=ANY(${ids.tags})) AND ${window('c')} AND ${deleted('c')}) IS DISTINCT FROM TRUE)`, 'Indexed fixture has an active or unrelated chunk')
|
||||
return `BEGIN;
|
||||
SET LOCAL lock_timeout='3s';
|
||||
SET LOCAL statement_timeout='15s';
|
||||
SET LOCAL idle_in_transaction_session_timeout='15s';
|
||||
-- Relations have no foreign keys: table locks also exclude new out-of-scope links during verification.
|
||||
LOCK TABLE knowledge_tags, knowledge_tag_relations IN SHARE ROW EXCLUSIVE MODE;
|
||||
LOCK TABLE knowledges, chunks IN SHARE MODE;
|
||||
DO $indexed_cleanup$
|
||||
BEGIN
|
||||
PERFORM 1 FROM users WHERE id=ANY(${ids.users}) FOR UPDATE;
|
||||
PERFORM 1 FROM knowledge_bases WHERE id=ANY(${ids.kbs}) FOR UPDATE;
|
||||
${guards.join('\n')}
|
||||
END
|
||||
$indexed_cleanup$;
|
||||
WITH removed AS (
|
||||
DELETE FROM knowledge_tag_relations WHERE tag_id=ANY(${ids.tags}) AND knowledge_id=ANY(${ids.docs}) RETURNING 1
|
||||
) SELECT json_build_object('kind','relations','deleted',count(*)) FROM removed;
|
||||
WITH removed AS (
|
||||
DELETE FROM knowledge_tags WHERE tenant_id=${tenantID} AND knowledge_base_id=ANY(${ids.kbs}) AND id=ANY(${ids.tags}) RETURNING 1
|
||||
) SELECT json_build_object('kind','tags','deleted',count(*)) FROM removed;
|
||||
COMMIT;
|
||||
`
|
||||
}
|
||||
|
||||
function cleanupIndexedFixtureTags(report, run = execFileSync, now = Date.now()) {
|
||||
const input = buildCleanupSQL(report, now)
|
||||
const output = run('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres',
|
||||
'psql', '-X', '-q', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-At', '-v', 'ON_ERROR_STOP=1'],
|
||||
{input, encoding: 'utf8', timeout: 45000, maxBuffer: 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe']}).trim()
|
||||
const results = output.split('\n').map(line => JSON.parse(line))
|
||||
assert.deepEqual(results.map(result => result.kind), ['relations', 'tags'], 'Unexpected cleanup result')
|
||||
for (const result of results) assert.ok(Number.isSafeInteger(result.deleted) && result.deleted >= 0 && result.deleted <= report.fixtures.tags.length, 'Invalid cleanup count')
|
||||
return {relations_deleted: results[0].deleted, tags_deleted: results[1].deleted}
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = process.argv.slice(2)
|
||||
const apply = args[0] === '--apply'
|
||||
assert.equal(args.length, apply ? 2 : 1, 'Usage: node scripts/indexed_fixture_tag_cleanup.cjs [--apply] REPORT.json')
|
||||
const report = JSON.parse(fs.readFileSync(args[apply ? 1 : 0], 'utf8'))
|
||||
if (apply) console.log(JSON.stringify(cleanupIndexedFixtureTags(report)))
|
||||
else process.stdout.write(buildCleanupSQL(report))
|
||||
}
|
||||
|
||||
module.exports = {validateReport, buildCleanupSQL, cleanupIndexedFixtureTags}
|
||||
if (require.main === module) {
|
||||
try { main() } catch (error) { console.error(error.message); process.exitCode = 1 }
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const test = require('node:test')
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const {execFileSync} = require('node:child_process')
|
||||
const {validateReport, buildCleanupSQL, cleanupIndexedFixtureTags} = require('./indexed_fixture_tag_cleanup.cjs')
|
||||
|
||||
const id = number => `00000000-0000-4000-8000-${String(number).padStart(12, '0')}`
|
||||
const literal = value => value === null ? 'NULL' : typeof value === 'number' ? String(value) : `'${String(value).replaceAll("'", "''")}'`
|
||||
const start = '2026-09-19T17:45:09.219Z'
|
||||
const created = '2026-09-19T17:45:12.000Z'
|
||||
const deleted = '2026-09-19T17:45:40.000Z'
|
||||
const finish = '2026-09-19T17:45:45.637Z'
|
||||
const now = Date.parse(finish) + 1000
|
||||
|
||||
function fixture() {
|
||||
return {base: 'https://agent.pducv.cn/api/v1', tenant_id: 10059,
|
||||
started_at: start, time: finish, prefix: 'indexed-isolation-1789839909244',
|
||||
fixtures: {users: [id(1), id(2)], roles: [id(3)], kbs: [id(4)], docs: [id(5)], tags: [id(6), id(7)]},
|
||||
cleanup: [{name: `knowledge ${id(4)}`, passed: true}], passed: true}
|
||||
}
|
||||
|
||||
for (const [name, mutate] of [
|
||||
['wrong tenant', r => {r.tenant_id = 1}],
|
||||
['wrong API', r => {r.base = 'http://localhost/api/v1'}],
|
||||
['unsafe prefix', r => {r.prefix += "'; DELETE FROM users;--"}],
|
||||
['foreign prefix', r => {r.prefix = 'resource-isolation-1789839909244'}],
|
||||
['prefix outside window', r => {r.prefix = 'indexed-isolation-1'}],
|
||||
['invalid timestamp', r => {r.time = 'today'}],
|
||||
['noncanonical timestamp', r => {r.time = '2026-09-19'}],
|
||||
['inverted window', r => {r.started_at = r.time}],
|
||||
['future report', r => {r.time = '2099-01-01T00:00:00.000Z'}],
|
||||
['excessive window', r => {r.started_at = '2026-09-17T00:00:00.000Z'}],
|
||||
['missing fixture IDs', r => {delete r.fixtures.tags}],
|
||||
['invalid fixture UUID', r => {r.fixtures.tags[0] = 'all'}],
|
||||
['nonstring UUID', r => {r.fixtures.tags[0] = 123}],
|
||||
['duplicate UUID', r => {r.fixtures.tags[1] = r.fixtures.tags[0]}],
|
||||
['reused UUID across types', r => {r.fixtures.tags[0] = r.fixtures.docs[0]}],
|
||||
['too many fixtures', r => {r.fixtures.kbs.push(id(8))}],
|
||||
['missing owner', r => {r.fixtures.users = []}],
|
||||
['missing parent', r => {r.fixtures.kbs = []}],
|
||||
['missing deletion proof', r => {delete r.cleanup}],
|
||||
['failed deletion proof', r => {r.cleanup[0].passed = false}],
|
||||
['deletion proof with error', r => {r.cleanup[0].error = 'failed'}],
|
||||
['duplicate deletion proof', r => {r.cleanup.push({...r.cleanup[0]})}],
|
||||
['wrong deletion proof', r => {r.cleanup[0].name = `knowledge ${id(8)}`}],
|
||||
]) {
|
||||
test(`rejects ${name} before executing SQL`, () => {
|
||||
const report = fixture()
|
||||
mutate(report)
|
||||
let calls = 0
|
||||
assert.throws(() => cleanupIndexedFixtureTags(report, () => {calls++; return ''}, now))
|
||||
assert.equal(calls, 0)
|
||||
})
|
||||
}
|
||||
|
||||
test('accepts legacy reports and failed runs with successful knowledge deletion', () => {
|
||||
const report = fixture()
|
||||
delete report.tenant_id
|
||||
report.passed = false
|
||||
report.failures = ['Search fixture failed']
|
||||
assert.doesNotThrow(() => validateReport(report, now))
|
||||
})
|
||||
|
||||
test('uses one bounded transaction through stdin and only deletes tags and their relations', () => {
|
||||
let call
|
||||
const result = cleanupIndexedFixtureTags(fixture(), (command, args, options) => {
|
||||
call = {command, args, options}
|
||||
return '{"kind":"relations","deleted":1}\n{"kind":"tags","deleted":2}\n'
|
||||
}, now)
|
||||
assert.deepEqual(result, {relations_deleted: 1, tags_deleted: 2})
|
||||
assert.equal(call.command, 'docker')
|
||||
assert.ok(call.args.includes('colima-yawan'))
|
||||
assert.ok(call.args.includes('yawan_native_acceptance_20260915'))
|
||||
assert.ok(call.args.includes('ON_ERROR_STOP=1'))
|
||||
assert.ok(call.args.includes('-X'))
|
||||
assert.equal(call.options.timeout, 45000)
|
||||
assert.deepEqual(call.options.stdio, ['pipe', 'pipe', 'pipe'])
|
||||
const sql = call.options.input
|
||||
assert.match(sql, /^BEGIN;\nSET LOCAL lock_timeout='3s';/)
|
||||
assert.match(sql, /SET LOCAL statement_timeout='15s'/)
|
||||
assert.match(sql, /SET LOCAL idle_in_transaction_session_timeout='15s'/)
|
||||
assert.match(sql, /LOCK TABLE knowledge_tags, knowledge_tag_relations IN SHARE ROW EXCLUSIVE MODE/)
|
||||
assert.match(sql, /LOCK TABLE knowledges, chunks IN SHARE MODE/)
|
||||
assert.match(sql, /FROM users.*FOR UPDATE/)
|
||||
assert.match(sql, /FROM knowledge_bases.*FOR UPDATE/)
|
||||
assert.match(sql, /COMMIT;\n$/)
|
||||
assert.deepEqual([...sql.matchAll(/DELETE FROM (\w+)/g)].map(match => match[1]), ['knowledge_tag_relations', 'knowledge_tags'])
|
||||
assert.doesNotMatch(sql, /\b(?:UPDATE|INSERT INTO|TRUNCATE|DROP|ALTER)\s+\w+/)
|
||||
assert.ok(sql.indexOf('RAISE EXCEPTION') < sql.indexOf('DELETE FROM'))
|
||||
})
|
||||
|
||||
test('propagates SQL refusal and rejects malformed command results', () => {
|
||||
assert.throws(() => cleanupIndexedFixtureTags(fixture(), () => {throw new Error('transaction refused')}, now), /transaction refused/)
|
||||
for (const output of ['', '{}', '{"kind":"relations","deleted":-1}\n{"kind":"tags","deleted":2}', '{"kind":"relations","deleted":1}\n{"kind":"tags","deleted":3}']) {
|
||||
assert.throws(() => cleanupIndexedFixtureTags(fixture(), () => output, now))
|
||||
}
|
||||
})
|
||||
|
||||
const pgBin = process.env.POSTGRES_TEST_BIN || '/opt/homebrew/opt/postgresql@16/bin'
|
||||
test('cleanup SQL runs against an isolated local PostgreSQL cluster', {skip: !fs.existsSync(path.join(pgBin, 'initdb'))}, async t => {
|
||||
const directory = fs.mkdtempSync('/tmp/indexed-tag-test-')
|
||||
const data = path.join(directory, 'data')
|
||||
let running = false
|
||||
const run = (binary, args, input) => execFileSync(path.join(pgBin, binary), args, {
|
||||
input, encoding: 'utf8', timeout: 30000, stdio: ['pipe', 'pipe', 'pipe'],
|
||||
})
|
||||
const query = sql => run('psql', ['-X', '-q', '-h', directory, '-p', '55459', '-U', 'fixture', '-d', 'postgres', '-At', '-v', 'ON_ERROR_STOP=1'], sql).trim()
|
||||
const tables = ['users', 'knowledge_bases', 'knowledges', 'knowledge_tags', 'knowledge_tag_relations', 'chunks']
|
||||
const snapshot = () => JSON.parse(query(`SELECT json_build_object(${tables.map(name => `${literal(name)},(SELECT COALESCE(json_agg(t ORDER BY row_to_json(t)::text),'[]'::json) FROM ${name} t)`).join(',')});`))
|
||||
const insert = (table, rows) => rows.length ? `INSERT INTO ${table} (${Object.keys(rows[0]).join(',')}) VALUES ${rows.map(row => `(${Object.values(row).map(literal).join(',')})`).join(',')};` : ''
|
||||
function seed(report = fixture()) {
|
||||
const users = report.fixtures.users.map((userID, index) => ({id: userID, tenant_id: 10059,
|
||||
username: `${report.prefix}-${['owner', 'reader'][index]}`, email: `${report.prefix}-${['owner', 'reader'][index]}@example.test`, created_at: created, deleted_at: deleted}))
|
||||
const kbs = report.fixtures.kbs.map(kbID => ({id: kbID, tenant_id: 10059, creator_id: report.fixtures.users[0], name: report.prefix, created_at: created, deleted_at: deleted}))
|
||||
const docs = report.fixtures.docs.map(docID => ({id: docID, tenant_id: 10059, knowledge_base_id: report.fixtures.kbs[0], title: report.prefix + '.txt', file_name: report.prefix + '.txt', type: 'file', created_at: created, deleted_at: deleted}))
|
||||
const tags = report.fixtures.tags.map((tagID, index) => ({id: tagID, tenant_id: 10059, knowledge_base_id: report.fixtures.kbs[0], name: `${report.prefix}-${['category', 'empty-category'][index]}`, created_at: created}))
|
||||
const relations = docs.length && tags.length ? [{knowledge_id: docs[0].id, tag_id: tags[0].id, created_at: created}] : []
|
||||
const chunks = docs.length ? [{id: id(8), tenant_id: 10059, knowledge_base_id: report.fixtures.kbs[0], knowledge_id: docs[0].id, tag_id: '', created_at: created, deleted_at: deleted}] : []
|
||||
users.push({id: id(101), tenant_id: 10059, username: 'real-user', email: 'real-user@example.org', created_at: created, deleted_at: null})
|
||||
kbs.push({id: id(104), tenant_id: 10059, creator_id: id(101), name: 'Real knowledge', created_at: created, deleted_at: null})
|
||||
docs.push({id: id(105), tenant_id: 10059, knowledge_base_id: id(104), title: 'Real file', file_name: 'real.txt', type: 'file', created_at: created, deleted_at: null})
|
||||
tags.push({id: id(106), tenant_id: 10059, knowledge_base_id: id(104), name: 'Real category', created_at: created})
|
||||
relations.push({knowledge_id: id(105), tag_id: id(106), created_at: created})
|
||||
chunks.push({id: id(108), tenant_id: 10059, knowledge_base_id: id(104), knowledge_id: id(105), tag_id: id(106), created_at: created, deleted_at: null})
|
||||
query(`TRUNCATE ${tables.join(',')};\n` + Object.entries({users, knowledge_bases: kbs, knowledges: docs, knowledge_tags: tags, knowledge_tag_relations: relations, chunks}).map(([table, rows]) => insert(table, rows)).join('\n'))
|
||||
}
|
||||
try {
|
||||
run('initdb', ['-D', data, '-U', 'fixture', '--auth=trust', '--no-locale', '--encoding=UTF8'])
|
||||
run('pg_ctl', ['-D', data, '-l', path.join(directory, 'postgres.log'), '-o', `-F -c listen_addresses='' -k ${directory} -p 55459`, '-w', 'start'])
|
||||
running = true
|
||||
query(`CREATE TABLE users (id text PRIMARY KEY, tenant_id int, username text, email text, created_at timestamptz, deleted_at timestamptz);
|
||||
CREATE TABLE knowledge_bases (id text PRIMARY KEY, tenant_id int, creator_id text, name text, created_at timestamptz, deleted_at timestamptz);
|
||||
CREATE TABLE knowledges (id text PRIMARY KEY, tenant_id int, knowledge_base_id text, title text, file_name text, type text, created_at timestamptz, deleted_at timestamptz);
|
||||
CREATE TABLE knowledge_tags (id text PRIMARY KEY, tenant_id int, knowledge_base_id text, name text, created_at timestamptz);
|
||||
CREATE TABLE knowledge_tag_relations (knowledge_id text, tag_id text, created_at timestamptz, PRIMARY KEY (knowledge_id,tag_id));
|
||||
CREATE TABLE chunks (id text PRIMARY KEY, tenant_id int, knowledge_base_id text, knowledge_id text, tag_id text, created_at timestamptz, deleted_at timestamptz);`)
|
||||
|
||||
await t.test('removes only two fixture tags and one relation, retaining all soft-delete history', () => {
|
||||
seed()
|
||||
const before = snapshot()
|
||||
const result = query(buildCleanupSQL(fixture(), now)).split('\n').map(JSON.parse)
|
||||
assert.deepEqual(result, [{kind: 'relations', deleted: 1}, {kind: 'tags', deleted: 2}])
|
||||
const after = snapshot()
|
||||
for (const table of ['users', 'knowledge_bases', 'knowledges', 'chunks']) assert.deepEqual(after[table], before[table])
|
||||
assert.deepEqual(after.knowledge_tags, before.knowledge_tags.filter(row => row.id === id(106)))
|
||||
assert.deepEqual(after.knowledge_tag_relations, before.knowledge_tag_relations.filter(row => row.tag_id === id(106)))
|
||||
assert.deepEqual(query(buildCleanupSQL(fixture(), now)).split('\n').map(JSON.parse), [{kind: 'relations', deleted: 0}, {kind: 'tags', deleted: 0}])
|
||||
assert.deepEqual(snapshot(), after)
|
||||
})
|
||||
|
||||
for (const [name, sql] of [
|
||||
['wrong user tenant', `UPDATE users SET tenant_id=1 WHERE id='${id(1)}'`],
|
||||
['wrong owner username', `UPDATE users SET username='real-user-2' WHERE id='${id(1)}'`],
|
||||
['wrong owner email', `UPDATE users SET email='real@example.org' WHERE id='${id(1)}'`],
|
||||
['wrong reader identity', `UPDATE users SET username='other-reader' WHERE id='${id(2)}'`],
|
||||
['owner outside window', `UPDATE users SET created_at='2026-09-01' WHERE id='${id(1)}'`],
|
||||
['missing owner', `DELETE FROM users WHERE id='${id(1)}'`],
|
||||
['wrong KB tenant', `UPDATE knowledge_bases SET tenant_id=1 WHERE id='${id(4)}'`],
|
||||
['wrong KB owner', `UPDATE knowledge_bases SET creator_id='${id(101)}' WHERE id='${id(4)}'`],
|
||||
['wrong KB name', `UPDATE knowledge_bases SET name='Real knowledge 2' WHERE id='${id(4)}'`],
|
||||
['active KB', `UPDATE knowledge_bases SET deleted_at=NULL WHERE id='${id(4)}'`],
|
||||
['KB outside window', `UPDATE knowledge_bases SET created_at='2026-09-01' WHERE id='${id(4)}'`],
|
||||
['missing KB history', `DELETE FROM knowledge_bases WHERE id='${id(4)}'`],
|
||||
['wrong document tenant', `UPDATE knowledges SET tenant_id=1 WHERE id='${id(5)}'`],
|
||||
['wrong document KB', `UPDATE knowledges SET knowledge_base_id='${id(104)}' WHERE id='${id(5)}'`],
|
||||
['wrong document title', `UPDATE knowledges SET title='Real file' WHERE id='${id(5)}'`],
|
||||
['wrong document file', `UPDATE knowledges SET file_name='real.txt' WHERE id='${id(5)}'`],
|
||||
['wrong document type', `UPDATE knowledges SET type='url' WHERE id='${id(5)}'`],
|
||||
['active document', `UPDATE knowledges SET deleted_at=NULL WHERE id='${id(5)}'`],
|
||||
['document outside window', `UPDATE knowledges SET created_at='2026-09-01' WHERE id='${id(5)}'`],
|
||||
['missing document history', `DELETE FROM knowledges WHERE id='${id(5)}'`],
|
||||
['unreported document', `UPDATE knowledges SET knowledge_base_id='${id(4)}' WHERE id='${id(105)}'`],
|
||||
['wrong tag tenant', `UPDATE knowledge_tags SET tenant_id=1 WHERE id='${id(6)}'`],
|
||||
['wrong tag KB', `UPDATE knowledge_tags SET knowledge_base_id='${id(104)}' WHERE id='${id(6)}'`],
|
||||
['wrong tag name', `UPDATE knowledge_tags SET name='Real category 2' WHERE id='${id(6)}'`],
|
||||
['null tag metadata', `UPDATE knowledge_tags SET name=NULL WHERE id='${id(6)}'`],
|
||||
['tag outside window', `UPDATE knowledge_tags SET created_at='2026-09-01' WHERE id='${id(6)}'`],
|
||||
['unreported tag', `UPDATE knowledge_tags SET knowledge_base_id='${id(4)}' WHERE id='${id(106)}'`],
|
||||
['unrelated document relation', `INSERT INTO knowledge_tag_relations VALUES ('${id(105)}','${id(6)}','${created}')`],
|
||||
['unreported relation tag', `INSERT INTO knowledge_tag_relations VALUES ('${id(5)}','${id(106)}','${created}')`],
|
||||
['relation without retained tag', `DELETE FROM knowledge_tags WHERE id='${id(6)}'`],
|
||||
['relation outside window', `UPDATE knowledge_tag_relations SET created_at='2026-09-01' WHERE tag_id='${id(6)}'`],
|
||||
['active chunk', `UPDATE chunks SET deleted_at=NULL WHERE id='${id(8)}'`],
|
||||
['unrelated chunk with fixture tag', `UPDATE chunks SET tag_id='${id(6)}' WHERE id='${id(108)}'`],
|
||||
['fixture chunk with external tag', `UPDATE chunks SET tag_id='${id(106)}' WHERE id='${id(8)}'`],
|
||||
['wrong chunk tenant', `UPDATE chunks SET tenant_id=1 WHERE id='${id(8)}'`],
|
||||
['chunk outside window', `UPDATE chunks SET created_at='2026-09-01' WHERE id='${id(8)}'`],
|
||||
]) {
|
||||
await t.test(`rolls back without deleting anything for ${name}`, () => {
|
||||
seed()
|
||||
query(sql)
|
||||
const before = snapshot()
|
||||
assert.throws(() => query(buildCleanupSQL(fixture(), now)), /ERROR:/)
|
||||
assert.deepEqual(snapshot(), before)
|
||||
})
|
||||
}
|
||||
|
||||
for (const [name, mutate] of [
|
||||
['no fixtures yet', r => {for (const key of Object.keys(r.fixtures)) r.fixtures[key] = []; r.cleanup = []}],
|
||||
['owner created only', r => {r.fixtures.users.pop(); r.fixtures.kbs = []; r.fixtures.tags = []; r.fixtures.docs = []; r.cleanup = []}],
|
||||
['KB created without tags', r => {r.fixtures.tags = []; r.fixtures.docs = []}],
|
||||
['first tag created before failure', r => {r.fixtures.tags.pop(); r.fixtures.docs = []}],
|
||||
['two tags before upload failure', r => {r.fixtures.docs = []}],
|
||||
]) {
|
||||
await t.test(`supports partial fixture: ${name}`, () => {
|
||||
const report = fixture()
|
||||
mutate(report)
|
||||
seed(report)
|
||||
assert.doesNotThrow(() => query(buildCleanupSQL(report, now)))
|
||||
assert.equal(snapshot().knowledge_tags.length, 1)
|
||||
assert.equal(snapshot().knowledge_tag_relations.length, 1)
|
||||
})
|
||||
}
|
||||
|
||||
await t.test('supports one previously removed empty tag', () => {
|
||||
seed()
|
||||
query(`DELETE FROM knowledge_tags WHERE id='${id(7)}'`)
|
||||
assert.deepEqual(query(buildCleanupSQL(fixture(), now)).split('\n').map(JSON.parse), [{kind: 'relations', deleted: 1}, {kind: 'tags', deleted: 1}])
|
||||
})
|
||||
|
||||
await t.test('rolls back relation removal when the later tag delete fails', () => {
|
||||
seed()
|
||||
const before = snapshot()
|
||||
query(`CREATE FUNCTION refuse_tag_delete() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'test tag delete failure'; END $$;
|
||||
CREATE TRIGGER refuse_tag_delete BEFORE DELETE ON knowledge_tags FOR EACH ROW EXECUTE FUNCTION refuse_tag_delete();`)
|
||||
try {
|
||||
assert.throws(() => query(buildCleanupSQL(fixture(), now)), /test tag delete failure/)
|
||||
assert.deepEqual(snapshot(), before)
|
||||
} finally { query('DROP TRIGGER refuse_tag_delete ON knowledge_tags; DROP FUNCTION refuse_tag_delete();') }
|
||||
})
|
||||
} finally {
|
||||
if (running) run('pg_ctl', ['-D', data, '-m', 'immediate', '-w', 'stop'])
|
||||
fs.rmSync(directory, {recursive: true, force: true})
|
||||
}
|
||||
})
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install WeKnora git hooks (core.hooksPath -> scripts/git-hooks).
|
||||
#
|
||||
# Usage (from repo root):
|
||||
# ./scripts/install-git-hooks.sh
|
||||
#
|
||||
# Bypass hooks for one command:
|
||||
# SKIP_HOOKS=1 git commit ...
|
||||
# SKIP_HOOKS=1 git push ...
|
||||
#
|
||||
# Skip slow go test on pre-push:
|
||||
# HOOK_SKIP_TEST=1 git push ...
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
HOOKS_DIR="$ROOT/scripts/git-hooks"
|
||||
|
||||
if [[ ! -d "$HOOKS_DIR" ]]; then
|
||||
echo "hooks directory not found: $HOOKS_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
chmod +x "$HOOKS_DIR"/common.sh "$HOOKS_DIR"/pre-commit "$HOOKS_DIR"/pre-push
|
||||
|
||||
git -C "$ROOT" config core.hooksPath scripts/git-hooks
|
||||
|
||||
echo "Installed git hooks -> scripts/git-hooks"
|
||||
echo " pre-commit: whitespace, gofmt (auto-fix), golangci-lint (if installed)"
|
||||
echo " pre-push: PR gofmt + full go vet/build (like CI); scoped go test"
|
||||
echo ""
|
||||
echo "Optional: SKIP_HOOKS=1 or HOOK_SKIP_TEST=1 — see script header."
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# Read-only preflight for the legacy SQLite -> native WeKnora migration.
|
||||
# It never writes to the source database and never imports data. The output is
|
||||
# intentionally small enough to attach to a migration review.
|
||||
DB_PATH=${LEGACY_DB_PATH:-/Volumes/Extreme\ SSD/yawan-storage/yawan-workbench/server/data/workspace.db}
|
||||
if [ ! -f "$DB_PATH" ]; then
|
||||
DB_PATH=${LEGACY_DB_PATH_FALLBACK:-/Users/macclaw/Desktop/yawan-workbench/server/data-dev/workspace.db}
|
||||
fi
|
||||
if [ ! -f "$DB_PATH" ]; then
|
||||
echo "legacy database not found; set LEGACY_DB_PATH" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
command -v sqlite3 >/dev/null 2>&1 || { echo "sqlite3 is required" >&2; exit 2; }
|
||||
|
||||
sha=$(shasum -a 256 "$DB_PATH" | awk '{print $1}')
|
||||
size=$(wc -c < "$DB_PATH" | tr -d ' ')
|
||||
integrity=$(sqlite3 "$DB_PATH" 'PRAGMA integrity_check;' | tr '\n' ' ' | sed 's/[[:space:]]*$//')
|
||||
if [ "$integrity" != "ok" ]; then
|
||||
echo "legacy database integrity check failed: $integrity" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
count() {
|
||||
table=$1
|
||||
# sqlite3 exits non-zero for a missing table; surface that as zero only for
|
||||
# optional legacy tables so an audit can run against older snapshots.
|
||||
sqlite3 "$DB_PATH" "SELECT count(*) FROM \"$table\";" 2>/dev/null || echo 0
|
||||
}
|
||||
|
||||
printf '{\n'
|
||||
printf ' "path": %s,\n' "$(printf '%s' "$DB_PATH" | awk '{gsub(/\\/, "\\\\"); gsub(/"/, "\\\""); printf "\"%s\"", $0}')"
|
||||
printf ' "bytes": %s,\n' "$size"
|
||||
printf ' "sha256": "%s",\n' "$sha"
|
||||
printf ' "integrity": "ok",\n'
|
||||
printf ' "counts": {\n'
|
||||
tables='appuser knowledgebase knowledgefile knowledgecategory agent agentskill agentskillbinding canvas trainingmap segment digitalhuman memory knowledgeusage pointaccount pointtransaction auditlog systemlog'
|
||||
first=1
|
||||
for table in $tables; do
|
||||
[ "$first" -eq 1 ] || printf ',\n'
|
||||
first=0
|
||||
printf ' "%s": %s' "$table" "$(count "$table")"
|
||||
done
|
||||
printf '\n }\n}\n'
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
const database = process.env.GRAPH_MIGRATION_DATABASE || 'yawan_native_acceptance_20260915';
|
||||
if (!/^yawan_native_acceptance_[0-9]+$/.test(database)) throw new Error('Only isolated acceptance databases are allowed');
|
||||
if (!process.argv.includes('--apply')) throw new Error('Usage: node scripts/map_native_graph_evidence.mjs --apply');
|
||||
const tenant = Number(process.env.GRAPH_MIGRATION_TENANT || 10059);
|
||||
if (!Number.isSafeInteger(tenant) || tenant < 1) throw new Error('Invalid tenant');
|
||||
const base = ['--context','colima-yawan','exec','-i','WeKnora-postgres','psql','-X','-v','ON_ERROR_STOP=1','-U','postgres','-d',database,'-At'];
|
||||
const quote = value => `'${String(value).replaceAll("'", "''")}'`;
|
||||
function sql(query) {
|
||||
const result = spawnSync('docker',base,{input:query,encoding:'utf8',maxBuffer:256*1024*1024});
|
||||
if(result.status!==0) throw new Error(result.stderr);
|
||||
return result.stdout.trim().split('\n').filter(Boolean);
|
||||
}
|
||||
const normalize = value => value.normalize('NFKC').replace(/\s+/gu,'');
|
||||
const documents = sql(`WITH candidates AS (SELECT k.id,k.knowledge_base_id,s.source_file,count(*) OVER (PARTITION BY s.knowledge_base_id,s.source_file) AS matches FROM (SELECT DISTINCT knowledge_base_id,source_file FROM native_graph_sources WHERE tenant_id=${tenant}) s JOIN knowledges k ON k.tenant_id=${tenant} AND k.knowledge_base_id=s.knowledge_base_id AND k.file_name=regexp_replace(s.source_file,'\\.jsonl$','.md') AND k.deleted_at IS NULL) SELECT json_build_object('id',id,'kb',knowledge_base_id,'file',source_file) FROM candidates WHERE matches=1 ORDER BY id;`).map(JSON.parse);
|
||||
let mapped = 0, ambiguous = 0;
|
||||
for (const document of documents) {
|
||||
const rows=sql(`SELECT json_build_object('id',source_chunk_id,'text',excerpt) FROM native_graph_sources WHERE tenant_id=${tenant} AND knowledge_base_id=${quote(document.kb)} AND source_file=${quote(document.file)} AND native_chunk_id IS NULL;`).map(JSON.parse);
|
||||
const prefixes=new Map();
|
||||
for (const row of rows) {
|
||||
row.normalized=normalize(row.text);row.matches=[];
|
||||
if([...row.normalized].length<80)continue;
|
||||
const key=row.normalized.slice(0,16);
|
||||
if(!prefixes.has(key))prefixes.set(key,[]);
|
||||
prefixes.get(key).push(row);
|
||||
}
|
||||
if(!prefixes.size)continue;
|
||||
const chunks=sql(`SELECT json_build_object('id',id,'content',content) FROM chunks WHERE tenant_id=${tenant} AND knowledge_base_id=${quote(document.kb)} AND knowledge_id=${quote(document.id)} AND chunk_type='text' AND deleted_at IS NULL AND is_enabled=true ORDER BY id;`).map(JSON.parse);
|
||||
for(const chunk of chunks) {
|
||||
const content=normalize(chunk.content);
|
||||
const seen=new Set();
|
||||
for(let pos=0;pos<=content.length-16;pos++) {
|
||||
const candidates=prefixes.get(content.slice(pos,pos+16));
|
||||
if(!candidates)continue;
|
||||
for(const row of candidates) {
|
||||
if(seen.has(row.id)||row.matches.length>1)continue;
|
||||
if(content.startsWith(row.normalized,pos)) {
|
||||
row.matches.push({id:chunk.id,hash:createHash('sha256').update(chunk.content).digest('hex')});seen.add(row.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const matches=rows.filter(row=>row.matches.length===1);
|
||||
ambiguous+=rows.filter(row=>row.matches.length>1).length;
|
||||
if(matches.length) {
|
||||
const values=matches.map(row=>`(${quote(row.id)},${quote(row.matches[0].id)},${quote(row.matches[0].hash)})`).join(',\n');
|
||||
sql(`BEGIN; UPDATE native_graph_sources s SET native_knowledge_id=${quote(document.id)},native_chunk_id=m.chunk_id,native_content_sha256=m.hash,mapping_state='normalized_full_excerpt' FROM (VALUES ${values}) m(source_id,chunk_id,hash) WHERE s.tenant_id=${tenant} AND s.knowledge_base_id=${quote(document.kb)} AND s.source_file=${quote(document.file)} AND s.source_chunk_id=m.source_id AND s.native_chunk_id IS NULL; COMMIT;`);
|
||||
}
|
||||
mapped+=matches.length;
|
||||
console.log(JSON.stringify({file:document.file,candidates:rows.length,mapped:matches.length}));
|
||||
}
|
||||
console.log(JSON.stringify({mapped,ambiguous,normalization:'Unicode NFKC and whitespace removal; entire excerpt >= 80 codepoints; unique native chunk in matching tenant, KB and source document'}));
|
||||
Executable
+122
@@ -0,0 +1,122 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Get the script directory and project root
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
|
||||
|
||||
# Load .env file if it exists (for development mode)
|
||||
if [ -f "$PROJECT_ROOT/.env" ]; then
|
||||
echo "Loading .env file from $PROJECT_ROOT/.env"
|
||||
set -a
|
||||
source "$PROJECT_ROOT/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# Database connection details (can be overridden by environment variables)
|
||||
DB_HOST=${DB_HOST:-localhost}
|
||||
DB_PORT=${DB_PORT:-5432}
|
||||
DB_USER=${DB_USER:-postgres}
|
||||
DB_PASSWORD=${DB_PASSWORD:-postgres}
|
||||
DB_NAME=${DB_NAME:-WeKnora}
|
||||
|
||||
# Use versioned migrations directory
|
||||
MIGRATIONS_DIR="${MIGRATIONS_DIR:-migrations/versioned}"
|
||||
|
||||
# Check if migrate tool is installed
|
||||
if ! command -v migrate &> /dev/null; then
|
||||
echo "Error: migrate tool is not installed"
|
||||
echo "Install it with: go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Construct the database URL
|
||||
# If DB_URL is already set in .env, use it but ensure sslmode=disable is set
|
||||
# Otherwise, construct it from individual components
|
||||
if [ -n "$DB_URL" ]; then
|
||||
# If DB_URL already exists, ensure sslmode=disable is set (unless sslmode is already specified)
|
||||
if [[ "$DB_URL" != *"sslmode="* ]]; then
|
||||
# Add sslmode=disable if not present
|
||||
if [[ "$DB_URL" == *"?"* ]]; then
|
||||
DB_URL="${DB_URL}&sslmode=disable"
|
||||
else
|
||||
DB_URL="${DB_URL}?sslmode=disable"
|
||||
fi
|
||||
elif [[ "$DB_URL" == *"sslmode=require"* ]] || [[ "$DB_URL" == *"sslmode=prefer"* ]]; then
|
||||
# Replace sslmode=require/prefer with sslmode=disable for local dev
|
||||
DB_URL="${DB_URL//sslmode=require/sslmode=disable}"
|
||||
DB_URL="${DB_URL//sslmode=prefer/sslmode=disable}"
|
||||
fi
|
||||
else
|
||||
# Use Python to properly URL encode password if it contains special characters
|
||||
# This handles special characters in passwords correctly
|
||||
if command -v python3 &> /dev/null; then
|
||||
ENCODED_PASSWORD=$(python3 -c "import urllib.parse; print(urllib.parse.quote('$DB_PASSWORD', safe=''))")
|
||||
else
|
||||
# Fallback: try to use printf for basic encoding (may not work for all special chars)
|
||||
ENCODED_PASSWORD="$DB_PASSWORD"
|
||||
fi
|
||||
DB_URL="postgres://${DB_USER}:${ENCODED_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable"
|
||||
fi
|
||||
|
||||
# Execute migration based on command
|
||||
case "$1" in
|
||||
up)
|
||||
echo "Running migrations up..."
|
||||
echo "DB_URL: ${DB_URL}"
|
||||
echo "DB_USER: ${DB_USER}"
|
||||
echo "DB_PASSWORD: ${DB_PASSWORD}"
|
||||
echo "DB_HOST: ${DB_HOST}"
|
||||
echo "DB_PORT: ${DB_PORT}"
|
||||
echo "DB_NAME: ${DB_NAME}"
|
||||
echo "MIGRATIONS_DIR: ${MIGRATIONS_DIR}"
|
||||
migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} up
|
||||
;;
|
||||
down)
|
||||
echo "Running migrations down..."
|
||||
migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} down
|
||||
;;
|
||||
create)
|
||||
if [ -z "$2" ]; then
|
||||
echo "Error: Migration name is required"
|
||||
echo "Usage: $0 create <migration_name>"
|
||||
exit 1
|
||||
fi
|
||||
echo "Creating migration files for $2..."
|
||||
migrate create -ext sql -dir ${MIGRATIONS_DIR} -seq $2
|
||||
echo "Created:"
|
||||
echo " - ${MIGRATIONS_DIR}/$(ls -t ${MIGRATIONS_DIR} | head -1)"
|
||||
echo " - ${MIGRATIONS_DIR}/$(ls -t ${MIGRATIONS_DIR} | head -2 | tail -1)"
|
||||
;;
|
||||
version)
|
||||
echo "Checking current migration version..."
|
||||
migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} version
|
||||
;;
|
||||
force)
|
||||
if [ -z "$2" ]; then
|
||||
echo "Error: Version number is required"
|
||||
echo "Usage: $0 force <version>"
|
||||
echo "Note: Use -1 to reset to no version (allows re-running all migrations)"
|
||||
exit 1
|
||||
fi
|
||||
VERSION="$2"
|
||||
echo "Forcing migration version to $VERSION..."
|
||||
# Use env to pass the command, avoiding shell flag parsing issues with negative numbers
|
||||
env migrate -path "${MIGRATIONS_DIR}" -database "${DB_URL}" force -- "$VERSION"
|
||||
;;
|
||||
goto)
|
||||
if [ -z "$2" ]; then
|
||||
echo "Error: Version number is required"
|
||||
echo "Usage: $0 goto <version>"
|
||||
exit 1
|
||||
fi
|
||||
echo "Migrating to version $2..."
|
||||
migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} goto $2
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 {up|down|create <migration_name>|version|force <version>|goto <version>}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "Migration command completed successfully"
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { once } from 'node:events';
|
||||
import { createInterface } from 'node:readline';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const snapshot = process.argv[2];
|
||||
if (!snapshot || !process.argv.includes('--apply')) throw new Error('Usage: node scripts/migrate_native_graph.mjs SNAPSHOT --apply');
|
||||
const database = process.env.GRAPH_MIGRATION_DATABASE || 'yawan_native_acceptance_20260915';
|
||||
if (!/^yawan_native_acceptance_[0-9]+$/.test(database)) throw new Error('Only isolated acceptance databases are allowed');
|
||||
const tenant = Number(process.env.GRAPH_MIGRATION_TENANT || 10059);
|
||||
if (!Number.isSafeInteger(tenant) || tenant < 1) throw new Error('Invalid tenant');
|
||||
const docker = ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres', 'psql', '-X', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', database];
|
||||
const hash = createHash('sha256');
|
||||
for await (const part of createReadStream(snapshot)) hash.update(part);
|
||||
const digest = hash.digest('hex');
|
||||
const sqlite = (sql) => {
|
||||
const result = spawnSync('sqlite3', ['-readonly', '-json', snapshot, sql], {encoding:'utf8', maxBuffer:8*1024*1024});
|
||||
if (result.status !== 0) throw new Error(result.stderr);
|
||||
return JSON.parse(result.stdout || '[]');
|
||||
};
|
||||
const quote = value => `'${String(value).replaceAll("'", "''")}'`;
|
||||
const sourceKBs = sqlite('SELECT id,name FROM knowledgebase ORDER BY id');
|
||||
const proc = spawn('docker', docker, {stdio:['pipe','pipe','pipe']});
|
||||
proc.stdout.pipe(process.stdout); proc.stderr.pipe(process.stderr);
|
||||
const completion = once(proc, 'close');
|
||||
const write = async data => {if (!proc.stdin.write(data)) await once(proc.stdin,'drain');};
|
||||
await write('BEGIN; SET statement_timeout = 0;\n');
|
||||
await write(await (await import('node:fs/promises')).readFile(fileURLToPath(new URL('../migrations/versioned/000200_native_graph.up.sql', import.meta.url)), 'utf8'));
|
||||
await write(`\nCREATE TEMP TABLE graph_kb_map(legacy_id bigint PRIMARY KEY,native_id varchar(36) NOT NULL);\n`);
|
||||
for (const kb of sourceKBs) {
|
||||
await write(`DO $$ BEGIN IF (SELECT count(*) FROM knowledge_bases WHERE tenant_id=${tenant} AND name=${quote(kb.name)} AND deleted_at IS NULL)<>1 THEN RAISE EXCEPTION 'Ambiguous or missing knowledge base ${kb.id}'; END IF; END $$;\nINSERT INTO graph_kb_map SELECT ${kb.id},id FROM knowledge_bases WHERE tenant_id=${tenant} AND name=${quote(kb.name)} AND deleted_at IS NULL;\n`);
|
||||
}
|
||||
const tables = ['knowledgegraphversion','knowledgeentity','knowledgerelation','knowledgegraphevidence','knowledgegraphchunkstate','knowledgegraphsourceunit','knowledgegraphclaim','knowledgeentityalias','knowledgegraphcommunity','knowledgegraphcommunityreport'];
|
||||
const expected = {};
|
||||
for (const table of tables) {
|
||||
const columns = sqlite(`PRAGMA table_info(${table})`).map(c=>c.name);
|
||||
expected[table] = sqlite(`SELECT count(*) AS n FROM ${table}`)[0].n;
|
||||
const sql = `SELECT json_object(${columns.flatMap(c=>[quote(c), '"'+c+'"']).join(',')}) FROM ${table} ORDER BY id;`;
|
||||
await write(`CREATE TEMP TABLE graph_import_${table}(payload jsonb);\nCOPY graph_import_${table}(payload) FROM STDIN WITH (FORMAT csv);\n`);
|
||||
const source = spawn('sqlite3',['-readonly','-noheader',snapshot,sql],{stdio:['ignore','pipe','pipe']});
|
||||
const sourceCompletion = once(source,'close');
|
||||
source.stderr.pipe(process.stderr);
|
||||
for await (const line of createInterface({input:source.stdout,crlfDelay:Infinity})) await write('"'+line.replaceAll('"','""')+'"\n');
|
||||
const [sourceCode]=await sourceCompletion;
|
||||
if(sourceCode!==0){proc.kill();throw new Error(`SQLite export failed: ${table}`);}
|
||||
await write(`\\.\nDO $$ BEGIN IF EXISTS(SELECT 1 FROM graph_import_${table} s JOIN graph_kb_map k ON k.legacy_id=(s.payload->>'kb_id')::bigint JOIN native_graph_records r ON r.knowledge_base_id=k.native_id AND r.tenant_id=${tenant} AND r.source_table=${quote(table)} AND r.legacy_id=(s.payload->>'id')::bigint WHERE r.payload<>s.payload) THEN RAISE EXCEPTION 'Source record differs from preserved graph archive'; END IF; END $$;\nINSERT INTO native_graph_records(tenant_id,knowledge_base_id,source_table,legacy_id,graph_version_id,payload,source_snapshot_sha256) SELECT ${tenant},k.native_id,${quote(table)},(s.payload->>'id')::bigint,COALESCE((s.payload->>'graph_version_id')::bigint,(s.payload->>'id')::bigint),s.payload,${quote(digest)} FROM graph_import_${table} s JOIN graph_kb_map k ON k.legacy_id=(s.payload->>'kb_id')::bigint ON CONFLICT DO NOTHING;\nDO $$ BEGIN IF (SELECT count(*) FROM native_graph_records WHERE tenant_id=${tenant} AND knowledge_base_id IN (SELECT native_id FROM graph_kb_map) AND source_table=${quote(table)})<>${expected[table]} THEN RAISE EXCEPTION 'Graph count mismatch ${table}'; END IF; END $$;\nDROP TABLE graph_import_${table};\n`);
|
||||
}
|
||||
await write(`
|
||||
DO $$ BEGIN IF EXISTS(SELECT 1 FROM native_graph_records WHERE tenant_id=${tenant} AND knowledge_base_id IN (SELECT native_id FROM graph_kb_map) AND source_table='knowledgegraphevidence' GROUP BY knowledge_base_id,payload->>'source_chunk_id' HAVING count(DISTINCT payload->>'source_content_hash')>1) THEN RAISE EXCEPTION 'Conflicting source content hashes'; END IF; END $$;
|
||||
INSERT INTO native_graph_sources(tenant_id,knowledge_base_id,source_chunk_id,source_file,source_content_hash,excerpt)
|
||||
SELECT DISTINCT ON (knowledge_base_id,payload->>'source_chunk_id') tenant_id,knowledge_base_id,payload->>'source_chunk_id',payload->>'source_file',payload->>'source_content_hash',payload->>'excerpt'
|
||||
FROM native_graph_records WHERE tenant_id=${tenant} AND knowledge_base_id IN (SELECT native_id FROM graph_kb_map) AND source_table='knowledgegraphevidence' ORDER BY knowledge_base_id,payload->>'source_chunk_id',length(payload->>'excerpt') DESC,legacy_id ON CONFLICT DO NOTHING;
|
||||
CREATE TEMP TABLE graph_native_chunks AS SELECT c.id,c.knowledge_id,c.knowledge_base_id,k.file_name,encode(sha256(convert_to(c.content,'UTF8')),'hex') AS content_hash FROM chunks c JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id AND k.deleted_at IS NULL WHERE c.tenant_id=${tenant} AND c.knowledge_base_id IN (SELECT native_id FROM graph_kb_map) AND c.deleted_at IS NULL AND c.is_enabled=true AND c.chunk_type='text';
|
||||
CREATE INDEX ON graph_native_chunks(knowledge_base_id,content_hash);
|
||||
ANALYZE graph_native_chunks;
|
||||
UPDATE native_graph_sources s SET native_chunk_id=m.id,native_knowledge_id=m.knowledge_id,native_content_sha256=m.content_hash,mapping_state='exact_content_hash' FROM (SELECT * FROM (SELECT *,count(*) OVER (PARTITION BY knowledge_base_id,file_name,content_hash) AS matches FROM graph_native_chunks) candidates WHERE matches=1) m WHERE s.tenant_id=${tenant} AND s.knowledge_base_id=m.knowledge_base_id AND regexp_replace(s.source_file,'\\.jsonl$','.md')=m.file_name AND s.source_content_hash=m.content_hash AND s.native_chunk_id IS NULL;
|
||||
COMMIT;
|
||||
VACUUM (ANALYZE) native_graph_records, native_graph_sources;
|
||||
SELECT source_table,count(*) AS preserved FROM native_graph_records WHERE tenant_id=${tenant} GROUP BY source_table ORDER BY source_table;
|
||||
SELECT mapping_state,count(*) AS source_chunks FROM native_graph_sources WHERE tenant_id=${tenant} GROUP BY mapping_state;
|
||||
`);
|
||||
proc.stdin.end();
|
||||
const [code] = await completion;
|
||||
if(code!==0)throw new Error(`PostgreSQL import failed (${code}); transaction rolled back`);
|
||||
console.log(JSON.stringify({snapshot_sha256:digest,tenant,database,expected}));
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only vector copy audit and distinct-query acceptance latency sample.
|
||||
|
||||
Reports only counts, checksums, and timings; never knowledge prose or credentials.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from milvus_copy_acceptance import FIELDS, post
|
||||
from milvus_verify_acceptance import (ACCEPTANCE, Client, VerificationError,
|
||||
inspect_results, read_object, search_payload)
|
||||
|
||||
SOURCE = 'weknora_embeddings_512'
|
||||
TARGET = 'yawan_acceptance_hybrid_512'
|
||||
MILVUS = 'http://127.0.0.1:19530'
|
||||
QUERIES = {
|
||||
'1': ['产品资料应如何整理和解释', '介绍产品时需要说明哪些使用方法'],
|
||||
'2': ['如何安排由浅入深的练习流程', '基础练习中怎样判断动作是否正确'],
|
||||
'3': ['怎样通过沟通理解对方的需求', '遇到紧张情绪时如何调整表达方式'],
|
||||
'4': ['如何描述亲密关系中的感受和期待', '怎样建立双方舒适的互动氛围'],
|
||||
'5': ['学习新技巧时如何分解具体步骤', '练习过程有哪些常见错误需要纠正'],
|
||||
'6': ['如何理解客户需求并解释产品价值', '怎样回应疑问并帮助客户作出选择'],
|
||||
}
|
||||
|
||||
|
||||
def query(collection, filter_text, fields, limit=100):
|
||||
return post(MILVUS, 'entities/query', {
|
||||
'collectionName': collection, 'filter': filter_text,
|
||||
'outputFields': fields, 'limit': limit, 'consistencyLevel': 'Strong',
|
||||
})
|
||||
|
||||
|
||||
def digest(row):
|
||||
return hashlib.sha256(json.dumps({field: row[field] for field in FIELDS},
|
||||
sort_keys=True, ensure_ascii=False,
|
||||
separators=(',', ':')).encode()).hexdigest()
|
||||
|
||||
|
||||
def run():
|
||||
state = read_object(ACCEPTANCE / 'legacy-migration-state.json', 'Migration state')
|
||||
copy = read_object(ACCEPTANCE / 'hybrid-copy-state.json', 'Copy state')
|
||||
manifest = json.loads((ACCEPTANCE / 'hybrid-chunk-manifest.json').read_text())
|
||||
report = {'source': SOURCE, 'target': TARGET, 'storage': [], 'queries': [],
|
||||
'scope': 'Six migrated business knowledge bases only',
|
||||
'latency_scope': 'Sequential distinct queries, warmed service; cache not cleared or instrumented',
|
||||
'started_at': time.time()}
|
||||
business_ids = {kb['id'] for kb in state['knowledge_bases'].values()}
|
||||
missing_ids = set(copy['missing'])
|
||||
business_missing = [row['id'] for row in manifest
|
||||
if row['id'] in missing_ids and row['knowledge_base_id'] in business_ids]
|
||||
report['missing_business_chunks'] = len(business_missing)
|
||||
report['missing_historical_test_chunks'] = len(missing_ids) - len(business_missing)
|
||||
for legacy_id, kb in sorted(state['knowledge_bases'].items()):
|
||||
entries = [row for row in manifest if row['knowledge_base_id'] == kb['id']]
|
||||
filter_text = 'knowledge_base_id == ' + json.dumps(kb['id'])
|
||||
counts = {name: int(query(name, filter_text, ['count(*)'])[0]['count(*)'])
|
||||
for name in (SOURCE, TARGET)}
|
||||
positions = sorted({0, len(entries) // 2, len(entries) - 1})
|
||||
ids = [entries[index]['id'] for index in positions]
|
||||
samples = {}
|
||||
for name in (SOURCE, TARGET):
|
||||
rows = query(name, 'chunk_id in ' + json.dumps(ids), FIELDS, len(ids))
|
||||
samples[name] = {row['chunk_id']: digest(row) for row in rows}
|
||||
equal = (set(samples[SOURCE]) == set(ids) and samples[SOURCE] == samples[TARGET])
|
||||
report['storage'].append({
|
||||
'legacy_kb': legacy_id, 'kb_id': kb['id'], 'expected_chunks': len(entries),
|
||||
'source_count': counts[SOURCE], 'target_count': counts[TARGET],
|
||||
'sample_count': len(ids), 'sample_fields_equal': equal,
|
||||
'passed': equal and all(count == len(entries) for count in counts.values()),
|
||||
})
|
||||
credentials = read_object(ACCEPTANCE / 'migration-token.json', 'Migration credentials')
|
||||
client = Client(45)
|
||||
status, login, _ = client.request('/api/v1/auth/login', payload={
|
||||
'email': credentials['email'], 'password': credentials['password'],
|
||||
})
|
||||
if status != 200 or login.get('active_tenant', {}).get('id') != state['tenant_id']:
|
||||
raise VerificationError('Unexpected login scope')
|
||||
for legacy_id, queries in sorted(QUERIES.items()):
|
||||
kb_id = state['knowledge_bases'][legacy_id]['id']
|
||||
for index, text in enumerate(queries):
|
||||
status, result, latency = client.request(
|
||||
'/api/v1/knowledge-bases/' + kb_id + '/hybrid-search', login['token'],
|
||||
search_payload(text, 'hybrid', 10))
|
||||
check = inspect_results(result, kb_id, 10, 'hybrid') if status == 200 else {'passed': False}
|
||||
report['queries'].append({'legacy_kb': legacy_id, 'query_index': index,
|
||||
'query_sha256': hashlib.sha256(text.encode()).hexdigest(),
|
||||
'status': status, 'latency_ms': latency, **check})
|
||||
latencies = sorted(row['latency_ms'] for row in report['queries'])
|
||||
report['latency_ms'] = {'min': min(latencies), 'median': statistics.median(latencies),
|
||||
'p95_nearest_rank': latencies[math.ceil(len(latencies) * .95) - 1],
|
||||
'max': max(latencies), 'count': len(latencies)}
|
||||
report['passed'] = (not business_missing and all(row['passed'] for row in report['storage'])
|
||||
and all(row['passed'] for row in report['queries']))
|
||||
report['finished_at'] = time.time()
|
||||
output = ACCEPTANCE / 'milvus-storage-and-latency.json'
|
||||
output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
|
||||
print(json.dumps({key: report[key] for key in (
|
||||
'passed', 'missing_business_chunks', 'missing_historical_test_chunks', 'latency_ms')}, indent=2))
|
||||
return 0 if report['passed'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(run())
|
||||
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Copy existing vectors to an isolated BM25 collection, without re-embedding.
|
||||
|
||||
The manifest is an export of the acceptance DB's chunk IDs. Source collections
|
||||
are read-only; acknowledgements and a manifest digest make retries resumable.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
FIELDS = ['id', 'embedding', 'content', 'source_id', 'source_type', 'chunk_id',
|
||||
'knowledge_id', 'knowledge_base_id', 'tag_id', 'is_enabled']
|
||||
|
||||
|
||||
def post(base, path, payload):
|
||||
request = urllib.request.Request(base + '/v2/vectordb/' + path,
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={'Content-Type': 'application/json'})
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
result = json.load(response)
|
||||
if result.get('code') != 0:
|
||||
raise RuntimeError(f"{path}: {result.get('message', result.get('code'))}")
|
||||
return result.get('data')
|
||||
|
||||
|
||||
def save(path, data):
|
||||
temporary = path.with_suffix('.tmp')
|
||||
temporary.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--manifest', type=Path, required=True)
|
||||
parser.add_argument('--state', type=Path, required=True)
|
||||
parser.add_argument('--target', default='yawan_acceptance_hybrid_512')
|
||||
parser.add_argument('--source', default='weknora_embeddings_512')
|
||||
args = parser.parse_args()
|
||||
if not args.target.startswith('yawan_acceptance_') or args.target == args.source:
|
||||
parser.error('target must be a distinct acceptance collection')
|
||||
base = 'http://127.0.0.1:19530'
|
||||
raw = args.manifest.read_bytes()
|
||||
manifest = json.loads(raw)
|
||||
digest = hashlib.sha256(raw).hexdigest()
|
||||
state = json.loads(args.state.read_text()) if args.state.exists() else {
|
||||
'manifest_sha256': digest, 'source': args.source, 'target': args.target,
|
||||
'offset': 0, 'copied': 0, 'missing': [], 'started_at': time.time(),
|
||||
}
|
||||
for key, value in [('manifest_sha256', digest), ('source', args.source), ('target', args.target)]:
|
||||
if state[key] != value:
|
||||
raise RuntimeError(f'{key} differs from checkpoint')
|
||||
description = post(base, 'collections/describe', {'collectionName': args.target})
|
||||
if 'content_sparse' not in {f['name'] for f in description['fields']}:
|
||||
raise RuntimeError('target must have BM25 schema before copying')
|
||||
for offset in range(state['offset'], len(manifest), 256):
|
||||
batch = manifest[offset:offset + 256]
|
||||
expected = {x['id']: x for x in batch}
|
||||
records = post(base, 'entities/query', {
|
||||
'collectionName': args.source,
|
||||
'filter': 'chunk_id in ' + json.dumps(list(expected)),
|
||||
'outputFields': FIELDS, 'limit': len(batch) * 2,
|
||||
'consistencyLevel': 'Strong',
|
||||
})
|
||||
found = set()
|
||||
for row in records:
|
||||
entry = expected.get(row['chunk_id'])
|
||||
if not entry or row['knowledge_base_id'] != entry['knowledge_base_id'] or row['knowledge_id'] != entry['knowledge_id']:
|
||||
raise RuntimeError('source scope differs from acceptance manifest')
|
||||
if row['chunk_id'] in found:
|
||||
raise RuntimeError('duplicate source vectors for chunk')
|
||||
found.add(row['chunk_id'])
|
||||
if records:
|
||||
result = post(base, 'entities/upsert', {'collectionName': args.target, 'data': records})
|
||||
if result.get('upsertCount') != len(records):
|
||||
raise RuntimeError('target did not acknowledge every vector')
|
||||
state['missing'].extend(sorted(expected.keys() - found))
|
||||
state['copied'] += len(records)
|
||||
state['offset'] = offset + len(batch)
|
||||
state['updated_at'] = time.time()
|
||||
save(args.state, state)
|
||||
if offset % (256 * 20) == 0 or state['offset'] == len(manifest):
|
||||
print(json.dumps({'processed': state['offset'], 'total': len(manifest),
|
||||
'copied': state['copied'], 'missing': len(state['missing'])}), flush=True)
|
||||
state['complete'] = not state['missing']
|
||||
save(args.state, state)
|
||||
print(json.dumps({'complete': state['complete'], 'copied': state['copied'],
|
||||
'missing': len(state['missing'])}), flush=True)
|
||||
return 0 if state['complete'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Executable
+203
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Upload one temporary acceptance document, verify retrieval, then delete it.
|
||||
|
||||
Service mutations use only 127.0.0.1:18080 and the KB created by this run.
|
||||
Milvus access, when enabled, is restricted to read-only entities/query calls.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from milvus_verify_acceptance import (ACCEPTANCE, BASE, MODES, NoRedirect,
|
||||
VerificationError, inspect_results,
|
||||
read_object, search_payload)
|
||||
|
||||
|
||||
class SmokeClient:
|
||||
def __init__(self):
|
||||
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect())
|
||||
self.token = ''
|
||||
self.kb_id = None
|
||||
self.knowledge_id = None
|
||||
|
||||
def request(self, route, method='GET', payload=None, raw=None, content_type=None):
|
||||
kb = '/api/v1/knowledge-bases/' + str(self.kb_id)
|
||||
knowledge = '/api/v1/knowledge/' + str(self.knowledge_id)
|
||||
allowed = {('POST', '/api/v1/auth/login'), ('POST', '/api/v1/knowledge-bases')}
|
||||
if self.kb_id:
|
||||
allowed.update({('POST', kb + '/knowledge/file'), ('POST', kb + '/hybrid-search'),
|
||||
('DELETE', kb), ('GET', kb)})
|
||||
if self.knowledge_id:
|
||||
allowed.add(('GET', knowledge))
|
||||
if (method, route) not in allowed:
|
||||
raise VerificationError('Request is outside the temporary resource allowlist')
|
||||
headers = {'Accept': 'application/json'}
|
||||
if self.token:
|
||||
headers['Authorization'] = 'Bearer ' + self.token
|
||||
if payload is not None:
|
||||
raw = json.dumps(payload).encode()
|
||||
content_type = 'application/json'
|
||||
if content_type:
|
||||
headers['Content-Type'] = content_type
|
||||
started = time.monotonic()
|
||||
request = urllib.request.Request(BASE + route, data=raw, headers=headers, method=method)
|
||||
try:
|
||||
try:
|
||||
response = self.opener.open(request, timeout=45)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
status = response.code
|
||||
body = response.read(8 * 1024 * 1024)
|
||||
value = json.loads(body) if status in (200, 201, 202) else None
|
||||
except (OSError, ValueError):
|
||||
raise VerificationError('Acceptance request failed or timed out') from None
|
||||
return status, value, round((time.monotonic() - started) * 1000, 2)
|
||||
|
||||
def count_vectors(self, collection):
|
||||
if collection not in ('yawan_acceptance_hybrid_512', 'weknora_embeddings_512') or not self.kb_id:
|
||||
raise VerificationError('Milvus query is outside the allowed collections')
|
||||
request = urllib.request.Request('http://127.0.0.1:19530/v2/vectordb/entities/query',
|
||||
data=json.dumps({'collectionName': collection,
|
||||
'filter': 'knowledge_base_id == ' + json.dumps(self.kb_id),
|
||||
'outputFields': ['count(*)'], 'consistencyLevel': 'Strong'}).encode(),
|
||||
headers={'Content-Type': 'application/json'})
|
||||
try:
|
||||
with self.opener.open(request, timeout=30) as response:
|
||||
value = json.load(response)
|
||||
if value.get('code') != 0:
|
||||
raise VerificationError('Milvus count query failed')
|
||||
return int(value['data'][0]['count(*)'])
|
||||
except (OSError, ValueError, KeyError, IndexError):
|
||||
raise VerificationError('Milvus count query failed or returned an invalid shape') from None
|
||||
|
||||
|
||||
def run(args, report):
|
||||
credentials = read_object(ACCEPTANCE / 'migration-token.json', 'Migration credentials')
|
||||
state = read_object(ACCEPTANCE / 'legacy-migration-state.json', 'Migration state')
|
||||
client = SmokeClient()
|
||||
status, login, _ = client.request('/api/v1/auth/login', 'POST',
|
||||
{'email': credentials['email'], 'password': credentials['password']})
|
||||
if status != 200 or login.get('active_tenant', {}).get('id') != state['tenant_id']:
|
||||
raise VerificationError('Login failed or used an unexpected tenant')
|
||||
client.token = login['token']
|
||||
suffix = uuid.uuid4().hex[:12]
|
||||
kb_name = '临时验收-新上传检索-' + suffix
|
||||
try:
|
||||
status, value, latency = client.request('/api/v1/knowledge-bases', 'POST', {
|
||||
'name': kb_name, 'description': 'Temporary acceptance-only upload smoke test',
|
||||
'type': 'document', 'embedding_model_id': state['model_id'],
|
||||
'summary_model_id': '',
|
||||
'chunking_config': {'chunk_size': 512, 'chunk_overlap': 0},
|
||||
'indexing_strategy': {'vector_enabled': True, 'keyword_enabled': True,
|
||||
'wiki_enabled': False, 'graph_enabled': False},
|
||||
'storage_provider_config': {'provider': 'local'},
|
||||
'auto_tag_config': {'enabled': False},
|
||||
})
|
||||
if status != 201 or not isinstance(value.get('data'), dict):
|
||||
raise VerificationError('Temporary knowledge base creation failed')
|
||||
client.kb_id = value['data']['id']
|
||||
report.update(kb_id=client.kb_id, create_ms=latency)
|
||||
migrated_ids = {row['id'] for row in state['knowledge_bases'].values()}
|
||||
if client.kb_id in migrated_ids:
|
||||
client.kb_id = None
|
||||
raise VerificationError('Created KB unexpectedly matched protected migration IDs')
|
||||
if args.verify_milvus:
|
||||
report['vectors_before'] = {name: client.count_vectors(name) for name in
|
||||
('yawan_acceptance_hybrid_512', 'weknora_embeddings_512')}
|
||||
if any(report['vectors_before'].values()):
|
||||
raise VerificationError('Fresh temporary KB unexpectedly already has vectors')
|
||||
boundary = 'acceptance' + uuid.uuid4().hex
|
||||
document = ('# 资料归档演练\n\n编号:' + suffix + '\n\n'
|
||||
'资料归档采用三步流程:先登记文件名称,再核对目录分类,最后确认文件可以检索。'
|
||||
'检索测试使用资料归档和目录核对两个关键词。完成演练后删除临时资料,不影响其他知识库。\n')
|
||||
raw = ('--' + boundary + '\r\nContent-Disposition: form-data; name="file"; '
|
||||
'filename="acceptance-upload-' + suffix + '.md"\r\n'
|
||||
'Content-Type: text/markdown; charset=utf-8\r\n\r\n' + document +
|
||||
'\r\n--' + boundary + '--\r\n').encode('utf-8')
|
||||
status, value, latency = client.request('/api/v1/knowledge-bases/' + client.kb_id + '/knowledge/file',
|
||||
'POST', raw=raw, content_type='multipart/form-data; boundary=' + boundary)
|
||||
if status not in (200, 201) or not isinstance(value.get('data'), dict):
|
||||
raise VerificationError('Temporary file upload failed')
|
||||
client.knowledge_id = value['data']['id']
|
||||
report.update(knowledge_id=client.knowledge_id, upload_ms=latency, file_bytes=len(document.encode('utf-8')))
|
||||
started = time.monotonic()
|
||||
while time.monotonic() - started < args.index_timeout:
|
||||
status, value, _ = client.request('/api/v1/knowledge/' + client.knowledge_id)
|
||||
parse_status = value.get('data', {}).get('parse_status') if isinstance(value, dict) else None
|
||||
if parse_status == 'completed':
|
||||
break
|
||||
if parse_status in ('failed', 'cancelled', 'deleting') or status != 200:
|
||||
raise VerificationError('Temporary file indexing failed')
|
||||
time.sleep(2)
|
||||
else:
|
||||
raise VerificationError('Temporary file indexing exceeded its deadline')
|
||||
report['indexing_seconds'] = round(time.monotonic() - started, 2)
|
||||
for mode in MODES:
|
||||
status, value, latency = client.request('/api/v1/knowledge-bases/' + client.kb_id + '/hybrid-search',
|
||||
'POST', search_payload('资料归档 目录核对', mode, 5))
|
||||
result = inspect_results(value, client.kb_id, 5, mode) if status == 200 else {'passed': False}
|
||||
result.update(mode=mode, status=status, latency_ms=latency)
|
||||
report['retrieval'].append(result)
|
||||
if args.verify_milvus:
|
||||
report['vectors_after_upload'] = {name: client.count_vectors(name) for name in
|
||||
('yawan_acceptance_hybrid_512', 'weknora_embeddings_512')}
|
||||
report['isolated_vector_write'] = (report['vectors_after_upload']['yawan_acceptance_hybrid_512'] > 0
|
||||
and report['vectors_after_upload']['weknora_embeddings_512'] == 0)
|
||||
finally:
|
||||
if client.kb_id:
|
||||
status, _, latency = client.request('/api/v1/knowledge-bases/' + client.kb_id, 'DELETE')
|
||||
report['cleanup'] = {'delete_status': status, 'delete_ms': latency, 'passed': False}
|
||||
if status not in (200, 202, 204):
|
||||
raise VerificationError('Temporary KB cleanup request failed')
|
||||
status, _, _ = client.request('/api/v1/knowledge-bases/' + client.kb_id)
|
||||
report['cleanup']['kb_get_status'] = status
|
||||
report['cleanup']['passed'] = status == 404
|
||||
if args.verify_milvus:
|
||||
started = time.monotonic()
|
||||
while time.monotonic() - started < 90:
|
||||
counts = {name: client.count_vectors(name) for name in
|
||||
('yawan_acceptance_hybrid_512', 'weknora_embeddings_512')}
|
||||
if all(count == 0 for count in counts.values()):
|
||||
break
|
||||
time.sleep(2)
|
||||
report['cleanup'].update(vector_counts=counts,
|
||||
vector_cleanup_seconds=round(time.monotonic() - started, 2))
|
||||
report['cleanup']['passed'] = report['cleanup']['passed'] and all(count == 0 for count in counts.values())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--verify-milvus', action='store_true', help='Read counts for this temporary KB from acceptance and production collections')
|
||||
parser.add_argument('--output', type=Path)
|
||||
parser.add_argument('--index-timeout', type=int, default=180)
|
||||
args = parser.parse_args()
|
||||
if not 10 <= args.index_timeout <= 600:
|
||||
parser.error('index-timeout must be 10-600 seconds')
|
||||
if args.output and args.output.resolve() in {file.resolve() for file in
|
||||
(ACCEPTANCE / 'migration-token.json', ACCEPTANCE / 'legacy-migration-state.json', ACCEPTANCE / 'browser-auth.json')}:
|
||||
parser.error('Output must not overwrite existing authentication or migration state')
|
||||
report = {'acceptance_url': BASE, 'retrieval': [], 'failures': []}
|
||||
try:
|
||||
run(args, report)
|
||||
except VerificationError as error:
|
||||
report['failures'].append(str(error))
|
||||
except Exception:
|
||||
report['failures'].append('Unexpected smoke-test failure; no server error or credentials included')
|
||||
report['passed'] = (not report['failures'] and len(report['retrieval']) == 3
|
||||
and all(row['passed'] for row in report['retrieval'])
|
||||
and report.get('cleanup', {}).get('passed', False)
|
||||
and (not args.verify_milvus or report.get('isolated_vector_write', False)))
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + '\n'
|
||||
if args.output:
|
||||
args.output.write_text(text, encoding='utf-8')
|
||||
print(text, end='')
|
||||
return 0 if report['passed'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Executable
+274
@@ -0,0 +1,274 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify retrieval in the isolated acceptance API without changing its data.
|
||||
|
||||
Only login and read-only search POSTs are allowed. Credentials and response
|
||||
prose are never included in the report. No files are written unless --output
|
||||
is supplied, and credential files are never refreshed or overwritten.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
BASE = 'http://127.0.0.1:18080'
|
||||
ACCEPTANCE = Path(__file__).resolve().parents[2] / 'acceptance'
|
||||
UUID = re.compile(r'^[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$')
|
||||
MODES = {
|
||||
'keyword-only': (False, True),
|
||||
'vector-only': (True, False),
|
||||
'hybrid': (False, False),
|
||||
}
|
||||
QUERIES = {
|
||||
'1': '产品 使用 说明',
|
||||
'2': '练习 方法 步骤',
|
||||
'3': '沟通 情绪 理解',
|
||||
'4': '关系 感受 表达',
|
||||
'5': '技巧 步骤 学习',
|
||||
'6': '需求 价值 选择',
|
||||
}
|
||||
|
||||
|
||||
class VerificationError(Exception):
|
||||
"""A fixed, credential-free error suitable for the report."""
|
||||
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
return None
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self, timeout):
|
||||
self.timeout = timeout
|
||||
self.opener = urllib.request.build_opener(
|
||||
urllib.request.ProxyHandler({}), NoRedirect())
|
||||
|
||||
def request(self, route, token=None, payload=None):
|
||||
search_route = re.fullmatch(
|
||||
r'/api/v1/knowledge-bases/[0-9a-fA-F-]+/hybrid-search', route)
|
||||
allowed_read = route in ('/health', '/api/v1/auth/me')
|
||||
allowed_login = route == '/api/v1/auth/login'
|
||||
if not ((payload is None and allowed_read) or
|
||||
(payload is not None and (search_route or allowed_login))):
|
||||
raise VerificationError('Request outside the read-only acceptance allowlist')
|
||||
headers = {'Accept': 'application/json'}
|
||||
if token:
|
||||
headers['Authorization'] = 'Bearer ' + token
|
||||
body = None
|
||||
if payload is not None:
|
||||
headers['Content-Type'] = 'application/json'
|
||||
body = json.dumps(payload).encode('utf-8')
|
||||
request = urllib.request.Request(BASE + route, data=body, headers=headers)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
try:
|
||||
response = self.opener.open(request, timeout=self.timeout)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
status = response.code
|
||||
raw = response.read(8 * 1024 * 1024 + 1)
|
||||
if len(raw) > 8 * 1024 * 1024:
|
||||
raise VerificationError('Response exceeded the verification size limit')
|
||||
# Denied requests need only a status; never retain their error prose.
|
||||
value = json.loads(raw) if status == 200 else None
|
||||
return status, value, round((time.monotonic() - started) * 1000, 2)
|
||||
except VerificationError:
|
||||
raise
|
||||
except (OSError, urllib.error.URLError, ValueError):
|
||||
raise VerificationError('Request failed, timed out, or returned invalid JSON') from None
|
||||
|
||||
|
||||
def read_object(file, label):
|
||||
try:
|
||||
value = json.loads(file.read_text(encoding='utf-8'))
|
||||
except (OSError, ValueError):
|
||||
raise VerificationError(label + ' could not be read as JSON') from None
|
||||
if not isinstance(value, dict):
|
||||
raise VerificationError(label + ' must contain a JSON object')
|
||||
return value
|
||||
|
||||
|
||||
def search_payload(query, mode, count):
|
||||
disable_keywords, disable_vector = MODES[mode]
|
||||
return {
|
||||
'query_text': query,
|
||||
'match_count': count,
|
||||
'vector_threshold': 0,
|
||||
'keyword_threshold': 0,
|
||||
'disable_keywords_match': disable_keywords,
|
||||
'disable_vector_match': disable_vector,
|
||||
'skip_context_enrichment': True,
|
||||
}
|
||||
|
||||
|
||||
def inspect_results(value, kb_id, count, mode):
|
||||
if not isinstance(value, dict) or value.get('success') is not True:
|
||||
return {'passed': False, 'failure': 'Invalid search response envelope'}
|
||||
rows = value.get('data') or []
|
||||
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
|
||||
return {'passed': False, 'failure': 'Invalid search results shape'}
|
||||
scores = [row.get('score') for row in rows]
|
||||
finite = all(isinstance(score, (int, float)) and not isinstance(score, bool)
|
||||
and math.isfinite(score) for score in scores)
|
||||
descending = finite and all(left + 1e-9 >= right
|
||||
for left, right in zip(scores, scores[1:]))
|
||||
in_scope = all(row.get('knowledge_base_id') == kb_id for row in rows)
|
||||
ids = [row.get('id') for row in rows]
|
||||
unique = all(isinstance(item, str) and item for item in ids) and len(set(ids)) == len(ids)
|
||||
# Match types 0 and 1 are vector and keyword hits respectively.
|
||||
expected = {'keyword-only': {1}, 'vector-only': {0}, 'hybrid': {0, 1}}[mode]
|
||||
correct_mode = all(type(row.get('match_type')) is int and row['match_type'] in expected for row in rows)
|
||||
return {
|
||||
'passed': bool(rows) and len(rows) <= count and finite and descending and in_scope and unique and correct_mode,
|
||||
'count': len(rows),
|
||||
'nonempty': bool(rows),
|
||||
'within_requested_count': len(rows) <= count,
|
||||
'scores': [round(score, 8) for score in scores] if finite else [],
|
||||
'scores_finite': finite,
|
||||
'scores_descending': descending,
|
||||
'kb_scope_valid': in_scope,
|
||||
'unique_chunk_ids': unique,
|
||||
'match_types_valid': correct_mode,
|
||||
}
|
||||
|
||||
|
||||
def verify(args, report):
|
||||
credentials = read_object(args.credentials, 'Migration credentials')
|
||||
migration = read_object(args.state, 'Migration state')
|
||||
foreign = read_object(args.foreign_auth, 'Foreign tenant authentication')
|
||||
entries = migration.get('knowledge_bases', {})
|
||||
if not isinstance(entries, dict) or set(entries) != set(QUERIES):
|
||||
raise VerificationError('Migration state must describe exactly the six expected knowledge bases')
|
||||
kb_ids = []
|
||||
for legacy_id, row in entries.items():
|
||||
if not isinstance(row, dict) or not UUID.fullmatch(str(row.get('id', ''))):
|
||||
raise VerificationError('Migration state contains an invalid knowledge base ID')
|
||||
kb_ids.append(row['id'])
|
||||
if len(set(kb_ids)) != 6:
|
||||
raise VerificationError('Migration state contains duplicate knowledge base IDs')
|
||||
if not all(isinstance(credentials.get(key), str) and credentials[key] for key in ('email', 'password')):
|
||||
raise VerificationError('Migration login credentials are incomplete')
|
||||
if not isinstance(foreign.get('token'), str) or not foreign['token']:
|
||||
raise VerificationError('Existing foreign tenant token is required')
|
||||
|
||||
client = Client(args.timeout)
|
||||
status, _, latency = client.request('/health')
|
||||
report['checks'].append({'check': 'acceptance_health', 'status': status,
|
||||
'latency_ms': latency, 'passed': status == 200})
|
||||
if status != 200:
|
||||
raise VerificationError('Acceptance API is not healthy')
|
||||
|
||||
status, login, latency = client.request('/api/v1/auth/login', payload={
|
||||
'email': credentials['email'], 'password': credentials['password'],
|
||||
})
|
||||
if status != 200 or not isinstance(login, dict) or not isinstance(login.get('token'), str):
|
||||
raise VerificationError('Migration login failed')
|
||||
token = login['token']
|
||||
tenant_id = login.get('active_tenant', {}).get('id')
|
||||
correct_tenant = tenant_id is not None and tenant_id == migration.get('tenant_id') == credentials.get('tenant_id')
|
||||
report['checks'].append({'check': 'migration_login_scope', 'status': status,
|
||||
'latency_ms': latency, 'passed': correct_tenant})
|
||||
if not correct_tenant:
|
||||
raise VerificationError('Migration login is outside the expected tenant')
|
||||
|
||||
status, identity, latency = client.request('/api/v1/auth/me', foreign['token'])
|
||||
details = identity.get('data', {}) if isinstance(identity, dict) else {}
|
||||
foreign_tenant = (details.get('tenant') or {}).get('id')
|
||||
user = details.get('user') or {}
|
||||
memberships = details.get('memberships') or []
|
||||
isolated = (status == 200 and foreign_tenant is not None and foreign_tenant != tenant_id
|
||||
and not user.get('is_system_admin') and not user.get('can_access_all_tenants')
|
||||
and not any(member.get('tenant_id') == tenant_id for member in memberships if isinstance(member, dict)))
|
||||
report['checks'].append({'check': 'foreign_identity_valid_and_isolated', 'status': status,
|
||||
'latency_ms': latency, 'passed': isolated})
|
||||
if not isolated:
|
||||
report['failures'].append('Foreign authentication is expired, invalid, or not isolated; denial tests cannot prove tenant isolation')
|
||||
|
||||
for legacy_id in sorted(entries):
|
||||
kb_id = entries[legacy_id]['id']
|
||||
route = '/api/v1/knowledge-bases/' + kb_id + '/hybrid-search'
|
||||
payload = search_payload(QUERIES[legacy_id], 'keyword-only', args.top_k)
|
||||
for check, auth, enabled, statuses in (
|
||||
('unauthenticated_denied', None, True, (401, 403)),
|
||||
('foreign_tenant_denied', foreign['token'], isolated, (403, 404)),
|
||||
):
|
||||
if not enabled:
|
||||
continue
|
||||
result = {'check': check, 'legacy_kb': legacy_id, 'kb_id': kb_id}
|
||||
try:
|
||||
status, _, latency = client.request(route, auth, payload)
|
||||
result.update(status=status, latency_ms=latency, passed=status in statuses)
|
||||
except VerificationError as error:
|
||||
result.update(passed=False, failure=str(error))
|
||||
report['checks'].append(result)
|
||||
|
||||
for mode in MODES:
|
||||
result = {'legacy_kb': legacy_id, 'kb_id': kb_id, 'mode': mode,
|
||||
'query': QUERIES[legacy_id], 'top_k': args.top_k}
|
||||
try:
|
||||
status, value, latency = client.request(route, token, search_payload(QUERIES[legacy_id], mode, args.top_k))
|
||||
result.update(status=status, latency_ms=latency)
|
||||
result.update(inspect_results(value, kb_id, args.top_k, mode) if status == 200
|
||||
else {'passed': False, 'failure': 'Search HTTP status was not 200'})
|
||||
except VerificationError as error:
|
||||
result.update(passed=False, failure=str(error))
|
||||
report['retrieval'].append(result)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--credentials', type=Path, default=ACCEPTANCE / 'migration-token.json')
|
||||
parser.add_argument('--state', type=Path, default=ACCEPTANCE / 'legacy-migration-state.json')
|
||||
parser.add_argument('--foreign-auth', type=Path, default=ACCEPTANCE / 'browser-auth.json')
|
||||
parser.add_argument('--output', type=Path, help='Write the same JSON report to this file only')
|
||||
parser.add_argument('--timeout', type=float, default=45, help='Timeout in seconds for every HTTP request (1-180)')
|
||||
parser.add_argument('--top-k', type=int, default=10, help='Requested primary matches (1-50)')
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.timeout <= 180 or not 1 <= args.top_k <= 50:
|
||||
parser.error('timeout must be 1-180 seconds and top-k must be 1-50')
|
||||
if args.output and args.output.resolve() in {file.resolve() for file in (args.credentials, args.state, args.foreign_auth)}:
|
||||
parser.error('output must not overwrite authentication or migration state')
|
||||
started = time.monotonic()
|
||||
report = {
|
||||
'acceptance_url': BASE,
|
||||
'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'checks': [], 'retrieval': [], 'failures': [],
|
||||
'limitations': ['API validation does not independently attest the configured backend or collection',
|
||||
'Nonempty and ordered matches are smoke checks, not a relevance-quality benchmark'],
|
||||
}
|
||||
try:
|
||||
verify(args, report)
|
||||
except VerificationError as error:
|
||||
report['failures'].append(str(error))
|
||||
except Exception:
|
||||
# Do not expose response contents, credentials, or traceback locals.
|
||||
report['failures'].append('Unexpected verification failure; inspect the verifier and response shapes locally')
|
||||
all_checks = report['checks'] + report['retrieval']
|
||||
report['passed'] = (not report['failures'] and len(report['retrieval']) == 18
|
||||
and all(item.get('passed') for item in all_checks))
|
||||
report['duration_seconds'] = round(time.monotonic() - started, 2)
|
||||
report['summary'] = {'checks': len(all_checks),
|
||||
'passed': sum(bool(item.get('passed')) for item in all_checks),
|
||||
'failed': sum(not item.get('passed') for item in all_checks)}
|
||||
encoded = json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + '\n'
|
||||
if args.output:
|
||||
try:
|
||||
args.output.write_text(encoded, encoding='utf-8')
|
||||
except OSError:
|
||||
report['passed'] = False
|
||||
report['failures'].append('Report output could not be written')
|
||||
encoded = json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + '\n'
|
||||
print(encoded, end='')
|
||||
return 0 if report['passed'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,835 @@
|
||||
// Run explicitly by filename, like scripts/wiki_backfill_trigger.go.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/common"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hibiken/asynq"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
type graphRecoveryCandidate struct {
|
||||
ID string `json:"chunk_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
SeqID int64 `json:"seq_id"`
|
||||
ChunkIndex int `json:"chunk_index"`
|
||||
ContentHash string `json:"content_sha256"`
|
||||
}
|
||||
|
||||
// Failure records survive Redis retention and watcher restarts. Raw provider
|
||||
// errors are deliberately excluded: state and stable identities are sufficient
|
||||
// to find the task's private diagnostics and authorize a bounded manual retry.
|
||||
type graphRecoveryFailure struct {
|
||||
TaskID string `gorm:"primaryKey" json:"task_id"`
|
||||
TenantID uint64 `gorm:"index:idx_graph_backfill_scope;index:idx_graph_backfill_candidate,priority:1" json:"tenant_id"`
|
||||
KnowledgeBaseID string `gorm:"index:idx_graph_backfill_scope;index:idx_graph_backfill_candidate,priority:2" json:"knowledge_base_id"`
|
||||
ChunkID string `gorm:"index:idx_graph_backfill_candidate,priority:3" json:"chunk_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
ContentHash string `gorm:"index:idx_graph_backfill_candidate,priority:4" json:"content_sha256"`
|
||||
Revision string `gorm:"index:idx_graph_backfill_candidate,priority:5" json:"revision"`
|
||||
State string `gorm:"index:idx_graph_backfill_candidate,priority:6" json:"state"`
|
||||
Reason string `json:"reason"`
|
||||
ManualRetries int `json:"manual_retries"`
|
||||
RetryReason string `json:"retry_reason,omitempty"`
|
||||
FirstFailedAt time.Time `json:"first_failed_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
func (graphRecoveryFailure) TableName() string { return "native_graph_backfill_failures" }
|
||||
|
||||
func graphFailureFilter(includeFailed bool) string {
|
||||
if includeFailed {
|
||||
return ""
|
||||
}
|
||||
return ` AND NOT EXISTS(SELECT 1 FROM native_graph_backfill_failures f WHERE f.tenant_id=c.tenant_id AND f.knowledge_base_id=c.knowledge_base_id AND f.chunk_id=c.id AND f.content_hash=encode(sha256(convert_to(c.content,'UTF8')),'hex') AND f.revision=? AND f.state='failed')`
|
||||
}
|
||||
|
||||
const graphCurrentFailureSource = `EXISTS(SELECT 1 FROM chunks c JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id AND k.deleted_at IS NULL WHERE c.tenant_id=f.tenant_id AND c.knowledge_base_id=f.knowledge_base_id AND c.id=f.chunk_id AND c.knowledge_id=f.knowledge_id AND c.deleted_at IS NULL AND c.is_enabled AND c.chunk_type='text' AND encode(sha256(convert_to(c.content,'UTF8')),'hex')=f.content_hash)`
|
||||
const graphFailureCheckpoint = `EXISTS(SELECT 1 FROM native_graph_records r WHERE r.tenant_id=f.tenant_id AND r.knowledge_base_id=f.knowledge_base_id AND r.source_table='knowledgegraphchunkstate' AND r.payload->>'origin'='native_extraction_v1' AND r.payload->>'extraction_status'='complete' AND CAST(COALESCE(r.payload->>'relation_count','0') AS INTEGER)>0 AND r.payload->>'native_chunk_id'=f.chunk_id AND r.payload->>'native_knowledge_id'=f.knowledge_id AND r.payload->>'source_content_hash'=f.content_hash)`
|
||||
|
||||
func graphUnresolvedFailures(db *gorm.DB, tenant uint64, kbID, revision string, reconcile bool) ([]graphRecoveryFailure, error) {
|
||||
if reconcile {
|
||||
// Keep historical rows, but a deleted/changed source or a newly verified
|
||||
// checkpoint must not leave a completed knowledge base blocked forever.
|
||||
for _, resolution := range []struct{ state, condition string }{
|
||||
{"superseded", "NOT " + graphCurrentFailureSource},
|
||||
{"resolved", graphCurrentFailureSource + " AND " + graphFailureCheckpoint},
|
||||
} {
|
||||
if err := db.Exec("UPDATE native_graph_backfill_failures AS f SET state=?,updated_at=? WHERE tenant_id=? AND knowledge_base_id=? AND revision=? AND state NOT IN ('resolved','superseded') AND "+resolution.condition, resolution.state, time.Now().UTC(), tenant, kbID, revision).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
var failures []graphRecoveryFailure
|
||||
err := db.Table("native_graph_backfill_failures AS f").Where("f.tenant_id=? AND f.knowledge_base_id=? AND f.revision=? AND f.state='failed'", tenant, kbID, revision).Where(graphCurrentFailureSource).Where("NOT " + graphFailureCheckpoint).Order("f.first_failed_at,f.task_id").Find(&failures).Error
|
||||
return failures, err
|
||||
}
|
||||
|
||||
func graphRecoveryCandidates(db *gorm.DB, tenant uint64, kbID, revision string, after int64, limit int, hasFailures, includeFailed bool, exact ...graphRecoveryCandidate) ([]graphRecoveryCandidate, error) {
|
||||
var candidates []graphRecoveryCandidate
|
||||
filter := ""
|
||||
queryArgs := []any{tenant, kbID, tenant, kbID, after}
|
||||
if hasFailures && !includeFailed {
|
||||
filter = graphFailureFilter(false)
|
||||
queryArgs = append(queryArgs, revision)
|
||||
}
|
||||
if len(exact) > 0 {
|
||||
filter += " AND c.id=? AND c.knowledge_id=? AND encode(sha256(convert_to(c.content,'UTF8')),'hex')=?"
|
||||
queryArgs = append(queryArgs, exact[0].ID, exact[0].KnowledgeID, exact[0].ContentHash)
|
||||
}
|
||||
queryArgs = append(queryArgs, limit)
|
||||
err := db.Raw(`WITH completed AS NOT MATERIALIZED(SELECT payload FROM native_graph_records WHERE tenant_id=? AND knowledge_base_id=? AND source_table='knowledgegraphchunkstate' AND payload->>'origin'='native_extraction_v1' AND payload->>'extraction_status'='complete' AND CAST(COALESCE(payload->>'relation_count','0') AS INTEGER)>0)
|
||||
SELECT c.id,c.knowledge_id,c.seq_id,c.chunk_index,encode(sha256(convert_to(c.content,'UTF8')),'hex') AS content_hash
|
||||
FROM chunks c JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id AND k.deleted_at IS NULL
|
||||
WHERE c.tenant_id=? AND c.knowledge_base_id=? AND c.deleted_at IS NULL AND c.is_enabled AND c.chunk_type='text' AND c.seq_id>? AND k.parse_status='completed'
|
||||
AND NOT EXISTS(SELECT 1 FROM completed r WHERE r.payload->>'native_chunk_id'=c.id AND r.payload->>'native_knowledge_id'=c.knowledge_id AND r.payload->>'source_content_hash'=encode(sha256(convert_to(c.content,'UTF8')),'hex'))
|
||||
`+filter+` ORDER BY c.seq_id LIMIT ?`, queryArgs...).Scan(&candidates).Error
|
||||
if err == nil && len(exact) > 0 && len(candidates) != 1 {
|
||||
return nil, errors.New("exact graph source unavailable, changed, completed or excluded by its failure ledger; no substitute was selected")
|
||||
}
|
||||
return candidates, err
|
||||
}
|
||||
|
||||
func graphValidateExactScope(chunkID, knowledgeID, hash string, after int64, limit, maxOutstanding int) error {
|
||||
if chunkID == "" && knowledgeID == "" && hash == "" {
|
||||
return nil
|
||||
}
|
||||
chunk, chunkErr := uuid.Parse(chunkID)
|
||||
knowledge, knowledgeErr := uuid.Parse(knowledgeID)
|
||||
decoded, hashErr := hex.DecodeString(hash)
|
||||
if chunkErr != nil || knowledgeErr != nil || chunk == uuid.Nil || knowledge == uuid.Nil ||
|
||||
chunk.String() != chunkID || knowledge.String() != knowledgeID ||
|
||||
hashErr != nil || len(decoded) != sha256.Size || hex.EncodeToString(decoded) != hash ||
|
||||
after != 0 || limit != 1 || maxOutstanding != 1 {
|
||||
return errors.New("exact graph scope requires canonical chunk and knowledge UUIDs, lowercase source SHA-256, limit=1, max-outstanding=1 and no cursor")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func graphTaskMatchesReviewedSource(payload []byte, tenant uint64, kbID string, c graphRecoveryCandidate) bool {
|
||||
var task types.ExtractChunkPayload
|
||||
return json.Unmarshal(payload, &task) == nil && task.TenantID == tenant && task.ChunkID == c.ID &&
|
||||
(task.KnowledgeID == "" || task.KnowledgeID == c.KnowledgeID) &&
|
||||
task.ExpectedSource != nil && task.ExpectedSource.KnowledgeBaseID == kbID &&
|
||||
task.ExpectedSource.KnowledgeID == c.KnowledgeID && task.ExpectedSource.ContentSHA256 == c.ContentHash
|
||||
}
|
||||
|
||||
type graphRecoveryQueue interface {
|
||||
RunTask(string, string) error
|
||||
}
|
||||
|
||||
func graphQueueSlots(info *asynq.QueueInfo, limit int) int {
|
||||
if info == nil || info.Paused {
|
||||
return 0
|
||||
}
|
||||
remaining := limit - info.Active - info.Pending - info.Scheduled - info.Retry - info.Aggregating
|
||||
if remaining < 0 {
|
||||
return 0
|
||||
}
|
||||
return remaining
|
||||
}
|
||||
|
||||
func graphResumeCursor(after int64, candidates []graphRecoveryCandidate, settled map[string]bool) int64 {
|
||||
for _, c := range candidates {
|
||||
if !settled[c.ID] {
|
||||
break
|
||||
}
|
||||
after = c.SeqID
|
||||
}
|
||||
return after
|
||||
}
|
||||
|
||||
func graphRecoveryTaskID(tenant uint64, kbID string, candidate graphRecoveryCandidate, revision string) string {
|
||||
identity := sha256.Sum256([]byte(fmt.Sprintf("%d:%s:%s:%s:%s", tenant, kbID, candidate.ID, candidate.ContentHash, revision)))
|
||||
return "native-graph-" + hex.EncodeToString(identity[:])
|
||||
}
|
||||
|
||||
func recordGraphFailure(db *gorm.DB, tenant uint64, kbID, revision, taskID string, c graphRecoveryCandidate, reason string) error {
|
||||
var failure graphRecoveryFailure
|
||||
err := db.First(&failure, "task_id=?", taskID).Error
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
failure = graphRecoveryFailure{TaskID: taskID, TenantID: tenant, KnowledgeBaseID: kbID, ChunkID: c.ID, KnowledgeID: c.KnowledgeID, ContentHash: c.ContentHash, Revision: revision, FirstFailedAt: time.Now().UTC()}
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if failure.FirstFailedAt.IsZero() {
|
||||
failure.FirstFailedAt = time.Now().UTC()
|
||||
}
|
||||
failure.State, failure.Reason, failure.UpdatedAt = "failed", reason, time.Now().UTC()
|
||||
return db.Save(&failure).Error
|
||||
}
|
||||
|
||||
func reserveGraphDispatch(db *gorm.DB, tenant uint64, kbID, revision, taskID string, c graphRecoveryCandidate) (bool, error) {
|
||||
var prior graphRecoveryFailure
|
||||
err := db.First(&prior, "task_id=?", taskID).Error
|
||||
if err == nil {
|
||||
// An absent Redis task is not permission to repeat an earlier attempt.
|
||||
return false, recordGraphFailure(db, tenant, kbID, revision, taskID, c, "task_missing_after_dispatch")
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return false, err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
entry := graphRecoveryFailure{TaskID: taskID, TenantID: tenant, KnowledgeBaseID: kbID, ChunkID: c.ID, KnowledgeID: c.KnowledgeID, ContentHash: c.ContentHash, Revision: revision, State: "dispatch_requested", UpdatedAt: now}
|
||||
if err := db.Create(&entry).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func trackGraphTask(db *gorm.DB, tenant uint64, kbID, revision, taskID string, c graphRecoveryCandidate) error {
|
||||
entry := graphRecoveryFailure{TaskID: taskID, TenantID: tenant, KnowledgeBaseID: kbID, ChunkID: c.ID, KnowledgeID: c.KnowledgeID, ContentHash: c.ContentHash, Revision: revision, State: "dispatch_requested", UpdatedAt: time.Now().UTC()}
|
||||
return db.Clauses(clause.OnConflict{DoNothing: true}).Create(&entry).Error
|
||||
}
|
||||
|
||||
func graphArchivedReason(lastError string) string {
|
||||
message := strings.ToLower(lastError)
|
||||
if strings.Contains(message, "graph_needs_review") {
|
||||
return "relation_coverage_review_required"
|
||||
}
|
||||
if strings.Contains(message, "read response") && (strings.Contains(message, "context deadline exceeded") || strings.Contains(message, "timeout")) {
|
||||
return "provider_response_timeout"
|
||||
}
|
||||
if strings.Contains(message, "context deadline exceeded") || strings.Contains(message, "deadline exceeded") || strings.Contains(message, "timeout") {
|
||||
return "task_timeout"
|
||||
}
|
||||
if strings.Contains(message, "context canceled") || strings.Contains(message, "context cancelled") {
|
||||
return "task_cancelled"
|
||||
}
|
||||
return "archived"
|
||||
}
|
||||
|
||||
// Reconciliation receives only the Inspector read API, never a queue client.
|
||||
type graphTaskReader interface {
|
||||
GetTaskInfo(string, string) (*asynq.TaskInfo, error)
|
||||
}
|
||||
|
||||
type graphReconcileReport struct {
|
||||
Mode string `json:"mode"`
|
||||
Apply bool `json:"apply"`
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
Revision string `json:"revision"`
|
||||
Source graphRecoveryCandidate `json:"source"`
|
||||
TaskID string `json:"task_id"`
|
||||
QueueState string `json:"queue_state"`
|
||||
LedgerState string `json:"ledger_state"`
|
||||
ProposedState string `json:"proposed_state"`
|
||||
SourceCurrent bool `json:"source_current"`
|
||||
CheckpointComplete bool `json:"checkpoint_complete"`
|
||||
Reason string `json:"reason"`
|
||||
Blocked bool `json:"blocked"`
|
||||
WouldUpdate bool `json:"would_update"`
|
||||
LedgerUpdated bool `json:"ledger_updated"`
|
||||
Enqueued int `json:"enqueued"`
|
||||
Retried int `json:"retried"`
|
||||
ModelCalls int `json:"model_calls"`
|
||||
}
|
||||
|
||||
const graphReconcileCurrentSource = graphCurrentFailureSource + ` AND EXISTS(SELECT 1 FROM knowledges k JOIN knowledge_bases b ON b.id=k.knowledge_base_id AND b.tenant_id=k.tenant_id WHERE k.id=f.knowledge_id AND k.tenant_id=f.tenant_id AND k.knowledge_base_id=f.knowledge_base_id AND k.deleted_at IS NULL AND k.parse_status='completed' AND b.deleted_at IS NULL)`
|
||||
|
||||
func graphValidateReconciliation(reconcile bool, chunkID, revision string, retry bool, retryReason string) error {
|
||||
if reconcile && (chunkID == "" || strings.TrimSpace(revision) == "" || retry || retryReason != "") {
|
||||
return errors.New("reconcile-only requires one exact source, GRAPH_BACKFILL_REVISION and no retry flags")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func graphReconcileSourceStatus(db *gorm.DB, taskID string) (current, complete bool, err error) {
|
||||
var status struct {
|
||||
Current, Complete bool
|
||||
}
|
||||
result := db.Raw("SELECT ("+graphReconcileCurrentSource+") AS current,("+graphFailureCheckpoint+") AS complete FROM native_graph_backfill_failures f WHERE f.task_id=?", taskID).Scan(&status)
|
||||
if result.Error != nil {
|
||||
return false, false, errors.New("cannot validate graph source and checkpoint")
|
||||
}
|
||||
if result.RowsAffected != 1 {
|
||||
return false, false, errors.New("graph reconciliation ledger disappeared")
|
||||
}
|
||||
return status.Current, status.Complete, nil
|
||||
}
|
||||
|
||||
func reconcileGraphTask(db *gorm.DB, reader graphTaskReader, tenant uint64, kbID, revision string, c graphRecoveryCandidate, apply bool) (graphReconcileReport, error) {
|
||||
report := graphReconcileReport{Mode: "reconcile_only", Apply: apply, TenantID: tenant, KnowledgeBaseID: kbID, Revision: revision, Source: c,
|
||||
TaskID: graphRecoveryTaskID(tenant, kbID, c, revision), QueueState: "unknown", Blocked: true}
|
||||
var ledger graphRecoveryFailure
|
||||
err := db.First(&ledger, "task_id=? AND tenant_id=? AND knowledge_base_id=? AND chunk_id=? AND knowledge_id=? AND content_hash=? AND revision=?",
|
||||
report.TaskID, tenant, kbID, c.ID, c.KnowledgeID, c.ContentHash, revision).Error
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
report.Reason = "ledger_missing_or_identity_mismatch"
|
||||
return report, nil
|
||||
}
|
||||
if err != nil {
|
||||
return report, errors.New("cannot read exact graph ledger")
|
||||
}
|
||||
report.LedgerState, report.ProposedState = ledger.State, ledger.State
|
||||
report.SourceCurrent, report.CheckpointComplete, err = graphReconcileSourceStatus(db, report.TaskID)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if !report.SourceCurrent {
|
||||
report.Reason = "source_changed_or_unavailable"
|
||||
return report, nil
|
||||
}
|
||||
if ledger.State == "failed" && strings.HasPrefix(ledger.Reason, "semantic_review_rejected") {
|
||||
report.Reason = "manual_semantic_review_required"
|
||||
return report, nil
|
||||
}
|
||||
info, err := reader.GetTaskInfo(types.QueueGraph, report.TaskID)
|
||||
if errors.Is(err, asynq.ErrTaskNotFound) || errors.Is(err, asynq.ErrQueueNotFound) || (err == nil && info == nil) {
|
||||
report.QueueState, report.Reason = "missing", "task_missing_after_dispatch"
|
||||
return report, nil
|
||||
}
|
||||
if err != nil {
|
||||
report.Reason = "queue_inspection_failed"
|
||||
return report, errors.New("cannot inspect graph task; no reconciliation applied")
|
||||
}
|
||||
report.QueueState = info.State.String()
|
||||
if info.ID != report.TaskID || info.Queue != types.QueueGraph || info.Type != types.TypeChunkExtract ||
|
||||
!graphTaskMatchesReviewedSource(info.Payload, tenant, kbID, c) || info.MaxRetry != 0 {
|
||||
report.Reason = "queue_task_identity_mismatch"
|
||||
return report, nil
|
||||
}
|
||||
if info.State != asynq.TaskStateArchived && info.State != asynq.TaskStateCompleted {
|
||||
report.Blocked, report.Reason = false, "task_not_terminal"
|
||||
return report, nil
|
||||
}
|
||||
// A worker can commit between the first source read and terminal queue read.
|
||||
report.SourceCurrent, report.CheckpointComplete, err = graphReconcileSourceStatus(db, report.TaskID)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if !report.SourceCurrent {
|
||||
report.Reason = "source_changed_or_unavailable"
|
||||
return report, nil
|
||||
}
|
||||
if report.CheckpointComplete {
|
||||
report.ProposedState, report.Reason, report.Blocked = "resolved", "native_checkpoint_verified", false
|
||||
} else {
|
||||
report.ProposedState, report.Reason = "failed", "completed_without_native_checkpoint"
|
||||
if info.State == asynq.TaskStateArchived {
|
||||
report.Reason = graphArchivedReason(info.LastErr)
|
||||
}
|
||||
}
|
||||
if ledger.State == "superseded" || (ledger.State == "resolved" && report.ProposedState != "resolved") {
|
||||
report.ProposedState, report.Reason, report.Blocked = ledger.State, "ledger_terminal_conflict", true
|
||||
return report, nil
|
||||
}
|
||||
// Preserve earlier failure and retry history; refine only an empty/generic
|
||||
// reason. Resolution never erases the original failure reason or timestamp.
|
||||
reason := ledger.Reason
|
||||
if report.ProposedState == "failed" && (reason == "" || reason == "archived") {
|
||||
reason = report.Reason
|
||||
}
|
||||
report.WouldUpdate = ledger.State != report.ProposedState || ledger.Reason != reason ||
|
||||
(report.ProposedState == "failed" && ledger.FirstFailedAt.IsZero())
|
||||
if !apply || !report.WouldUpdate {
|
||||
return report, nil
|
||||
}
|
||||
updates := map[string]any{"state": report.ProposedState, "reason": reason, "updated_at": time.Now().UTC()}
|
||||
if report.ProposedState == "failed" && ledger.FirstFailedAt.IsZero() {
|
||||
failedAt := info.LastFailedAt
|
||||
if failedAt.IsZero() {
|
||||
failedAt = time.Now().UTC()
|
||||
}
|
||||
updates["first_failed_at"] = failedAt
|
||||
}
|
||||
checkpoint := graphFailureCheckpoint
|
||||
if !report.CheckpointComplete {
|
||||
checkpoint = "NOT " + checkpoint
|
||||
}
|
||||
// Revalidate source, checkpoint, and ledger state in the update itself so a
|
||||
// concurrent source edit or recovery cannot be overwritten by this snapshot.
|
||||
result := db.Table("native_graph_backfill_failures AS f").Where("f.task_id=? AND f.state=? AND f.reason=? AND f.manual_retries=? AND f.updated_at=?",
|
||||
report.TaskID, ledger.State, ledger.Reason, ledger.ManualRetries, ledger.UpdatedAt).
|
||||
Where("f.tenant_id=? AND f.knowledge_base_id=? AND f.chunk_id=? AND f.knowledge_id=? AND f.content_hash=? AND f.revision=?", tenant, kbID, c.ID, c.KnowledgeID, c.ContentHash, revision).
|
||||
Where(graphReconcileCurrentSource).Where(checkpoint).Updates(updates)
|
||||
if result.Error != nil {
|
||||
return report, errors.New("cannot update exact graph ledger")
|
||||
}
|
||||
if result.RowsAffected != 1 {
|
||||
report.Reason, report.Blocked = "reconciliation_state_changed", true
|
||||
return report, nil
|
||||
}
|
||||
report.LedgerUpdated = true
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func graphRecoveryRedisOptions() (asynq.RedisClientOpt, error) {
|
||||
redisDB, err := strconv.Atoi(os.Getenv("REDIS_DB"))
|
||||
if err != nil || os.Getenv("REDIS_ADDR") == "" {
|
||||
return asynq.RedisClientOpt{}, errors.New("runtime Redis configuration is missing")
|
||||
}
|
||||
return asynq.RedisClientOpt{Addr: os.Getenv("REDIS_ADDR"), Username: os.Getenv("REDIS_USERNAME"), Password: os.Getenv("REDIS_PASSWORD"), DB: redisDB, TLSConfig: common.RedisTLSConfig()}, nil
|
||||
}
|
||||
|
||||
// The checkpoint is queried again after observing a completed task because a
|
||||
// worker may have committed between candidate selection and Redis inspection.
|
||||
func graphCandidateCompleted(db *gorm.DB, tenant uint64, kbID string, c graphRecoveryCandidate) (bool, error) {
|
||||
var count int64
|
||||
err := db.Table("native_graph_records").Where("tenant_id=? AND knowledge_base_id=? AND source_table='knowledgegraphchunkstate' AND payload->>'origin'='native_extraction_v1' AND payload->>'extraction_status'='complete' AND CAST(COALESCE(payload->>'relation_count','0') AS INTEGER)>0 AND payload->>'native_chunk_id'=? AND payload->>'native_knowledge_id'=? AND payload->>'source_content_hash'=?", tenant, kbID, c.ID, c.KnowledgeID, c.ContentHash).Count(&count).Error
|
||||
return count > 0, err
|
||||
}
|
||||
|
||||
func graphFailureRetry(db *gorm.DB, queue graphRecoveryQueue, taskID, reason string, slots int) (bool, error) {
|
||||
if slots <= 0 {
|
||||
return false, nil
|
||||
}
|
||||
if len(strings.TrimSpace(reason)) < 12 {
|
||||
return false, errors.New("manual retry requires a reason describing the investigated failure and fix")
|
||||
}
|
||||
var failure graphRecoveryFailure
|
||||
if err := db.First(&failure, "task_id=?", taskID).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
if failure.ManualRetries >= 1 {
|
||||
return false, nil
|
||||
}
|
||||
// Reserve the one allowed manual retry before sending to Redis. A crash
|
||||
// here fails closed and leaves a visible record instead of retrying forever.
|
||||
failure.ManualRetries++
|
||||
failure.State, failure.RetryReason, failure.UpdatedAt = "retry_requested", reason, time.Now().UTC()
|
||||
if err := db.Save(&failure).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := queue.RunTask(types.QueueGraph, taskID); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func graphRecoveryDSN() (string, error) {
|
||||
if dsn := os.Getenv("GRAPH_BACKFILL_DSN"); dsn != "" {
|
||||
return dsn, nil
|
||||
}
|
||||
if os.Getenv("DB_HOST") == "" || os.Getenv("DB_NAME") == "" || os.Getenv("DB_USER") == "" {
|
||||
return "", errors.New("runtime PostgreSQL configuration is missing")
|
||||
}
|
||||
port := os.Getenv("DB_PORT")
|
||||
if port == "" {
|
||||
port = "5432"
|
||||
}
|
||||
u := url.URL{Scheme: "postgresql", User: url.UserPassword(os.Getenv("DB_USER"), os.Getenv("DB_PASSWORD")), Host: net.JoinHostPort(os.Getenv("DB_HOST"), port), Path: "/" + os.Getenv("DB_NAME")}
|
||||
q := u.Query()
|
||||
ssl := os.Getenv("DB_SSL_MODE")
|
||||
if ssl == "" {
|
||||
ssl = "disable"
|
||||
}
|
||||
q.Set("sslmode", ssl)
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func runGraphRecovery() error {
|
||||
tenant := flag.Uint64("tenant", 0, "required tenant")
|
||||
kbID := flag.String("kb", "", "required knowledge base UUID")
|
||||
limit := flag.Int("limit", 20, "bounded chunks to enqueue (1-200)")
|
||||
after := flag.Int64("after", 0, "resume after this chunk sequence ID")
|
||||
apply := flag.Bool("apply", false, "apply the selected mode; otherwise report only")
|
||||
reconcileOnly := flag.Bool("reconcile-only", false, "reconcile one existing exact-source ledger only; never enqueue or retry")
|
||||
retry := flag.Bool("retry-failed", false, "one audited manual retry for tasks still archived in Redis")
|
||||
retryReason := flag.String("retry-reason", "", "audited reason for the single permitted manual retry")
|
||||
maxOutstanding := flag.Int("max-outstanding", 2, "maximum outstanding tasks in the shared graph queue (1-20)")
|
||||
chunkID := flag.String("chunk", "", "only this reviewed chunk; requires knowledge and source-sha256")
|
||||
knowledgeID := flag.String("knowledge", "", "reviewed source document UUID")
|
||||
sourceHash := flag.String("source-sha256", "", "reviewed exact source SHA-256")
|
||||
flag.Parse()
|
||||
revision := os.Getenv("GRAPH_BACKFILL_REVISION")
|
||||
if flag.NArg() != 0 {
|
||||
return errors.New("unexpected positional arguments")
|
||||
}
|
||||
if err := graphValidateReconciliation(*reconcileOnly, *chunkID, revision, *retry, *retryReason); err != nil {
|
||||
return err
|
||||
}
|
||||
if *tenant == 0 || *limit < 1 || *limit > 200 || *after < 0 || *maxOutstanding < 1 || *maxOutstanding > 20 {
|
||||
return errors.New("tenant, limit 1-200 and non-negative cursor are required")
|
||||
}
|
||||
if *retry && len(strings.TrimSpace(*retryReason)) < 12 {
|
||||
return errors.New("retry-failed requires an audited retry-reason describing the investigated failure and fix")
|
||||
}
|
||||
if _, err := uuid.Parse(*kbID); err != nil {
|
||||
return errors.New("invalid knowledge base UUID")
|
||||
}
|
||||
if err := graphValidateExactScope(*chunkID, *knowledgeID, *sourceHash, *after, *limit, *maxOutstanding); err != nil {
|
||||
return err
|
||||
}
|
||||
var exact []graphRecoveryCandidate
|
||||
if *chunkID != "" {
|
||||
exact = append(exact, graphRecoveryCandidate{ID: *chunkID, KnowledgeID: *knowledgeID, ContentHash: *sourceHash})
|
||||
}
|
||||
dsn, err := graphRecoveryDSN()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||
if err != nil {
|
||||
return errors.New("cannot open graph recovery database")
|
||||
}
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer sqlDB.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
db = db.WithContext(ctx)
|
||||
if *apply {
|
||||
if *reconcileOnly && os.Getenv("GRAPH_LEDGER_RECONCILE_VERIFIED") != "1" {
|
||||
return errors.New("review the reconciliation preview first; then set GRAPH_LEDGER_RECONCILE_VERIFIED=1")
|
||||
}
|
||||
if !*reconcileOnly && os.Getenv("GRAPH_NATIVE_WRITE_VERIFIED") != "1" {
|
||||
return errors.New("deploy and verify native graph persistence first; then set GRAPH_NATIVE_WRITE_VERIFIED=1")
|
||||
}
|
||||
// Hold a session lock on a dedicated connection while ledger writes use
|
||||
// autocommit. Redis calls must never precede a durable attempt record.
|
||||
lockConn, err := sqlDB.Conn(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer lockConn.Close()
|
||||
var locked bool
|
||||
if err := lockConn.QueryRowContext(ctx, "SELECT pg_try_advisory_lock(20260919, 7351)").Scan(&locked); err != nil {
|
||||
return err
|
||||
}
|
||||
if !locked {
|
||||
return errors.New("another graph backfill dispatcher is active")
|
||||
}
|
||||
defer func() {
|
||||
unlockCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, _ = lockConn.ExecContext(unlockCtx, "SELECT pg_advisory_unlock(20260919, 7351)")
|
||||
}()
|
||||
}
|
||||
if *reconcileOnly {
|
||||
if !db.Migrator().HasTable(&graphRecoveryFailure{}) {
|
||||
return errors.New("graph failure ledger is missing; reconciliation will not create it")
|
||||
}
|
||||
redisOpt, err := graphRecoveryRedisOptions()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
inspector := asynq.NewInspector(redisOpt)
|
||||
defer inspector.Close()
|
||||
report, err := reconcileGraphTask(db, inspector, *tenant, *kbID, revision, exact[0], *apply)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(report)
|
||||
}
|
||||
var kb struct {
|
||||
Name, SummaryModelID string
|
||||
Enabled bool
|
||||
}
|
||||
result := db.Raw(`SELECT name,summary_model_id,COALESCE((extract_config->>'enabled')::boolean,false) AS enabled FROM knowledge_bases WHERE tenant_id=? AND id=? AND deleted_at IS NULL`, *tenant, *kbID).Scan(&kb)
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected != 1 {
|
||||
return errors.New("knowledge base not found in tenant")
|
||||
}
|
||||
if *apply && (!strings.EqualFold(os.Getenv("LEGACY_GRAPH_ENABLE"), "true") || !kb.Enabled || kb.SummaryModelID == "") {
|
||||
return errors.New("native graph backend, knowledge base extraction and summary model must be enabled first")
|
||||
}
|
||||
hasFailures := db.Migrator().HasTable(&graphRecoveryFailure{})
|
||||
if *apply && !hasFailures {
|
||||
if err := db.AutoMigrate(&graphRecoveryFailure{}); err != nil {
|
||||
return err
|
||||
}
|
||||
hasFailures = true
|
||||
}
|
||||
if *apply {
|
||||
if _, err := graphUnresolvedFailures(db, *tenant, *kbID, revision, true); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var coverage struct{ Documents, TextChunks, NativeCompleted, MappedChunks int64 }
|
||||
err = db.Raw(`WITH completed AS NOT MATERIALIZED(SELECT payload FROM native_graph_records WHERE tenant_id=? AND knowledge_base_id=? AND source_table='knowledgegraphchunkstate' AND payload->>'origin'='native_extraction_v1' AND payload->>'extraction_status'='complete' AND CAST(COALESCE(payload->>'relation_count','0') AS INTEGER)>0)
|
||||
SELECT count(DISTINCT c.knowledge_id) AS documents,count(*) AS text_chunks,
|
||||
count(*) FILTER(WHERE EXISTS(SELECT 1 FROM completed r WHERE r.payload->>'native_chunk_id'=c.id AND r.payload->>'native_knowledge_id'=c.knowledge_id AND r.payload->>'source_content_hash'=encode(sha256(convert_to(c.content,'UTF8')),'hex'))) AS native_completed,
|
||||
count(*) FILTER(WHERE EXISTS(SELECT 1 FROM native_graph_sources s WHERE s.tenant_id=c.tenant_id AND s.knowledge_base_id=c.knowledge_base_id AND s.native_chunk_id=c.id AND s.native_content_sha256=encode(sha256(convert_to(c.content,'UTF8')),'hex'))) AS mapped_chunks
|
||||
FROM chunks c JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id AND k.deleted_at IS NULL
|
||||
WHERE c.tenant_id=? AND c.knowledge_base_id=? AND c.deleted_at IS NULL AND c.is_enabled AND c.chunk_type='text'`, *tenant, *kbID, *tenant, *kbID).Scan(&coverage).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
candidates, err := graphRecoveryCandidates(db, *tenant, *kbID, revision, *after, *limit, hasFailures, *retry, exact...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report := map[string]any{"tenant_id": *tenant, "knowledge_base_id": *kbID, "knowledge_base_name": kb.Name, "graph_enabled": kb.Enabled, "coverage": coverage, "candidate_count": len(candidates), "after": *after, "next_cursor": *after, "apply": *apply, "enqueued": 0, "already_queued": 0, "retried": 0, "archived": 0, "candidates": candidates}
|
||||
if !*apply {
|
||||
if hasFailures {
|
||||
failures, err := graphUnresolvedFailures(db, *tenant, *kbID, revision, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report["failures"], report["failure_count"] = failures, len(failures)
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(report)
|
||||
}
|
||||
redisOpt, err := graphRecoveryRedisOptions()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client := asynq.NewClient(redisOpt)
|
||||
defer client.Close()
|
||||
inspector := asynq.NewInspector(redisOpt)
|
||||
defer inspector.Close()
|
||||
enqueued, existing, retried, archived, deferred := 0, 0, 0, 0, 0
|
||||
queueInfo, err := inspector.GetQueueInfo(types.QueueGraph)
|
||||
if err != nil && !errors.Is(err, asynq.ErrQueueNotFound) {
|
||||
return err
|
||||
}
|
||||
if queueInfo == nil {
|
||||
queueInfo = &asynq.QueueInfo{}
|
||||
}
|
||||
slots := graphQueueSlots(queueInfo, *maxOutstanding)
|
||||
settled := make(map[string]bool)
|
||||
for _, c := range candidates {
|
||||
if len(exact) > 0 {
|
||||
// Recheck immediately before dispatch. The worker checks the same
|
||||
// immutable identity before calling the model if a source changes later.
|
||||
if _, err := graphRecoveryCandidates(db, *tenant, *kbID, revision, 0, 1, hasFailures, *retry, exact...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
taskID := graphRecoveryTaskID(*tenant, *kbID, c, revision)
|
||||
info, inspectErr := inspector.GetTaskInfo(types.QueueGraph, taskID)
|
||||
if inspectErr != nil && !errors.Is(inspectErr, asynq.ErrTaskNotFound) && !errors.Is(inspectErr, asynq.ErrQueueNotFound) {
|
||||
return inspectErr
|
||||
}
|
||||
if info != nil {
|
||||
if len(exact) > 0 && (!graphTaskMatchesReviewedSource(info.Payload, *tenant, *kbID, c) || info.MaxRetry != 0) {
|
||||
return errors.New("existing graph task does not have the reviewed source and single-execution guard; inspect it without resetting the failure ledger")
|
||||
}
|
||||
existing++
|
||||
if err := trackGraphTask(db, *tenant, *kbID, revision, taskID, c); err != nil {
|
||||
return err
|
||||
}
|
||||
if info.State == asynq.TaskStateArchived || info.State == asynq.TaskStateCompleted {
|
||||
verified, err := graphCandidateCompleted(db, *tenant, *kbID, c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if verified {
|
||||
settled[c.ID] = true
|
||||
continue
|
||||
}
|
||||
reason := "completed_without_native_checkpoint"
|
||||
if info.State == asynq.TaskStateArchived {
|
||||
reason = graphArchivedReason(info.LastErr)
|
||||
archived++
|
||||
}
|
||||
if err := recordGraphFailure(db, *tenant, *kbID, revision, taskID, c, reason); err != nil {
|
||||
return err
|
||||
}
|
||||
settled[c.ID] = true
|
||||
if *retry && info.State == asynq.TaskStateArchived {
|
||||
didRetry, err := graphFailureRetry(db, inspector, taskID, *retryReason, slots)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if didRetry {
|
||||
retried++
|
||||
slots--
|
||||
settled[c.ID] = false
|
||||
}
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if slots == 0 {
|
||||
deferred++
|
||||
continue
|
||||
}
|
||||
reserved, err := reserveGraphDispatch(db, *tenant, *kbID, revision, taskID, c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !reserved {
|
||||
settled[c.ID] = true
|
||||
continue
|
||||
}
|
||||
// KnowledgeID is intentionally empty: recovery has no ingestion slot
|
||||
// to decrement and must not finalize a concurrent document upload.
|
||||
payload, err := json.Marshal(types.ExtractChunkPayload{TenantID: *tenant, ChunkID: c.ID, ModelID: kb.SummaryModelID, ChunkIndex: c.ChunkIndex})
|
||||
if len(exact) > 0 {
|
||||
payload, err = json.Marshal(types.ExtractChunkPayload{TenantID: *tenant, ChunkID: c.ID, ModelID: kb.SummaryModelID, ChunkIndex: c.ChunkIndex,
|
||||
ExpectedSource: &types.GraphSourceIdentity{KnowledgeBaseID: *kbID, KnowledgeID: c.KnowledgeID, ContentSHA256: c.ContentHash}})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
maxRetry, timeout := 3, 10*time.Minute
|
||||
if len(exact) > 0 {
|
||||
maxRetry, timeout = 0, 3*time.Minute
|
||||
}
|
||||
_, err = client.EnqueueContext(ctx, asynq.NewTask(types.TypeChunkExtract, payload), asynq.Queue(types.QueueGraph), asynq.TaskID(taskID), asynq.MaxRetry(maxRetry), asynq.Timeout(timeout), asynq.Retention(24*time.Hour))
|
||||
if errors.Is(err, asynq.ErrTaskIDConflict) {
|
||||
existing++
|
||||
// Another producer raced with this dispatcher. Count the slot
|
||||
// conservatively; the next poll will inspect its actual state.
|
||||
slots--
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
enqueued++
|
||||
slots--
|
||||
}
|
||||
report["enqueued"], report["already_queued"], report["retried"] = enqueued, existing, retried
|
||||
// Do not let a manual resume skip deferred or still-running work. Watch
|
||||
// always starts at zero and additionally uses persisted failure exclusions.
|
||||
report["next_cursor"] = graphResumeCursor(*after, candidates, settled)
|
||||
report["archived"] = archived
|
||||
report["deferred_for_queue_limit"], report["max_outstanding"] = deferred, *maxOutstanding
|
||||
report["queue_paused"] = queueInfo.Paused
|
||||
report["queue_limit_scope"] = "serialized_backfill_dispatchers; other graph producers may enqueue concurrently"
|
||||
failures, err := graphUnresolvedFailures(db, *tenant, *kbID, revision, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report["failures"], report["failure_count"] = failures, len(failures)
|
||||
return json.NewEncoder(os.Stdout).Encode(report)
|
||||
}
|
||||
|
||||
func graphBatchStatus(report map[string]any) (complete, blocked bool) {
|
||||
coverage, _ := report["coverage"].(map[string]any)
|
||||
textChunks, totalOK := coverage["TextChunks"].(float64)
|
||||
completedChunks, completedOK := coverage["NativeCompleted"].(float64)
|
||||
candidates, candidatesOK := report["candidate_count"].(float64)
|
||||
archived, _ := report["archived"].(float64)
|
||||
failures, _ := report["failure_count"].(float64)
|
||||
paused, _ := report["queue_paused"].(bool)
|
||||
valid := totalOK && completedOK && candidatesOK && textChunks >= 0 && completedChunks >= 0 && completedChunks <= textChunks && candidates >= 0 && archived >= 0 && failures >= 0
|
||||
complete = valid && completedChunks == textChunks && candidates == 0 && failures == 0 && archived == 0
|
||||
blocked = archived > 0 || failures > 0 || !valid || (!complete && (candidates == 0 || paused))
|
||||
return
|
||||
}
|
||||
|
||||
// A restart resumes from verified database checkpoints. Each knowledge base has
|
||||
// only one bounded window in flight; failed windows require explicit recovery.
|
||||
func watchGraphRecovery() error {
|
||||
flags := flag.NewFlagSet("watch", flag.ContinueOnError)
|
||||
tenant := flags.Uint64("tenant", 0, "required tenant")
|
||||
kbs := flags.String("kbs", "", "comma-separated knowledge base UUIDs")
|
||||
limit := flags.Int("limit", 2, "maximum candidate window per knowledge base (1-20)")
|
||||
maxOutstanding := flags.Int("max-outstanding", 2, "maximum outstanding tasks across the shared graph queue (1-20)")
|
||||
interval := flags.Duration("interval", time.Minute, "poll interval (at least 30s)")
|
||||
stateFile := flags.String("state", "/data/progress.json", "durable progress report")
|
||||
if err := flags.Parse(os.Args[2:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if *tenant == 0 || *limit < 1 || *limit > 20 || *interval < 30*time.Second || *kbs == "" || *maxOutstanding < 1 || *maxOutstanding > 20 {
|
||||
return errors.New("watch requires tenant, knowledge bases, limit 1-20 and interval >=30s")
|
||||
}
|
||||
ids := strings.Split(*kbs, ",")
|
||||
if len(ids) > 10 {
|
||||
return errors.New("watch supports at most 10 knowledge bases")
|
||||
}
|
||||
for _, id := range ids {
|
||||
if _, err := uuid.Parse(id); err != nil {
|
||||
return errors.New("invalid knowledge base in watch scope")
|
||||
}
|
||||
}
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
for {
|
||||
reports := []map[string]any{}
|
||||
complete, blocked := true, false
|
||||
for _, id := range ids {
|
||||
commandCtx, cancel := context.WithTimeout(ctx, 6*time.Minute)
|
||||
command := exec.CommandContext(commandCtx, executable, "-tenant", strconv.FormatUint(*tenant, 10), "-kb", id, "-limit", strconv.Itoa(*limit), "-max-outstanding", strconv.Itoa(*maxOutstanding), "-apply")
|
||||
output, commandErr := command.Output()
|
||||
cancel()
|
||||
var report map[string]any
|
||||
if commandErr != nil || json.Unmarshal(output, &report) != nil {
|
||||
report = map[string]any{"knowledge_base_id": id, "error": "batch command failed; check runtime and model configuration"}
|
||||
blocked, complete = true, false
|
||||
} else {
|
||||
delete(report, "candidates")
|
||||
batchComplete, batchBlocked := graphBatchStatus(report)
|
||||
complete = complete && batchComplete
|
||||
blocked = blocked || batchBlocked
|
||||
if batchBlocked {
|
||||
report["attention"] = "Native coverage is incomplete or extraction failures need recovery"
|
||||
}
|
||||
}
|
||||
reports = append(reports, report)
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
status := "running"
|
||||
if complete {
|
||||
status = "complete"
|
||||
} else if blocked {
|
||||
status = "needs_attention"
|
||||
}
|
||||
state := map[string]any{"status": status, "checked_at": time.Now().UTC(), "tenant_id": *tenant, "batch_limit_per_kb": *limit, "max_outstanding": *maxOutstanding, "knowledge_bases": reports}
|
||||
data, err := json.MarshalIndent(state, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = os.MkdirAll(filepath.Dir(*stateFile), 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = os.WriteFile(*stateFile+".tmp", data, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = os.Rename(*stateFile+".tmp", *stateFile); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("{\"status\":%q,\"knowledge_bases\":%d}\n", status, len(reports))
|
||||
if complete {
|
||||
// Keep the successful checkpoint visible without restart-policy loops.
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(*interval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
var err error
|
||||
if len(os.Args) > 1 && os.Args[1] == "--watch" {
|
||||
err = watchGraphRecovery()
|
||||
} else {
|
||||
err = runGraphRecovery()
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,848 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/mattn/go-sqlite3"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
func TestGraphExactTaskCannotReuseAnUnpinnedPayload(t *testing.T) {
|
||||
candidate := graphRecoveryCandidate{ID: "chunk", KnowledgeID: "doc", ContentHash: strings.Repeat("a", 64)}
|
||||
for _, change := range []string{"unchanged", "legacy", "tenant", "chunk", "kb", "document", "hash", "invalid"} {
|
||||
t.Run(change, func(t *testing.T) {
|
||||
task := types.ExtractChunkPayload{TenantID: 1, ChunkID: candidate.ID, ExpectedSource: &types.GraphSourceIdentity{KnowledgeBaseID: "kb", KnowledgeID: candidate.KnowledgeID, ContentSHA256: candidate.ContentHash}}
|
||||
switch change {
|
||||
case "legacy":
|
||||
task.ExpectedSource = nil
|
||||
case "tenant":
|
||||
task.TenantID = 2
|
||||
case "chunk":
|
||||
task.ChunkID = "other"
|
||||
case "kb":
|
||||
task.ExpectedSource.KnowledgeBaseID = "other"
|
||||
case "document":
|
||||
task.ExpectedSource.KnowledgeID = "other"
|
||||
case "hash":
|
||||
task.ExpectedSource.ContentSHA256 = strings.Repeat("b", 64)
|
||||
}
|
||||
payload, _ := json.Marshal(task)
|
||||
if change == "invalid" {
|
||||
payload = []byte("invalid")
|
||||
}
|
||||
if graphTaskMatchesReviewedSource(payload, 1, "kb", candidate) != (change == "unchanged") {
|
||||
t.Fatal("unsafe existing task accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphExactScopeValidation(t *testing.T) {
|
||||
const chunk = "e40b74eb-0ac7-4ff6-a107-00ce5adfe610"
|
||||
const knowledge = "9669e1b1-e28e-49a4-ae13-e97e37e21b11"
|
||||
const hash = "4c4d84679328fc6dd3c17add7a91d13669c2d0cc2bc507098ed4fcc1c0dd84b2"
|
||||
for _, tc := range []struct {
|
||||
name, chunk, knowledge, hash string
|
||||
after int64
|
||||
limit, outstanding int
|
||||
valid bool
|
||||
}{
|
||||
{"normal batch", "", "", "", 2, 20, 2, true},
|
||||
{"exact", chunk, knowledge, hash, 0, 1, 1, true},
|
||||
{"missing chunk", "", knowledge, hash, 0, 1, 1, false},
|
||||
{"missing knowledge", chunk, "", hash, 0, 1, 1, false},
|
||||
{"missing hash", chunk, knowledge, "", 0, 1, 1, false},
|
||||
{"invalid chunk", "other", knowledge, hash, 0, 1, 1, false},
|
||||
{"nil uuid", "00000000-0000-0000-0000-000000000000", knowledge, hash, 0, 1, 1, false},
|
||||
{"noncanonical chunk", strings.ToUpper(chunk), knowledge, hash, 0, 1, 1, false},
|
||||
{"invalid hash", chunk, knowledge, strings.Repeat("z", 64), 0, 1, 1, false},
|
||||
{"uppercase hash", chunk, knowledge, strings.ToUpper(hash), 0, 1, 1, false},
|
||||
{"short hash", chunk, knowledge, hash[:62], 0, 1, 1, false},
|
||||
{"cursor", chunk, knowledge, hash, 1, 1, 1, false},
|
||||
{"batch", chunk, knowledge, hash, 0, 2, 1, false},
|
||||
{"parallel", chunk, knowledge, hash, 0, 1, 2, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := graphValidateExactScope(tc.chunk, tc.knowledge, tc.hash, tc.after, tc.limit, tc.outstanding)
|
||||
if (err == nil) != tc.valid {
|
||||
t.Fatalf("valid=%t err=%v", tc.valid, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphExactScopeNeverSubstitutesNextCandidate(t *testing.T) {
|
||||
for _, change := range []string{"unchanged", "hash", "document", "disabled", "deleted", "document deleted", "unparsed", "completed", "failed", "other tenant", "other kb", "missing"} {
|
||||
t.Run(change, func(t *testing.T) {
|
||||
db := graphTestDB(t, filepath.Join(t.TempDir(), "exact.db"))
|
||||
wanted := seedGraphChunk(t, db, 1, "kb", "wanted", "reviewed source", 1)
|
||||
seedGraphChunk(t, db, 1, "kb", "next", "next source", 2)
|
||||
switch change {
|
||||
case "hash":
|
||||
db.Exec("UPDATE chunks SET content='changed' WHERE id='wanted'")
|
||||
case "document":
|
||||
wanted.KnowledgeID = "other"
|
||||
case "disabled":
|
||||
db.Exec("UPDATE chunks SET is_enabled=false WHERE id='wanted'")
|
||||
case "deleted":
|
||||
db.Exec("UPDATE chunks SET deleted_at='2026-09-20' WHERE id='wanted'")
|
||||
case "document deleted":
|
||||
db.Exec("UPDATE knowledges SET deleted_at='2026-09-20' WHERE id=?", wanted.KnowledgeID)
|
||||
case "unparsed":
|
||||
db.Exec("UPDATE knowledges SET parse_status='pending' WHERE id=?", wanted.KnowledgeID)
|
||||
case "completed":
|
||||
saveGraphCheckpoint(t, db, 1, "kb", wanted, wanted.ContentHash)
|
||||
case "failed":
|
||||
if err := recordGraphFailure(db, 1, "kb", "grounded-v6", graphRecoveryTaskID(1, "kb", wanted, "grounded-v6"), wanted, "review_required"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "other tenant":
|
||||
db.Exec("UPDATE chunks SET tenant_id=2 WHERE id='wanted'")
|
||||
case "other kb":
|
||||
db.Exec("UPDATE chunks SET knowledge_base_id='other' WHERE id='wanted'")
|
||||
case "missing":
|
||||
wanted.ID = "absent"
|
||||
}
|
||||
got, err := graphRecoveryCandidates(db, 1, "kb", "grounded-v6", 0, 1, true, false, wanted)
|
||||
if change == "unchanged" {
|
||||
if err != nil || len(got) != 1 || got[0].ID != wanted.ID {
|
||||
t.Fatalf("exact source: %v %v", got, err)
|
||||
}
|
||||
} else if err == nil || len(got) != 0 {
|
||||
t.Fatalf("changed source selected a candidate: %v %v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphWatcherRequiresVerifiedCoverage(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name, report string
|
||||
complete, blocked bool
|
||||
}{
|
||||
{"all source chunks verified", `{"coverage":{"TextChunks":51,"NativeCompleted":51},"candidate_count":0}`, true, false},
|
||||
{"empty knowledge base", `{"coverage":{"TextChunks":0,"NativeCompleted":0},"candidate_count":0}`, true, false},
|
||||
{"unparsed source excludes remaining candidates", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":0}`, false, true},
|
||||
{"queued work is not complete", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":2,"already_queued":2}`, false, false},
|
||||
{"exhausted retries need attention", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":2,"archived":1}`, false, true},
|
||||
{"old mapping is not extraction", `{"coverage":{"TextChunks":51,"MappedChunks":51,"NativeCompleted":0},"candidate_count":0}`, false, true},
|
||||
{"missing coverage fails closed", `{"candidate_count":0}`, false, true},
|
||||
{"persistent failures survive candidate exclusion", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":0,"failure_count":2}`, false, true},
|
||||
{"failures remain visible while later work progresses", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":1,"failure_count":1}`, false, true},
|
||||
{"stale failure report must not claim complete", `{"coverage":{"TextChunks":51,"NativeCompleted":51},"candidate_count":0,"failure_count":1}`, false, true},
|
||||
{"impossible coverage fails closed", `{"coverage":{"TextChunks":51,"NativeCompleted":52},"candidate_count":1}`, false, true},
|
||||
{"paused queue needs attention", `{"coverage":{"TextChunks":51,"NativeCompleted":49},"candidate_count":2,"queue_paused":true}`, false, true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var report map[string]any
|
||||
if err := json.Unmarshal([]byte(test.report), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
complete, blocked := graphBatchStatus(report)
|
||||
if complete != test.complete || blocked != test.blocked {
|
||||
t.Fatalf("got complete=%t blocked=%t; want %t %t", complete, blocked, test.complete, test.blocked)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphArchivedReasonDoesNotExposeProviderDetails(t *testing.T) {
|
||||
for _, tc := range []struct{ message, reason string }{
|
||||
{"graph_needs_review: no source-grounded relationships: skip retry for the task", "relation_coverage_review_required"},
|
||||
{"provider rejected secret credential", "archived"},
|
||||
{"read response: context deadline exceeded; credential=secret", "provider_response_timeout"},
|
||||
{"read response: net/http: timeout awaiting response headers", "provider_response_timeout"},
|
||||
{"context deadline exceeded", "task_timeout"},
|
||||
{"context canceled", "task_cancelled"},
|
||||
{"", "archived"},
|
||||
} {
|
||||
if got := graphArchivedReason(tc.message); got != tc.reason {
|
||||
t.Fatalf("got %q; want %q", got, tc.reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
// The production candidate SQL is exercised unchanged. These deterministic
|
||||
// functions provide PostgreSQL's content-hash operations to the test database.
|
||||
sql.Register("graph_backfill_test", &sqlite3.SQLiteDriver{ConnectHook: func(conn *sqlite3.SQLiteConn) error {
|
||||
if err := conn.RegisterFunc("convert_to", func(content, _ string) string { return content }, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := conn.RegisterFunc("sha256", func(content string) []byte { hash := sha256.Sum256([]byte(content)); return hash[:] }, true); err != nil {
|
||||
return err
|
||||
}
|
||||
return conn.RegisterFunc("encode", func(content []byte, _ string) string { return hex.EncodeToString(content) }, true)
|
||||
}})
|
||||
}
|
||||
|
||||
func graphTestDB(t *testing.T, path string) *gorm.DB {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Dialector{DriverName: "graph_backfill_test", DSN: path}, &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
if err := db.AutoMigrate(&graphRecoveryFailure{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, statement := range []string{
|
||||
`CREATE TABLE IF NOT EXISTS knowledge_bases(id TEXT,tenant_id INTEGER,deleted_at TEXT)`,
|
||||
`CREATE TABLE IF NOT EXISTS chunks(id TEXT,tenant_id INTEGER,knowledge_base_id TEXT,knowledge_id TEXT,seq_id INTEGER,chunk_index INTEGER,content TEXT,deleted_at TEXT,is_enabled BOOLEAN,chunk_type TEXT)`,
|
||||
`CREATE TABLE IF NOT EXISTS knowledges(id TEXT,tenant_id INTEGER,knowledge_base_id TEXT,deleted_at TEXT,parse_status TEXT)`,
|
||||
`CREATE TABLE IF NOT EXISTS native_graph_records(tenant_id INTEGER,knowledge_base_id TEXT,source_table TEXT,payload JSON)`,
|
||||
} {
|
||||
if err := db.Exec(statement).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func seedGraphChunk(t *testing.T, db *gorm.DB, tenant uint64, kbID, id, content string, seq int) graphRecoveryCandidate {
|
||||
t.Helper()
|
||||
if err := db.Exec(`INSERT INTO knowledge_bases(id,tenant_id) SELECT ?,? WHERE NOT EXISTS(SELECT 1 FROM knowledge_bases WHERE id=? AND tenant_id=?)`, kbID, tenant, kbID, tenant).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
knowledgeID := "knowledge-" + id
|
||||
if err := db.Exec(`INSERT INTO knowledges(id,tenant_id,knowledge_base_id,parse_status) VALUES(?,?,?,'completed')`, knowledgeID, tenant, kbID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec(`INSERT INTO chunks(id,tenant_id,knowledge_base_id,knowledge_id,seq_id,chunk_index,content,is_enabled,chunk_type) VALUES(?,?,?,?,?,0,?,true,'text')`, id, tenant, kbID, knowledgeID, seq, content).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hash := sha256.Sum256([]byte(content))
|
||||
return graphRecoveryCandidate{ID: id, KnowledgeID: knowledgeID, SeqID: int64(seq), ContentHash: hex.EncodeToString(hash[:])}
|
||||
}
|
||||
|
||||
type graphTestReader struct {
|
||||
info *asynq.TaskInfo
|
||||
err error
|
||||
reads int
|
||||
onRead func()
|
||||
}
|
||||
|
||||
func (r *graphTestReader) GetTaskInfo(_, _ string) (*asynq.TaskInfo, error) {
|
||||
r.reads++
|
||||
if r.onRead != nil {
|
||||
r.onRead()
|
||||
}
|
||||
return r.info, r.err
|
||||
}
|
||||
|
||||
func seedGraphReconciliation(t *testing.T) (*gorm.DB, graphRecoveryCandidate, *graphTestReader) {
|
||||
t.Helper()
|
||||
db := graphTestDB(t, filepath.Join(t.TempDir(), "reconcile.db"))
|
||||
c := seedGraphChunk(t, db, 1, "kb", "source", "reviewed source", 1)
|
||||
id := graphRecoveryTaskID(1, "kb", c, "grounded-v6")
|
||||
if ok, err := reserveGraphDispatch(db, 1, "kb", "grounded-v6", id, c); err != nil || !ok {
|
||||
t.Fatalf("reserve: %t %v", ok, err)
|
||||
}
|
||||
payload, err := json.Marshal(types.ExtractChunkPayload{TenantID: 1, ChunkID: c.ID, ExpectedSource: &types.GraphSourceIdentity{KnowledgeBaseID: "kb", KnowledgeID: c.KnowledgeID, ContentSHA256: c.ContentHash}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db, c, &graphTestReader{info: &asynq.TaskInfo{ID: id, Queue: types.QueueGraph, Type: types.TypeChunkExtract, Payload: payload, State: asynq.TaskStateArchived, LastErr: "read response: context deadline exceeded; credential=must-not-leak", LastFailedAt: time.Date(2026, 9, 20, 8, 46, 33, 0, time.UTC)}}
|
||||
}
|
||||
|
||||
func graphReadLedger(t *testing.T, db *gorm.DB, id string) graphRecoveryFailure {
|
||||
t.Helper()
|
||||
var ledger graphRecoveryFailure
|
||||
if err := db.First(&ledger, "task_id=?", id).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ledger
|
||||
}
|
||||
|
||||
func TestGraphReconciliationRequiresExactNonDispatchMode(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, chunk, revision, retryReason string
|
||||
retry, valid bool
|
||||
}{
|
||||
{name: "exact", chunk: "chunk", revision: "grounded-v6", valid: true},
|
||||
{name: "missing source", revision: "grounded-v6"},
|
||||
{name: "missing revision", chunk: "chunk"},
|
||||
{name: "blank revision", chunk: "chunk", revision: " "},
|
||||
{name: "retry requested", chunk: "chunk", revision: "grounded-v6", retry: true},
|
||||
{name: "retry reason supplied", chunk: "chunk", revision: "grounded-v6", retryReason: "retry attempt"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := graphValidateReconciliation(true, tc.chunk, tc.revision, tc.retry, tc.retryReason)
|
||||
if (err == nil) != tc.valid {
|
||||
t.Fatalf("valid=%t err=%v", tc.valid, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphReconciliationCLIRejectsMixedModesBeforeOpeningDatabase(t *testing.T) {
|
||||
const chunk = "edf40fb3-f09d-4532-a6fa-533408fe19cb"
|
||||
const knowledge = "c3db6397-6b50-4d1e-a667-2e3efe503cc7"
|
||||
const hash = "63bdc854af31ea50dd0360d93a8b2340391e15e5a1c251db714ca1e9533b2e5b"
|
||||
for _, tc := range []struct {
|
||||
name, flag, value, want string
|
||||
}{
|
||||
{"retry", "--retry-failed", "true", "no retry flags"},
|
||||
{"retry reason", "--retry-reason", "previous investigation", "no retry flags"},
|
||||
{"cursor", "--after", "1", "no cursor"},
|
||||
{"batch", "--limit", "2", "limit=1"},
|
||||
{"parallel", "--max-outstanding", "2", "max-outstanding=1"},
|
||||
{"missing hash", "--source-sha256", "", "source SHA-256"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
originalFlags, originalArgs := flag.CommandLine, os.Args
|
||||
defer func() { flag.CommandLine, os.Args = originalFlags, originalArgs }()
|
||||
flag.CommandLine = flag.NewFlagSet("reconcile-test", flag.ContinueOnError)
|
||||
t.Setenv("GRAPH_BACKFILL_REVISION", "grounded-v6")
|
||||
t.Setenv("GRAPH_BACKFILL_DSN", "invalid connection must not be opened")
|
||||
os.Args = []string{"reconcile-test", "--reconcile-only", "--tenant", "10059", "--kb", "16c3e59c-4bb6-41cf-ae0a-6d0941b3d032", "--chunk", chunk, "--knowledge", knowledge, "--source-sha256", hash, "--limit", "1", "--max-outstanding", "1", tc.flag + "=" + tc.value}
|
||||
if err := runGraphRecovery(); err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("expected validation %q before connection, got %v", tc.want, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphReconciliationSanitizesPostgresCheckpointErrors(t *testing.T) {
|
||||
conn, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
db, err := gorm.Open(postgres.New(postgres.Config{Conn: conn}), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mock.ExpectQuery("SELECT .* AS current.* FROM native_graph_backfill_failures").WithArgs("task").WillReturnError(errors.New(`invalid input syntax for type integer: "private-source-value"`))
|
||||
current, complete, err := graphReconcileSourceStatus(db, "task")
|
||||
if err == nil || current || complete || strings.Contains(err.Error(), "private-source-value") {
|
||||
t.Fatalf("unsafe checkpoint query outcome: %t %t %v", current, complete, err)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphReconciliationPreviewThenApplyIsBoundedAndIdempotent(t *testing.T) {
|
||||
db, c, reader := seedGraphReconciliation(t)
|
||||
original := graphReadLedger(t, db, reader.info.ID)
|
||||
other := seedGraphChunk(t, db, 1, "kb", "other", "untouched source", 2)
|
||||
otherID := graphRecoveryTaskID(1, "kb", other, "grounded-v6")
|
||||
if _, err := reserveGraphDispatch(db, 1, "kb", "grounded-v6", otherID, other); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unrelated := graphReadLedger(t, db, otherID)
|
||||
preview, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, false)
|
||||
if err != nil || !preview.WouldUpdate || preview.LedgerUpdated || !preview.Blocked || preview.Reason != "provider_response_timeout" || preview.ProposedState != "failed" {
|
||||
t.Fatalf("preview: %+v %v", preview, err)
|
||||
}
|
||||
if got := graphReadLedger(t, db, reader.info.ID); !reflect.DeepEqual(got, original) {
|
||||
t.Fatal("preview changed durable state")
|
||||
}
|
||||
result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
|
||||
if err != nil || !result.LedgerUpdated || result.Enqueued != 0 || result.Retried != 0 || result.ModelCalls != 0 {
|
||||
t.Fatalf("apply: %+v %v", result, err)
|
||||
}
|
||||
updated := graphReadLedger(t, db, reader.info.ID)
|
||||
if updated.State != "failed" || updated.Reason != "provider_response_timeout" || !updated.FirstFailedAt.Equal(reader.info.LastFailedAt) || updated.ManualRetries != 0 {
|
||||
t.Fatalf("failure not durably recorded: %+v", updated)
|
||||
}
|
||||
encoded, _ := json.Marshal(result)
|
||||
if strings.Contains(string(encoded), "must-not-leak") {
|
||||
t.Fatal("report exposed provider details")
|
||||
}
|
||||
result, err = reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
|
||||
if err != nil || result.WouldUpdate || result.LedgerUpdated || !reflect.DeepEqual(updated, graphReadLedger(t, db, reader.info.ID)) {
|
||||
t.Fatalf("idempotence: %+v %v", result, err)
|
||||
}
|
||||
if !reflect.DeepEqual(unrelated, graphReadLedger(t, db, otherID)) {
|
||||
t.Fatal("reconciliation changed another source ledger")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphReconciliationRefinesArchivedReasonWithoutLosingHistory(t *testing.T) {
|
||||
db, c, reader := seedGraphReconciliation(t)
|
||||
failedAt := time.Date(2026, 9, 18, 1, 2, 3, 0, time.UTC)
|
||||
if err := db.Model(&graphRecoveryFailure{}).Where("task_id=?", reader.info.ID).Updates(map[string]any{"state": "failed", "reason": "archived", "first_failed_at": failedAt, "manual_retries": 1, "retry_reason": "previous authorized recovery"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
|
||||
after := graphReadLedger(t, db, reader.info.ID)
|
||||
if err != nil || !result.LedgerUpdated || after.Reason != "provider_response_timeout" || !after.FirstFailedAt.Equal(failedAt) || after.ManualRetries != 1 || after.RetryReason != "previous authorized recovery" {
|
||||
t.Fatalf("failure history changed: %+v %+v %v", result, after, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphReconciliationRequiresTerminalQueueAndPinnedIdentity(t *testing.T) {
|
||||
for _, change := range []string{"missing task", "missing queue", "nil task", "redis error", "active", "pending", "retry", "payload", "top document", "type", "queue", "task id", "max retry", "source hash", "source disabled", "source deleted", "document deleted", "unparsed", "kb deleted", "tenant", "document identity", "resolved", "superseded"} {
|
||||
t.Run(change, func(t *testing.T) {
|
||||
db, c, reader := seedGraphReconciliation(t)
|
||||
id := reader.info.ID
|
||||
switch change {
|
||||
case "missing task":
|
||||
reader.err = asynq.ErrTaskNotFound
|
||||
case "missing queue":
|
||||
reader.err = asynq.ErrQueueNotFound
|
||||
case "nil task":
|
||||
reader.info = nil
|
||||
case "redis error":
|
||||
reader.err = errors.New("secret connection details")
|
||||
case "active":
|
||||
reader.info.State = asynq.TaskStateActive
|
||||
case "pending":
|
||||
reader.info.State = asynq.TaskStatePending
|
||||
case "retry":
|
||||
reader.info.State = asynq.TaskStateRetry
|
||||
case "payload":
|
||||
reader.info.Payload = []byte(`{"tenant_id":2}`)
|
||||
case "top document":
|
||||
var p types.ExtractChunkPayload
|
||||
_ = json.Unmarshal(reader.info.Payload, &p)
|
||||
p.KnowledgeID = "other-document"
|
||||
reader.info.Payload, _ = json.Marshal(p)
|
||||
case "type":
|
||||
reader.info.Type = "other"
|
||||
case "queue":
|
||||
reader.info.Queue = "other"
|
||||
case "task id":
|
||||
reader.info.ID = "other"
|
||||
case "max retry":
|
||||
reader.info.MaxRetry = 1
|
||||
case "source hash":
|
||||
db.Exec("UPDATE chunks SET content='changed'")
|
||||
case "source disabled":
|
||||
db.Exec("UPDATE chunks SET is_enabled=false")
|
||||
case "source deleted":
|
||||
db.Exec("UPDATE chunks SET deleted_at='2026-09-20'")
|
||||
case "document deleted":
|
||||
db.Exec("UPDATE knowledges SET deleted_at='2026-09-20'")
|
||||
case "unparsed":
|
||||
db.Exec("UPDATE knowledges SET parse_status='processing'")
|
||||
case "kb deleted":
|
||||
db.Exec("UPDATE knowledge_bases SET deleted_at='2026-09-20'")
|
||||
case "tenant":
|
||||
db.Exec("UPDATE chunks SET tenant_id=2")
|
||||
case "document identity":
|
||||
c.KnowledgeID = "other-document"
|
||||
case "resolved", "superseded":
|
||||
db.Model(&graphRecoveryFailure{}).Where("task_id=?", id).Update("state", change)
|
||||
}
|
||||
before := graphReadLedger(t, db, id)
|
||||
result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
|
||||
if (err != nil) != (change == "redis error") || result.LedgerUpdated || result.WouldUpdate || result.Enqueued != 0 || result.Retried != 0 || result.ModelCalls != 0 {
|
||||
t.Fatalf("unsafe reconcile: %+v %v", result, err)
|
||||
}
|
||||
if err != nil && strings.Contains(err.Error(), "secret") {
|
||||
t.Fatal("inspection error exposed connection details")
|
||||
}
|
||||
if change != "active" && change != "pending" && change != "retry" && !result.Blocked {
|
||||
t.Fatalf("unsafe case not blocked: %+v", result)
|
||||
}
|
||||
if !reflect.DeepEqual(before, graphReadLedger(t, db, id)) {
|
||||
t.Fatal("blocked reconciliation changed ledger")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphReconciliationRequiresCurrentNonemptyCheckpointAndPreservesHistory(t *testing.T) {
|
||||
for _, checkpoint := range []string{"current", "stale hash", "other tenant", "other kb", "other document", "empty", "none", "missing redis"} {
|
||||
t.Run(checkpoint, func(t *testing.T) {
|
||||
db, c, reader := seedGraphReconciliation(t)
|
||||
reader.info.State = asynq.TaskStateCompleted
|
||||
failedAt := time.Date(2026, 9, 18, 0, 0, 0, 0, time.UTC)
|
||||
if err := db.Model(&graphRecoveryFailure{}).Where("task_id=?", reader.info.ID).Updates(map[string]any{"state": "failed", "reason": "provider_response_timeout", "first_failed_at": failedAt, "manual_retries": 1, "retry_reason": "previous authorized recovery"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if checkpoint != "none" {
|
||||
hash := c.ContentHash
|
||||
if checkpoint == "stale hash" {
|
||||
hash = "old-hash"
|
||||
}
|
||||
saveGraphCheckpoint(t, db, 1, "kb", c, hash)
|
||||
}
|
||||
switch checkpoint {
|
||||
case "other tenant":
|
||||
db.Exec("UPDATE native_graph_records SET tenant_id=2")
|
||||
case "other kb":
|
||||
db.Exec("UPDATE native_graph_records SET knowledge_base_id='other'")
|
||||
case "other document":
|
||||
db.Exec(`UPDATE native_graph_records SET payload=json_set(payload,'$.native_knowledge_id','other')`)
|
||||
case "empty":
|
||||
db.Exec(`UPDATE native_graph_records SET payload=json_set(payload,'$.relation_count',0)`)
|
||||
case "missing redis":
|
||||
reader.err = asynq.ErrTaskNotFound
|
||||
}
|
||||
before := graphReadLedger(t, db, reader.info.ID)
|
||||
result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
|
||||
if err != nil || result.LedgerUpdated != (checkpoint == "current") || result.Blocked == (checkpoint == "current") {
|
||||
t.Fatalf("checkpoint reconciliation: %+v %v", result, err)
|
||||
}
|
||||
after := graphReadLedger(t, db, reader.info.ID)
|
||||
if checkpoint == "current" && after.State != "resolved" {
|
||||
t.Fatal("verified checkpoint not resolved")
|
||||
}
|
||||
if after.Reason != before.Reason || !after.FirstFailedAt.Equal(before.FirstFailedAt) || after.ManualRetries != before.ManualRetries || after.RetryReason != before.RetryReason {
|
||||
t.Fatal("reconciliation discarded prior failure history")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphReconciliationRechecksSourceAfterQueueRead(t *testing.T) {
|
||||
db, c, reader := seedGraphReconciliation(t)
|
||||
before := graphReadLedger(t, db, reader.info.ID)
|
||||
reader.onRead = func() { db.Exec("UPDATE chunks SET content='concurrent edit'") }
|
||||
result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
|
||||
if err != nil || !result.Blocked || result.LedgerUpdated || result.Reason != "source_changed_or_unavailable" || !reflect.DeepEqual(before, graphReadLedger(t, db, reader.info.ID)) {
|
||||
t.Fatalf("source edit lost: %+v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphReconciliationCannotOverrideManualSemanticRejection(t *testing.T) {
|
||||
db, c, reader := seedGraphReconciliation(t)
|
||||
saveGraphCheckpoint(t, db, 1, "kb", c, c.ContentHash)
|
||||
if err := recordGraphFailure(db, 1, "kb", "grounded-v6", reader.info.ID, c, "semantic_review_rejected_product_scope"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := graphReadLedger(t, db, reader.info.ID)
|
||||
result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
|
||||
if err != nil || !result.Blocked || result.LedgerUpdated || result.Reason != "manual_semantic_review_required" || !reflect.DeepEqual(before, graphReadLedger(t, db, reader.info.ID)) {
|
||||
t.Fatalf("manual rejection was overridden: %+v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphReconciliationAtomicUpdateChecksConcurrentChanges(t *testing.T) {
|
||||
for _, change := range []string{"source", "checkpoint", "ledger", "ledger identity"} {
|
||||
t.Run(change, func(t *testing.T) {
|
||||
db, c, reader := seedGraphReconciliation(t)
|
||||
before := graphReadLedger(t, db, reader.info.ID)
|
||||
fired := false
|
||||
if err := db.Callback().Update().Before("gorm:update").Register("test:concurrent_change", func(tx *gorm.DB) {
|
||||
if fired {
|
||||
return
|
||||
}
|
||||
fired = true
|
||||
tx = tx.Session(&gorm.Session{NewDB: true})
|
||||
switch change {
|
||||
case "source":
|
||||
tx.Exec("UPDATE chunks SET content='concurrent edit'")
|
||||
case "checkpoint":
|
||||
saveGraphCheckpoint(t, tx, 1, "kb", c, c.ContentHash)
|
||||
case "ledger":
|
||||
tx.Exec("UPDATE native_graph_backfill_failures SET state='retry_requested', manual_retries=1 WHERE task_id=?", reader.info.ID)
|
||||
case "ledger identity":
|
||||
tx.Exec("UPDATE native_graph_backfill_failures SET knowledge_id='other-document' WHERE task_id=?", reader.info.ID)
|
||||
}
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := reconcileGraphTask(db, reader, 1, "kb", "grounded-v6", c, true)
|
||||
if err != nil || !fired || !result.Blocked || result.LedgerUpdated || result.Reason != "reconciliation_state_changed" {
|
||||
t.Fatalf("concurrent change overwritten: %+v %v", result, err)
|
||||
}
|
||||
after := graphReadLedger(t, db, reader.info.ID)
|
||||
if after.Reason != before.Reason || !after.FirstFailedAt.Equal(before.FirstFailedAt) {
|
||||
t.Fatal("failed compare-and-update mutated failure history")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func saveGraphCheckpoint(t *testing.T, db *gorm.DB, tenant uint64, kbID string, c graphRecoveryCandidate, hash string) {
|
||||
t.Helper()
|
||||
payload, err := json.Marshal(map[string]string{"origin": "native_extraction_v1", "extraction_status": "complete", "relation_count": "1", "native_chunk_id": c.ID, "native_knowledge_id": c.KnowledgeID, "source_content_hash": hash})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec(`INSERT INTO native_graph_records(tenant_id,knowledge_base_id,source_table,payload) VALUES(?,?,'knowledgegraphchunkstate',?)`, tenant, kbID, string(payload)).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphFailureAllowsLaterCandidateAfterRestart(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "recovery.db")
|
||||
db := graphTestDB(t, path)
|
||||
first := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
|
||||
second := seedGraphChunk(t, db, 1, "kb", "second", "second content", 2)
|
||||
taskID := graphRecoveryTaskID(1, "kb", first, "revision")
|
||||
reserved, err := reserveGraphDispatch(db, 1, "kb", "revision", taskID, first)
|
||||
if err != nil || !reserved {
|
||||
t.Fatalf("reserve: %t %v", reserved, err)
|
||||
}
|
||||
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, first, "archived"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, _ := db.DB()
|
||||
if err := sqlDB.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db = graphTestDB(t, path)
|
||||
candidates, err := graphRecoveryCandidates(db, 1, "kb", "revision", 0, 1, true, false)
|
||||
if err != nil || len(candidates) != 1 || candidates[0].ID != second.ID {
|
||||
t.Fatalf("blocked behind archived task: %+v %v", candidates, err)
|
||||
}
|
||||
failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", false)
|
||||
if err != nil || len(failures) != 1 || failures[0].Reason != "archived" || failures[0].FirstFailedAt.IsZero() {
|
||||
t.Fatalf("lost failure accounting: %+v %v", failures, err)
|
||||
}
|
||||
// A new extraction revision and another tenant must not inherit this exclusion.
|
||||
candidates, err = graphRecoveryCandidates(db, 1, "kb", "new-revision", 0, 1, true, false)
|
||||
if err != nil || len(candidates) != 1 || candidates[0].ID != first.ID {
|
||||
t.Fatalf("revision scope: %+v %v", candidates, err)
|
||||
}
|
||||
seedGraphChunk(t, db, 2, "kb", "first", "first content", 1)
|
||||
candidates, err = graphRecoveryCandidates(db, 2, "kb", "revision", 0, 1, true, false)
|
||||
if err != nil || len(candidates) != 1 || candidates[0].ID != first.ID {
|
||||
t.Fatalf("tenant scope: %+v %v", candidates, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphLegacyEmptyCheckpointIsNotCompletedOrResolved(t *testing.T) {
|
||||
db := graphTestDB(t, filepath.Join(t.TempDir(), "recovery.db"))
|
||||
c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
|
||||
saveGraphCheckpoint(t, db, 1, "kb", c, c.ContentHash)
|
||||
if err := db.Exec(`UPDATE native_graph_records SET payload=json_set(payload,'$.relation_count',0)`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taskID := graphRecoveryTaskID(1, "kb", c, "revision")
|
||||
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "empty_result"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
complete, err := graphCandidateCompleted(db, 1, "kb", c)
|
||||
if err != nil || complete {
|
||||
t.Fatalf("empty checkpoint completed: %t %v", complete, err)
|
||||
}
|
||||
failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", true)
|
||||
if err != nil || len(failures) != 1 {
|
||||
t.Fatalf("empty checkpoint resolved failure: %+v %v", failures, err)
|
||||
}
|
||||
candidates, err := graphRecoveryCandidates(db, 1, "kb", "new-revision", 0, 1, true, false)
|
||||
if err != nil || len(candidates) != 1 || candidates[0].ID != c.ID {
|
||||
t.Fatalf("empty checkpoint skipped rebuild: %+v %v", candidates, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphDispatchDoesNotRepeatAfterRedisRecordDisappears(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "recovery.db")
|
||||
db := graphTestDB(t, path)
|
||||
c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
|
||||
taskID := graphRecoveryTaskID(1, "kb", c, "revision")
|
||||
reserved, err := reserveGraphDispatch(db, 1, "kb", "revision", taskID, c)
|
||||
if err != nil || !reserved {
|
||||
t.Fatalf("reserve: %t %v", reserved, err)
|
||||
}
|
||||
sqlDB, _ := db.DB()
|
||||
if err := sqlDB.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db = graphTestDB(t, path)
|
||||
reserved, err = reserveGraphDispatch(db, 1, "kb", "revision", taskID, c)
|
||||
if err != nil || reserved {
|
||||
t.Fatalf("replayed unknown dispatch: %t %v", reserved, err)
|
||||
}
|
||||
failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", false)
|
||||
if err != nil || len(failures) != 1 || failures[0].Reason != "task_missing_after_dispatch" {
|
||||
t.Fatalf("lost missing task: %+v %v", failures, err)
|
||||
}
|
||||
}
|
||||
|
||||
type graphTestQueue struct {
|
||||
runs int
|
||||
err error
|
||||
}
|
||||
|
||||
func (q *graphTestQueue) RunTask(_, _ string) error { q.runs++; return q.err }
|
||||
|
||||
func TestGraphManualRetryIsBoundedAndAuditedAcrossRestart(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "recovery.db")
|
||||
db := graphTestDB(t, path)
|
||||
c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
|
||||
taskID := graphRecoveryTaskID(1, "kb", c, "revision")
|
||||
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queue := &graphTestQueue{}
|
||||
if retried, err := graphFailureRetry(db, queue, taskID, "too short", 1); err == nil || retried || queue.runs != 0 {
|
||||
t.Fatal("unaudited retry was accepted")
|
||||
}
|
||||
if retried, err := graphFailureRetry(db, queue, taskID, "Provider configuration investigated and fixed", 0); err != nil || retried || queue.runs != 0 {
|
||||
t.Fatal("queue cap was ignored")
|
||||
}
|
||||
retried, err := graphFailureRetry(db, queue, taskID, "Provider configuration investigated and fixed", 1)
|
||||
if err != nil || !retried || queue.runs != 1 {
|
||||
t.Fatalf("first retry: %t %v %+v", retried, err, queue)
|
||||
}
|
||||
sqlDB, _ := db.DB()
|
||||
if err := sqlDB.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db = graphTestDB(t, path)
|
||||
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retried, err = graphFailureRetry(db, queue, taskID, "Retry repeatedly after restart", 1)
|
||||
if err != nil || retried || queue.runs != 1 {
|
||||
t.Fatalf("retry was repeated: %t %v %+v", retried, err, queue)
|
||||
}
|
||||
failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", false)
|
||||
if err != nil || len(failures) != 1 || failures[0].ManualRetries != 1 || failures[0].RetryReason != "Provider configuration investigated and fixed" {
|
||||
t.Fatalf("lost retry audit: %+v %v", failures, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphRetryDispatchErrorKeepsReservation(t *testing.T) {
|
||||
db := graphTestDB(t, filepath.Join(t.TempDir(), "recovery.db"))
|
||||
c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
|
||||
taskID := graphRecoveryTaskID(1, "kb", c, "revision")
|
||||
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queue := &graphTestQueue{err: errors.New("lost response")}
|
||||
if retried, err := graphFailureRetry(db, queue, taskID, "Investigated timeout and fixed provider", 1); err == nil || retried {
|
||||
t.Fatal("dispatch error not propagated")
|
||||
}
|
||||
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if retried, err := graphFailureRetry(db, queue, taskID, "Investigated timeout and fixed provider", 1); err != nil || retried || queue.runs != 1 {
|
||||
t.Fatal("ambiguous dispatch was replayed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphFailureReconciliationRequiresCurrentNativeCheckpoint(t *testing.T) {
|
||||
db := graphTestDB(t, filepath.Join(t.TempDir(), "recovery.db"))
|
||||
c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
|
||||
taskID := graphRecoveryTaskID(1, "kb", c, "revision")
|
||||
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "completed_without_native_checkpoint"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saveGraphCheckpoint(t, db, 1, "kb", c, "stale-content-hash")
|
||||
complete, err := graphCandidateCompleted(db, 1, "kb", c)
|
||||
if err != nil || complete {
|
||||
t.Fatalf("stale checkpoint was accepted: %t %v", complete, err)
|
||||
}
|
||||
failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", true)
|
||||
if err != nil || len(failures) != 1 {
|
||||
t.Fatalf("prematurely resolved: %+v %v", failures, err)
|
||||
}
|
||||
saveGraphCheckpoint(t, db, 1, "kb", c, c.ContentHash)
|
||||
complete, err = graphCandidateCompleted(db, 1, "kb", c)
|
||||
if err != nil || !complete {
|
||||
t.Fatalf("racing checkpoint not detected: %t %v", complete, err)
|
||||
}
|
||||
failures, err = graphUnresolvedFailures(db, 1, "kb", "revision", true)
|
||||
if err != nil || len(failures) != 0 {
|
||||
t.Fatalf("completed source remained blocked: %+v %v", failures, err)
|
||||
}
|
||||
var historical graphRecoveryFailure
|
||||
if err := db.First(&historical, "task_id=?", taskID).Error; err != nil || historical.State != "resolved" || historical.Reason != "completed_without_native_checkpoint" {
|
||||
t.Fatalf("failure history lost: %+v %v", historical, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphChangedSourceSupersedesOldFailure(t *testing.T) {
|
||||
db := graphTestDB(t, filepath.Join(t.TempDir(), "recovery.db"))
|
||||
c := seedGraphChunk(t, db, 1, "kb", "first", "first content", 1)
|
||||
taskID := graphRecoveryTaskID(1, "kb", c, "revision")
|
||||
if err := recordGraphFailure(db, 1, "kb", "revision", taskID, c, "archived"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Exec("UPDATE chunks SET content='new content'").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failures, err := graphUnresolvedFailures(db, 1, "kb", "revision", true)
|
||||
if err != nil || len(failures) != 0 {
|
||||
t.Fatalf("obsolete failure blocked new content: %+v %v", failures, err)
|
||||
}
|
||||
candidates, err := graphRecoveryCandidates(db, 1, "kb", "revision", 0, 1, true, false)
|
||||
if err != nil || len(candidates) != 1 || candidates[0].ContentHash == c.ContentHash {
|
||||
t.Fatalf("new content unavailable: %+v %v", candidates, err)
|
||||
}
|
||||
var historical graphRecoveryFailure
|
||||
if err := db.First(&historical, "task_id=?", taskID).Error; err != nil || historical.State != "superseded" {
|
||||
t.Fatalf("history not preserved: %+v %v", historical, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphQueueWindowCountsEveryOutstandingState(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
info *asynq.QueueInfo
|
||||
limit, want int
|
||||
}{
|
||||
{"empty", &asynq.QueueInfo{}, 2, 2},
|
||||
{"pending", &asynq.QueueInfo{Pending: 1}, 2, 1},
|
||||
{"active", &asynq.QueueInfo{Active: 2}, 2, 0},
|
||||
{"scheduled", &asynq.QueueInfo{Scheduled: 2}, 2, 0},
|
||||
{"retry", &asynq.QueueInfo{Retry: 2}, 2, 0},
|
||||
{"aggregating", &asynq.QueueInfo{Aggregating: 2}, 2, 0},
|
||||
{"combined", &asynq.QueueInfo{Active: 1, Pending: 1, Retry: 1}, 4, 1},
|
||||
{"exceeded", &asynq.QueueInfo{Active: 4}, 2, 0},
|
||||
{"terminal records do not block later work", &asynq.QueueInfo{Archived: 100, Completed: 100}, 2, 2},
|
||||
{"paused", &asynq.QueueInfo{Paused: true}, 2, 0},
|
||||
{"unknown", nil, 2, 0},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := graphQueueSlots(test.info, test.limit); got != test.want {
|
||||
t.Fatalf("slots=%d want=%d", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphTaskIdentityScopesContentTenantAndRevision(t *testing.T) {
|
||||
c := graphRecoveryCandidate{ID: "chunk", ContentHash: "sha256"}
|
||||
id := graphRecoveryTaskID(1, "kb", c, "r1")
|
||||
if id != graphRecoveryTaskID(1, "kb", c, "r1") {
|
||||
t.Fatal("identity changed across restart")
|
||||
}
|
||||
for _, other := range []string{
|
||||
graphRecoveryTaskID(2, "kb", c, "r1"), graphRecoveryTaskID(1, "other", c, "r1"), graphRecoveryTaskID(1, "kb", c, "r2"),
|
||||
graphRecoveryTaskID(1, "kb", graphRecoveryCandidate{ID: "chunk", ContentHash: "changed"}, "r1"),
|
||||
} {
|
||||
if id == other {
|
||||
t.Fatal("unrelated attempts share task identity")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphResumeCursorDoesNotSkipUndispatchedOrUnverifiedWork(t *testing.T) {
|
||||
candidates := []graphRecoveryCandidate{{ID: "first", SeqID: 10}, {ID: "second", SeqID: 20}, {ID: "third", SeqID: 30}}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
settled map[string]bool
|
||||
want int64
|
||||
}{
|
||||
{"queued but not yet verified", map[string]bool{}, 5},
|
||||
{"second deferred by queue limit", map[string]bool{"first": true, "third": true}, 10},
|
||||
{"all durably handled", map[string]bool{"first": true, "second": true, "third": true}, 30},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := graphResumeCursor(5, candidates, test.settled); got != test.want {
|
||||
t.Fatalf("cursor=%d want=%d", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict'
|
||||
|
||||
const assert = require('node:assert/strict')
|
||||
const http = require('node:http')
|
||||
const crypto = require('node:crypto')
|
||||
const { readFileSync, writeFileSync, statSync } = require('node:fs')
|
||||
const path = require('node:path')
|
||||
|
||||
// Requires two disposable users and an already-enabled disposable workspace.
|
||||
// It never changes workspace switches, registers users, or resets a database.
|
||||
const base = (process.env.MEMORY_SMOKE_BASE || 'http://127.0.0.1:18080/api/v1').replace(/\/$/, '')
|
||||
const run = process.env.MEMORY_SMOKE_RUN || `memory-smoke-${crypto.randomBytes(8).toString('hex')}`
|
||||
assert.match(run, /^memory-smoke-[a-f0-9]{16}$/)
|
||||
const manifestPath = path.resolve(process.env.MEMORY_SMOKE_MANIFEST || `${run}.json`)
|
||||
const manifest = { run, base, started_at: new Date().toISOString(), resources: [], jobs: [], memories: [], candidates: [], checks: [], cleanup_errors: [] }
|
||||
const captured = []
|
||||
let owner, outsider, authorID, server, task, flow, model, author
|
||||
const persist = () => writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 })
|
||||
const pause = ms => new Promise(resolve => setTimeout(resolve, ms))
|
||||
const sqlString = value => `'${String(value).replaceAll("'", "''")}'`
|
||||
|
||||
function credentials(file) {
|
||||
assert.ok(file, 'Set MEMORY_SMOKE_OWNER_AUTH and MEMORY_SMOKE_OUTSIDER_AUTH to local JSON auth files')
|
||||
assert.equal(statSync(file).mode & 0o077, 0, 'Auth file must not be readable by other users')
|
||||
const value = JSON.parse(readFileSync(file, 'utf8'))
|
||||
assert.equal(typeof value.token, 'string', 'Auth file requires token')
|
||||
return value
|
||||
}
|
||||
|
||||
async function request(route, body, method = body === undefined ? 'GET' : 'POST', statuses = [200], identity = owner) {
|
||||
const headers = { 'Content-Type': 'application/json', Authorization: `Bearer ${identity.token}` }
|
||||
if (identity.tenantID) headers['X-Tenant-ID'] = String(identity.tenantID)
|
||||
const response = await fetch(base + route, { method, headers, body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(15000) })
|
||||
const text = await response.text()
|
||||
if (response.status === 204 && statuses.includes(204)) return {}
|
||||
let value
|
||||
try { value = JSON.parse(text) } catch { throw new Error(`${method} ${route}: non-JSON HTTP ${response.status}`) }
|
||||
assert.ok(statuses.includes(response.status), `${method} ${route}: HTTP ${response.status}; ${JSON.stringify(value.error || value.message || '').slice(0, 300)}`)
|
||||
return value
|
||||
}
|
||||
|
||||
async function listCandidates() {
|
||||
const response = await request('/workbench/operations/records/memories?page_size=100')
|
||||
const rows = (response.data || []).filter(row => task && row.source_ref.startsWith(`内容任务/${task.id}/`))
|
||||
for (const row of rows) if (!manifest.candidates.includes(row.id)) manifest.candidates.push(row.id)
|
||||
persist()
|
||||
return rows
|
||||
}
|
||||
|
||||
async function listMemories() {
|
||||
const response = await request('/memory/items?limit=200')
|
||||
for (const row of response.data || []) {
|
||||
if (row.content.includes(run) && !manifest.memories.includes(row.id)) manifest.memories.push(row.id)
|
||||
}
|
||||
persist()
|
||||
return response.data || []
|
||||
}
|
||||
|
||||
async function reconfirmLegacyFixture() {
|
||||
if (process.env.MEMORY_SMOKE_LEGACY !== '1') return null
|
||||
assert.equal(typeof process.send, 'function', 'Legacy fixture requires the isolated workspace wrapper')
|
||||
const seeded = await new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => { process.off('message', onMessage); reject(new Error('Legacy fixture seed timed out')) }, 15000)
|
||||
function onMessage(message) {
|
||||
if (message?.type !== 'legacy-memory-seeded') return
|
||||
clearTimeout(timer)
|
||||
process.off('message', onMessage)
|
||||
if (message.error) reject(new Error(message.error))
|
||||
else resolve(message)
|
||||
}
|
||||
process.on('message', onMessage)
|
||||
process.send({ type: 'seed-legacy-memory', run, agent_id: authorID })
|
||||
})
|
||||
assert.match(seeded.id, /^[a-f0-9-]{36}$/)
|
||||
manifest.candidates.push(seeded.id)
|
||||
manifest.legacy_history_id = seeded.history_id
|
||||
persist()
|
||||
const find = async () => (await request('/workbench/operations/records/memories?page_size=100')).data.find(row => row.id === seeded.id)
|
||||
const before = await find()
|
||||
assert.equal(before.status, 'accepted')
|
||||
assert.equal(before.legacy_history_id, seeded.history_id)
|
||||
assert.equal(before.can_reconfirm, true)
|
||||
assert.ok(!before.native_memory_id)
|
||||
const content = `${run}:解释方案时先给判断标准,再给两条示例。`
|
||||
const body = { status: 'accepted', content, reconfirm_legacy: true, confirm_personal_scope: true }
|
||||
const route = `/workbench/operations/memories/${seeded.id}`
|
||||
await request(route, body, 'POST', [403, 404], outsider)
|
||||
await request(route, { ...body, confirm_personal_scope: false }, 'POST', [400])
|
||||
await request(route, body)
|
||||
await request(route, body)
|
||||
const after = await find()
|
||||
assert.equal(after.content, before.content, 'Legacy original text was overwritten')
|
||||
assert.equal(after.source_ref, before.source_ref, 'Legacy provenance was overwritten')
|
||||
assert.equal(after.reconfirmed_content, content)
|
||||
assert.equal(after.can_reconfirm, false)
|
||||
assert.ok(after.reconfirmed_at && after.native_memory_id)
|
||||
const linked = (await listMemories()).filter(row => row.source_message_id === seeded.id)
|
||||
assert.equal(linked.length, 1, 'Legacy reconfirmation was not idempotent')
|
||||
assert.equal(linked[0].id, after.native_memory_id)
|
||||
assert.equal(linked[0].content, content)
|
||||
assert.equal(linked[0].subject_id, manifest.subject_id)
|
||||
assert.equal(linked[0].status, 'active')
|
||||
await request(`/memory/items/${linked[0].id}`, undefined, 'DELETE', [404], outsider)
|
||||
assert.ok(!(await request('/memory/items?limit=200', undefined, 'GET', [200], outsider)).data.some(row => row.id === linked[0].id))
|
||||
manifest.checks.push('Legacy owner reconfirmation requires consent, preserves provenance, stores one native preference and rejects foreign access')
|
||||
persist()
|
||||
return { id: linked[0].id, candidate_id: seeded.id, route, body, content, find }
|
||||
}
|
||||
|
||||
async function finishLegacyFixture(legacy) {
|
||||
if (!legacy) return
|
||||
assert.ok(captured.at(-1).system.includes(legacy.content), 'Reconfirmed legacy preference was missing from actual generation prompt')
|
||||
await request(`/memory/items/${legacy.id}`, undefined, 'DELETE', [200, 204])
|
||||
await request(legacy.route, legacy.body)
|
||||
assert.equal((await legacy.find()).native_memory_id, legacy.id, 'Legacy link was removed after native deletion')
|
||||
assert.ok(!(await listMemories()).some(row => row.id === legacy.id || row.source_message_id === legacy.candidate_id), 'Repeated confirmation resurrected deleted native memory')
|
||||
manifest.checks.push('Reconfirmed legacy preference reaches actual generation prompt; deliberate native deletion remains deleted after repeat confirmation')
|
||||
persist()
|
||||
}
|
||||
|
||||
async function act(action, extra = {}, expectedStatus = 'succeeded') {
|
||||
const job = (await request(`/workbench/resources/content-tasks/${task.id}/actions/${action}`, { version: task.version, ...extra }, 'POST', [202])).data
|
||||
manifest.jobs.push(job.id)
|
||||
persist()
|
||||
const deadline = Date.now() + 90000
|
||||
while (Date.now() < deadline) {
|
||||
const current = (await request(`/workbench/jobs/${job.id}`)).data
|
||||
if (!['queued', 'running'].includes(current.status)) {
|
||||
assert.equal(current.status, expectedStatus, `${action}: ${current.error || current.status}`)
|
||||
task = (await request(`/workbench/resources/content-tasks/${task.id}`)).data
|
||||
return current
|
||||
}
|
||||
await pause(350)
|
||||
}
|
||||
throw new Error(`Job timeout: ${job.id}; cleanup must wait until the job stops`)
|
||||
}
|
||||
|
||||
function startModel() {
|
||||
server = http.createServer(async (req, res) => {
|
||||
try {
|
||||
let text = ''
|
||||
for await (const chunk of req) {
|
||||
text += chunk
|
||||
assert.ok(text.length < 2 * 1024 * 1024, 'Model request exceeds fixture limit')
|
||||
}
|
||||
const body = JSON.parse(text)
|
||||
const messages = body.messages || []
|
||||
const system = messages.filter(message => message.role === 'system').map(message => message.content).join('\n')
|
||||
const prompt = String(messages.at(-1)?.content || '')
|
||||
captured.push({ system, prompt })
|
||||
let content = `测试正文 ${run}:先说明判断依据,再讨论应用步骤。`
|
||||
if (system.includes('内容编排师')) content = JSON.stringify({ segments: [1, 2, 3].map(n => ({ id: `s${n}`, title: `验证步骤${n}`, node_id: 'writer', agent_id: authorID })) })
|
||||
if (system.includes('内置跨稿质检员')) content = JSON.stringify({ passed: true, summary: '本地固定质检通过', issues: [] })
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' })
|
||||
res.end(JSON.stringify({ id: run, object: 'chat.completion', created: Math.floor(Date.now() / 1000), model: run,
|
||||
choices: [{ index: 0, message: { role: 'assistant', content }, finish_reason: 'stop' }], usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 } }))
|
||||
} catch {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' })
|
||||
res.end(JSON.stringify({ error: { message: 'Invalid smoke-model request' } }))
|
||||
}
|
||||
})
|
||||
return new Promise((resolve, reject) => {
|
||||
server.once('error', reject)
|
||||
server.listen(0, '0.0.0.0', () => resolve(server.address().port))
|
||||
})
|
||||
}
|
||||
|
||||
async function preflight() {
|
||||
const url = new URL(base)
|
||||
assert.ok(['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname), 'Use the loopback origin; public URLs are refused')
|
||||
owner = credentials(process.env.MEMORY_SMOKE_OWNER_AUTH)
|
||||
outsider = credentials(process.env.MEMORY_SMOKE_OUTSIDER_AUTH)
|
||||
const me = (await request('/auth/me')).data
|
||||
const other = (await request('/auth/me', undefined, 'GET', [200], outsider)).data
|
||||
assert.ok(me.user.username.startsWith('memory-smoke-'), 'Owner must be a dedicated memory-smoke- user')
|
||||
assert.ok(me.tenant.name.startsWith('memory-smoke-'), 'Workspace must be a dedicated memory-smoke- workspace')
|
||||
assert.ok(other.user.username.startsWith('memory-smoke-'), 'Outsider must be a dedicated memory-smoke- user')
|
||||
assert.notEqual(me.user.id, other.user.id)
|
||||
manifest.tenant_id = me.tenant.id
|
||||
manifest.owner_id = me.user.id
|
||||
manifest.subject_id = `web_user:${me.user.id}`
|
||||
const settings = (await request('/memory/settings')).data
|
||||
assert.equal(settings.effective, true, 'Enable memory on the disposable workspace/user before running')
|
||||
assert.equal((await request('/memory/items?limit=1')).total, 0, 'Owner already has memory; refuse to alter it')
|
||||
assert.equal((await request('/workbench/operations/records/memories?page_size=1')).total, 0, 'Owner already has review history; use another fixture user')
|
||||
for (const kind of ['flows', 'content-tasks', 'training-maps']) {
|
||||
const rows = (await request(`/workbench/resources/${kind}`)).data || []
|
||||
assert.equal(rows.length, 0, `Fixture workspace already has ${kind}`)
|
||||
}
|
||||
persist()
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await preflight()
|
||||
const port = await startModel()
|
||||
const modelHost = process.env.MEMORY_SMOKE_MODEL_HOST || 'host.docker.internal'
|
||||
assert.ok(['host.docker.internal', '127.0.0.1', 'localhost'].includes(modelHost), 'Fake-model host must resolve to this machine')
|
||||
model = (await request('/models', { name: run, type: 'KnowledgeQA', source: 'remote', parameters: { base_url: `http://${modelHost}:${port}/v1`, api_key: 'local-smoke-only', provider: 'generic' } }, 'POST', [201])).data
|
||||
manifest.model_id = model.id
|
||||
persist()
|
||||
author = (await request('/agents', { name: run, config: { model_id: model.id, system_prompt: '测试内容作者', role: 'production', agent_mode: 'quick-answer', memory_enabled: true } }, 'POST', [201])).data
|
||||
authorID = author.id
|
||||
manifest.agent_id = authorID
|
||||
persist()
|
||||
flow = (await request('/workbench/resources/flows', { name: run, data: { nodes: [{ id: 'writer', agent_id: authorID, label: '测试作者' }], edges: [] } }, 'POST', [201])).data
|
||||
manifest.resources.push({ id: flow.id, kind: 'flows' })
|
||||
persist()
|
||||
task = (await request('/workbench/resources/content-tasks', { name: run, data: { day: 1, flow_id: flow.id, agent_id: authorID, source_text: '判断依据、实际应用、总结回顾', knowledge_base_ids: [], duration_minutes: 135 } }, 'POST', [201])).data
|
||||
manifest.resources.push({ id: task.id, kind: 'content-tasks' })
|
||||
persist()
|
||||
assert.equal(task.data.duration_minutes, 135, 'Create lost duration')
|
||||
task = (await request(`/workbench/resources/content-tasks/${task.id}`, { name: task.name, version: task.version, data: { ...task.data, duration_minutes: 90 } }, 'PUT')).data
|
||||
assert.equal(task.data.duration_minutes, 90, 'Update lost duration')
|
||||
for (const invalid of [-1, 0.5, 1441]) {
|
||||
await request(`/workbench/resources/content-tasks/${task.id}`, { name: task.name, version: task.version, data: { ...task.data, duration_minutes: invalid } }, 'PUT', [400])
|
||||
assert.equal((await request(`/workbench/resources/content-tasks/${task.id}`)).data.data.duration_minutes, 90, 'Invalid duration changed persisted state')
|
||||
}
|
||||
manifest.checks.push('Duration persists 135-minute create and 90-minute update; negative/fractional/over-1440 updates are rejected')
|
||||
await act('decompose')
|
||||
assert.equal(task.data.segments.length, 3)
|
||||
const advice = `${run}:回答先列要点,再给例子。`
|
||||
await act('generate', { segment_id: 's1', advice })
|
||||
assert.equal((await listCandidates()).length, 0, 'Unaccepted draft created feedback memory')
|
||||
assert.equal((await listMemories()).length, 0, 'AI draft became a user memory')
|
||||
await act('accept', { segment_id: 's1' })
|
||||
let rows = await listCandidates()
|
||||
assert.equal(rows.length, 1)
|
||||
const candidate = rows[0]
|
||||
assert.equal(candidate.content, advice)
|
||||
assert.equal(candidate.status, 'pending')
|
||||
await act('accept', { segment_id: 's1' })
|
||||
assert.equal((await listCandidates()).length, 1, 'Repeated adoption duplicated feedback')
|
||||
await request(`/workbench/operations/memories/${candidate.id}`, { status: 'accepted' }, 'POST', [403, 404], outsider)
|
||||
await request(`/workbench/operations/memories/${candidate.id}`, { status: 'accepted' })
|
||||
const memories = await listMemories()
|
||||
assert.equal(memories.length, 1)
|
||||
assert.equal(memories[0].content, advice)
|
||||
assert.equal(memories[0].status, 'active')
|
||||
manifest.subject_id = memories[0].subject_id
|
||||
await request(`/memory/items/${memories[0].id}`, undefined, 'DELETE', [404], outsider)
|
||||
const otherRows = (await request('/memory/items?limit=200', undefined, 'GET', [200], outsider)).data || []
|
||||
assert.ok(!otherRows.some(row => row.id === memories[0].id))
|
||||
manifest.checks.push('Draft is excluded; accepted human advice is durable, idempotent and owner isolated; review writes active memory')
|
||||
const legacy = await reconfirmLegacyFixture()
|
||||
await act('generate', { segment_id: 's2', advice: `${run}:第二段使用简短段落。` })
|
||||
assert.ok(captured.at(-1).system.includes(advice), 'Approved memory did not reach the actual fake-model prompt')
|
||||
await finishLegacyFixture(legacy)
|
||||
await act('accept', { segment_id: 's2' })
|
||||
const skipped = (await listCandidates()).find(row => row.source_ref.endsWith('/内容段/s2'))
|
||||
assert.equal(skipped.status, 'pending')
|
||||
await act('skip', { segment_id: 's2' })
|
||||
assert.equal((await listCandidates()).find(row => row.id === skipped.id).status, 'invalidated')
|
||||
await request(`/workbench/operations/memories/${skipped.id}`, { status: 'accepted' }, 'POST', [409])
|
||||
await act('generate', { segment_id: 's3', advice: `${run}:总结保留清晰步骤。` })
|
||||
await act('accept', { segment_id: 's3' })
|
||||
const deleted = (await listCandidates()).find(row => row.source_ref.endsWith('/内容段/s3'))
|
||||
const originalProse = task.data.segments.map(row => row.content)
|
||||
await act('review')
|
||||
assert.equal(task.data.quality.duration_minutes, 90)
|
||||
await act('compose')
|
||||
assert.equal(task.data.composed, task.data.segments.filter(row => row.status === 'accepted').map(row => row.content).join('\n\n'))
|
||||
const callsBeforeDurationChange = captured.length
|
||||
task = (await request(`/workbench/resources/content-tasks/${task.id}`, { name: task.name, version: task.version, data: { ...task.data, duration_minutes: 225 } }, 'PUT')).data
|
||||
task = (await request(`/workbench/resources/content-tasks/${task.id}`)).data
|
||||
assert.equal(task.data.duration_minutes, 225)
|
||||
assert.deepEqual(task.data.segments.map(row => row.content), originalProse, 'Duration change lost saved prose')
|
||||
assert.deepEqual(task.data.segments.map(row => row.status), ['needs_review', 'skipped', 'needs_review'])
|
||||
assert.ok(!task.data.quality && !task.data.composed && !task.data.review, 'Duration change reused old quality approval')
|
||||
const blockedCompose = await act('compose', {}, 'failed')
|
||||
assert.ok(blockedCompose.error.includes('重新确认'), 'Composition failed for an unexpected reason')
|
||||
assert.equal(captured.length, callsBeforeDurationChange, 'Duration update or rejected composition invoked a model')
|
||||
assert.equal((await listCandidates()).find(row => row.id === deleted.id).status, 'invalidated')
|
||||
await act('accept', { segment_id: 's1' })
|
||||
await act('accept', { segment_id: 's3' })
|
||||
assert.equal((await listCandidates()).find(row => row.id === deleted.id).status, 'pending', 'Reacceptance did not restore pending source')
|
||||
assert.equal((await listCandidates()).find(row => row.id === candidate.id).status, 'accepted', 'Reacceptance reset an approved memory')
|
||||
manifest.checks.push('Changing a composed task duration preserves prose, invalidates quality and pending sources, blocks stale composition, and reacceptance restores only pending feedback')
|
||||
await request(`/workbench/resources/content-tasks/${task.id}`, undefined, 'DELETE')
|
||||
assert.equal((await listCandidates()).find(row => row.id === deleted.id).status, 'invalidated')
|
||||
assert.equal((await listMemories()).length, 1, 'Source deletion altered separately approved memory')
|
||||
manifest.checks.push('Approved preference is recalled; skip/deletion invalidate pending feedback without undoing explicit approval')
|
||||
manifest.model_calls = captured.length
|
||||
manifest.passed = true
|
||||
persist()
|
||||
}
|
||||
|
||||
async function cleanup() {
|
||||
if (!owner || !manifest.owner_id) return
|
||||
// Never delete while a worker can still write to the fixture resources.
|
||||
for (const id of manifest.jobs) {
|
||||
try {
|
||||
const row = (await request(`/workbench/jobs/${id}`)).data
|
||||
if (['queued', 'running'].includes(row.status)) {
|
||||
manifest.cleanup_errors.push(`Job ${id} is still ${row.status}; fixture retained for bounded cleanup`)
|
||||
persist()
|
||||
return
|
||||
}
|
||||
} catch (error) { manifest.cleanup_errors.push(error.message); persist(); return }
|
||||
}
|
||||
try { await listMemories(); await listCandidates() } catch (error) { manifest.cleanup_errors.push(error.message) }
|
||||
const remove = async route => {
|
||||
try { await request(route, undefined, 'DELETE', [200, 204, 404]) } catch (error) { manifest.cleanup_errors.push(error.message) }
|
||||
}
|
||||
for (const id of manifest.memories) await remove(`/memory/items/${id}`)
|
||||
for (const resource of [...manifest.resources].reverse()) await remove(`/workbench/resources/${resource.kind}/${resource.id}`)
|
||||
if (author) await remove(`/agents/${author.id}`)
|
||||
if (model) await remove(`/models/${model.id}`)
|
||||
|
||||
const tenant = Number(manifest.tenant_id)
|
||||
assert.ok(Number.isSafeInteger(tenant) && tenant > 0)
|
||||
const ownerID = sqlString(manifest.owner_id)
|
||||
const resourceIDs = manifest.resources.map(row => sqlString(row.id)).join(',') || "''"
|
||||
const candidateIDs = manifest.candidates.map(sqlString).join(',') || "''"
|
||||
const subject = sqlString(manifest.subject_id)
|
||||
// API deletion intentionally retains audit/history and rejection tombstones.
|
||||
// This optional exact-ID SQL removes only this run's workbench history.
|
||||
const cleanupSQL = `BEGIN;\nSET LOCAL lock_timeout = '3s';\nDO $$ BEGIN\n IF EXISTS (SELECT 1 FROM workbench_jobs WHERE tenant_id=${tenant} AND owner_id=${ownerID} AND resource_id IN (${resourceIDs}) AND status IN ('queued','running')) THEN RAISE EXCEPTION 'Fixture jobs still active'; END IF;\nEND $$;\nDELETE FROM workbench_pending_memories WHERE tenant_id=${tenant} AND owner_id=${ownerID} AND id IN (${candidateIDs});\nDELETE FROM workbench_revisions WHERE tenant_id=${tenant} AND resource_id IN (${resourceIDs});\nDELETE FROM workbench_events WHERE tenant_id=${tenant} AND owner_id=${ownerID} AND resource_id IN (${resourceIDs});\nDELETE FROM workbench_jobs WHERE tenant_id=${tenant} AND owner_id=${ownerID} AND resource_id IN (${resourceIDs});\nDELETE FROM memory_tombstones WHERE tenant_id=${tenant} AND subject_id=${subject} AND source_message_id IN (${candidateIDs});\nCOMMIT;\n`
|
||||
writeFileSync(`${manifestPath}.cleanup.sql`, cleanupSQL, { mode: 0o600 })
|
||||
manifest.finished_at = new Date().toISOString()
|
||||
manifest.cleanup_note = 'Created resources and native memories are API-deleted. Exact-ID cleanup SQL is supplied for review history/tombstones; audit/usage logs and fixture identities remain for verification. No original user data was deleted.'
|
||||
persist()
|
||||
}
|
||||
|
||||
if (process.argv.includes('--run')) {
|
||||
main().catch(error => { manifest.error = error.message; manifest.passed = false; process.exitCode = 1 }).finally(async () => {
|
||||
try { await cleanup() } catch (error) { manifest.cleanup_errors.push(error.message); process.exitCode = 1; persist() }
|
||||
server?.close()
|
||||
if (process.connected) process.disconnect()
|
||||
if (manifest.cleanup_errors.length) process.exitCode = 1
|
||||
console.log(JSON.stringify({ passed: manifest.passed || false, checks: manifest.checks, error: manifest.error, cleanup_errors: manifest.cleanup_errors, manifest: manifestPath }, null, 2))
|
||||
})
|
||||
} else {
|
||||
console.log('Set MEMORY_SMOKE_OWNER_AUTH and MEMORY_SMOKE_OUTSIDER_AUTH (token JSON files), optionally MEMORY_SMOKE_BASE/MEMORY_SMOKE_MODEL_HOST/MEMORY_SMOKE_MANIFEST, then run with --run. Requires a disposable memory-smoke- workspace and two memory-smoke- users; never registers or resets anything. See scripts/native_memory_api_smoke.md.')
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
# Native memory API smoke fixture
|
||||
|
||||
This script verifies the actual deployed HTTP API, workbench job leases/worker,
|
||||
database persistence, memory review and the prompt sent to a deterministic local
|
||||
fake model. It does not benchmark AI quality or invoke any existing model.
|
||||
|
||||
## Automatic disposable workspace
|
||||
|
||||
Use the wrapper to provision and clean up the required fixture:
|
||||
|
||||
```sh
|
||||
node scripts/native_memory_fixture.cjs --run
|
||||
```
|
||||
|
||||
It reads the existing private admin login file at
|
||||
`/Volumes/Extreme SSD/yawan-storage/weknora/acceptance/super-admin-login.json`.
|
||||
Override that path with `MEMORY_SMOKE_ADMIN_AUTH`. The file must have no group or
|
||||
other read permissions. The wrapper creates two uniquely named disposable users
|
||||
and an empty workspace, enables explicit-only memory with vector recall disabled,
|
||||
and gives only the fixture owner a disposable balance. The owner has workspace
|
||||
owner privileges to create the local fake model; the outsider remains an ordinary
|
||||
contributor. Neither user is a system administrator.
|
||||
|
||||
Credentials are written only into mode-0600 files within a mode-0700 run directory.
|
||||
The manifest is credential-free and is saved at each setup stage. The wrapper runs
|
||||
the smoke script, then checks no fixture workbench job remains active and removes
|
||||
only the new workspace, fixture users, auth tokens, balance, model, Agent, memory
|
||||
and workbench rows. It validates both workspace name/business marker and exact
|
||||
user identities before cleanup. A surviving smoke process, unexpected tables or foreign memberships stop
|
||||
cleanup for inspection. Administrator audit entries in the original workspace
|
||||
remain as evidence. There is no database reset, and no real provider is called.
|
||||
|
||||
If interrupted, rerun just the exact fixture cleanup:
|
||||
|
||||
```sh
|
||||
node scripts/native_memory_fixture.cjs --cleanup /absolute/path/memory-smoke-RUN/fixture.json
|
||||
```
|
||||
|
||||
The wrapper defaults to Docker context `colima-yawan`, container
|
||||
`WeKnora-postgres`, database `yawan_native_acceptance_20260915` and source tenant
|
||||
`10059`. This database is production despite its historical name. Connection
|
||||
overrides are `MEMORY_SMOKE_DOCKER`, `MEMORY_SMOKE_DOCKER_CONTEXT`,
|
||||
`MEMORY_SMOKE_DB_CONTAINER`, `MEMORY_SMOKE_DB_NAME` and
|
||||
`MEMORY_SMOKE_SOURCE_TENANT`. Cleanup requires the same recorded connection.
|
||||
`MEMORY_SMOKE_DIRECTORY` changes the parent directory for fixture reports.
|
||||
The wrapper is intended to be run after deployment by the release operator.
|
||||
|
||||
The wrapper also seeds one synthetic accepted legacy history row through a
|
||||
private child-process channel, after checking the new Agent's exact owner,
|
||||
workspace, name and memory setting. This is a new fixture record, never a real
|
||||
user's imported history. After migration 217, the actual API must require owner
|
||||
scope consent, preserve the original text/source, create exactly one linked native
|
||||
preference, include it in the next real generation request to the local stub, and
|
||||
reject access from the other user. Deleting that native preference then repeating
|
||||
confirmation must not restore it. The dedicated workspace cleanup removes this
|
||||
synthetic history too. Provenance matching during migration is tested separately
|
||||
with an isolated PostgreSQL database. Standalone existing-fixture mode does not
|
||||
seed or claim coverage of legacy reconfirmation.
|
||||
|
||||
## Existing fixture mode
|
||||
|
||||
Requirements:
|
||||
|
||||
- Node 20+ and the local backend origin after deployment.
|
||||
- A dedicated empty workspace whose name starts with `memory-smoke-`, with memory enabled.
|
||||
- Two disposable users whose usernames start with `memory-smoke-`. The owner must be able to create a model, Agent, flow and content task. The outsider must not be a system administrator. The outsider may share the fixture workspace or use another empty fixture workspace.
|
||||
- Two local mode-0600 JSON auth files with `{"token":"..."}`. Optional `tenantID` selects the active workspace through `X-Tenant-ID`. Do not put tokens in commands, source control or the result report.
|
||||
|
||||
```sh
|
||||
MEMORY_SMOKE_OWNER_AUTH=/absolute/path/owner-auth.json \
|
||||
MEMORY_SMOKE_OUTSIDER_AUTH=/absolute/path/outsider-auth.json \
|
||||
MEMORY_SMOKE_MANIFEST=/absolute/path/memory-smoke-result.json \
|
||||
node scripts/native_memory_api_smoke.cjs --run
|
||||
```
|
||||
|
||||
The default origin is `http://127.0.0.1:18080/api/v1`; override with
|
||||
`MEMORY_SMOKE_BASE`. Public hostnames are refused. The model stub listens on a
|
||||
random local host port so the backend container can reach it through
|
||||
`host.docker.internal`. Set `MEMORY_SMOKE_MODEL_HOST=127.0.0.1` for a backend
|
||||
running directly on the host. The stub receives only fixture content.
|
||||
|
||||
The script checks 135-minute task creation, 90-minute update, persistence, and
|
||||
rejection of negative/fractional/over-1440 durations without mutating saved state.
|
||||
It also checks that draft content does not become memory, adopting human
|
||||
advice creates one pending candidate, repeated adoption deduplicates it,
|
||||
another account cannot approve or delete it, explicit review writes an active
|
||||
memory, the next generation actually recalls that preference, and skip/source
|
||||
deletion invalidate only pending candidates. It also runs quality review and
|
||||
exact ordered composition, changes the duration to 225 minutes, verifies the
|
||||
original prose is retained while old review/composition results are cleared,
|
||||
and checks that composition is blocked until renewed confirmation and review.
|
||||
Reacceptance restores invalidated pending feedback without resetting explicitly
|
||||
approved memories. It refuses nonempty owner memory
|
||||
or content fixtures and does not alter workspace settings.
|
||||
|
||||
Cleanup uses exact created resource/memory IDs through APIs. It waits for every
|
||||
recorded job to be terminal first. On job timeout it deliberately retains the
|
||||
fixture and reports the IDs instead of deleting beneath a running worker. The
|
||||
manifest is written after every created object, contains no token, and supports
|
||||
manual recovery. API deletion leaves workbench review/history records and
|
||||
memory tombstones; the companion `.cleanup.sql` removes only this run's exact
|
||||
candidate/resource IDs within its tenant/owner scope. It refuses active jobs.
|
||||
Run that SQL against the matching database after reviewing the manifest. Audit
|
||||
and usage logs and the disposable accounts/workspace remain as test evidence;
|
||||
remove the dedicated fixture identities through the normal administration
|
||||
workflow when the evidence is no longer needed. No broad database reset is used.
|
||||
|
||||
This fixture tests native content feedback and recall. Chat extraction retry,
|
||||
timestamp pagination, per-session checkpoints and pending conflict rollback
|
||||
are covered by the offline Go integration suites; this script does not claim
|
||||
to simulate those asynchronous chat failures.
|
||||
@@ -0,0 +1,278 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict'
|
||||
|
||||
const assert = require('node:assert/strict')
|
||||
const crypto = require('node:crypto')
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const { execFileSync, spawn } = require('node:child_process')
|
||||
|
||||
const acceptance = process.env.MEMORY_SMOKE_DIRECTORY || '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
|
||||
const database = {
|
||||
context: process.env.MEMORY_SMOKE_DOCKER_CONTEXT || 'colima-yawan',
|
||||
container: process.env.MEMORY_SMOKE_DB_CONTAINER || 'WeKnora-postgres',
|
||||
name: process.env.MEMORY_SMOKE_DB_NAME || 'yawan_native_acceptance_20260915',
|
||||
}
|
||||
const literal = value => `'${String(value).replaceAll("'", "''")}'`
|
||||
let manifest, manifestPath, admin
|
||||
|
||||
function privateJSON(file, value) {
|
||||
const temporary = `${file}.${crypto.randomBytes(4).toString('hex')}.tmp`
|
||||
const fd = fs.openSync(temporary, 'wx', 0o600)
|
||||
try {
|
||||
fs.writeFileSync(fd, `${JSON.stringify(value, null, 2)}\n`)
|
||||
fs.fsyncSync(fd)
|
||||
} finally { fs.closeSync(fd) }
|
||||
fs.renameSync(temporary, file)
|
||||
}
|
||||
const persist = () => privateJSON(manifestPath, manifest)
|
||||
function sql(query) {
|
||||
try {
|
||||
return execFileSync(process.env.MEMORY_SMOKE_DOCKER || '/opt/homebrew/bin/docker', [
|
||||
'--context', database.context, 'exec', '-i', database.container,
|
||||
'psql', '-X', '-q', '-U', 'postgres', '-d', database.name,
|
||||
'-At', '-v', 'ON_ERROR_STOP=1',
|
||||
], { input: query, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 30000 }).trim()
|
||||
} catch {
|
||||
// SQL stderr may contain request fields. Keep credentials out of reports.
|
||||
throw new Error('Fixture database operation failed; inspect the bounded fixture SQL and database locally')
|
||||
}
|
||||
}
|
||||
async function api(route, body, identity = admin, tenant = manifest.source_tenant_id) {
|
||||
const response = await fetch(manifest.base + route, {
|
||||
method: body === undefined ? 'GET' : 'POST',
|
||||
headers: { 'Content-Type': 'application/json', ...(identity ? { Authorization: `Bearer ${identity.token}`, 'X-Tenant-ID': String(tenant) } : {}) },
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(15000),
|
||||
})
|
||||
assert.ok(response.ok, `${route}: HTTP ${response.status}`)
|
||||
return response.json()
|
||||
}
|
||||
function validateManifest() {
|
||||
assert.equal(manifest.format, 'native-memory-fixture-v1')
|
||||
assert.match(manifest.run, /^memory-smoke-[a-f0-9]{16}$/)
|
||||
assert.deepEqual(manifest.database, database, 'Use the same database connection as the fixture manifest')
|
||||
assert.ok(Number.isSafeInteger(manifest.source_tenant_id) && manifest.source_tenant_id > 0)
|
||||
if (manifest.tenant_id !== undefined) {
|
||||
assert.ok(Number.isSafeInteger(manifest.tenant_id) && manifest.tenant_id > 0 && manifest.tenant_id !== manifest.source_tenant_id)
|
||||
}
|
||||
assert.equal(manifest.users.length, 2)
|
||||
for (const [index, label] of ['owner', 'outsider'].entries()) {
|
||||
assert.equal(manifest.users[index].username, `${manifest.run}-${label}`)
|
||||
assert.equal(manifest.users[index].email, `${manifest.run}-${label}@example.test`)
|
||||
}
|
||||
const url = new URL(manifest.base)
|
||||
assert.ok(['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname), 'Only loopback API origins are permitted')
|
||||
}
|
||||
|
||||
async function provision() {
|
||||
const adminFile = process.env.MEMORY_SMOKE_ADMIN_AUTH || path.join(acceptance, 'super-admin-login.json')
|
||||
assert.equal(fs.statSync(adminFile).mode & 0o077, 0, 'Admin credentials must be a private file (mode 0600)')
|
||||
const credentials = JSON.parse(fs.readFileSync(adminFile, 'utf8'))
|
||||
admin = await api('/auth/login', { email: credentials.email, password: credentials.password }, null)
|
||||
assert.equal(typeof admin.token, 'string')
|
||||
// The random name and business marker allow exact recovery after interruption.
|
||||
manifest.tenant_id = Number(sql(`INSERT INTO tenants(name,business,status,memory_config)
|
||||
VALUES (${literal(manifest.run)},'native-memory-smoke-fixture','active',
|
||||
'{"enabled":true,"write_mode":"explicit_only","vector_recall":false,"embedding_model_id":""}') RETURNING id;`))
|
||||
assert.ok(Number.isSafeInteger(manifest.tenant_id) && manifest.tenant_id > 0 && manifest.tenant_id !== manifest.source_tenant_id)
|
||||
persist()
|
||||
for (const user of manifest.users) {
|
||||
const password = crypto.randomBytes(18).toString('base64url')
|
||||
const row = (await api('/access/users', { username: user.username, email: user.email, password, role: 'contributor', status: 'active' })).data
|
||||
user.id = row.id
|
||||
persist()
|
||||
const role = user === manifest.users[0] ? 'owner' : 'contributor'
|
||||
sql(`BEGIN;
|
||||
UPDATE users SET tenant_id=${manifest.tenant_id} WHERE id=${literal(user.id)} AND email=${literal(user.email)} AND tenant_id=${manifest.source_tenant_id};
|
||||
UPDATE tenant_members SET tenant_id=${manifest.tenant_id},role=${literal(role)} WHERE user_id=${literal(user.id)} AND tenant_id=${manifest.source_tenant_id};
|
||||
DELETE FROM workbench_access_assignments WHERE user_id=${literal(user.id)} AND tenant_id=${manifest.source_tenant_id};
|
||||
COMMIT;`)
|
||||
const identity = await api('/auth/login', { email: user.email, password }, null)
|
||||
assert.equal(Number(identity.user.tenant_id), manifest.tenant_id, 'Fixture login selected a different workspace')
|
||||
assert.equal(identity.user.is_system_admin, false, 'Fixture user must not be a system administrator')
|
||||
privateJSON(path.join(path.dirname(manifestPath), `${user === manifest.users[0] ? 'owner' : 'outsider'}-auth.json`), { token: identity.token, tenantID: manifest.tenant_id })
|
||||
if (role === 'owner') {
|
||||
await api('/workbench/operations/grants', { user_id: user.id, amount: 1_000_000_000, reason: manifest.run, idempotency_key: manifest.run }, identity, manifest.tenant_id)
|
||||
}
|
||||
}
|
||||
manifest.provisioned_at = new Date().toISOString()
|
||||
persist()
|
||||
}
|
||||
|
||||
function runSmoke() {
|
||||
const directory = path.dirname(manifestPath)
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(process.execPath, [path.join(__dirname, 'native_memory_api_smoke.cjs'), '--run'], {
|
||||
env: { ...process.env, MEMORY_SMOKE_BASE: manifest.base,
|
||||
MEMORY_SMOKE_RUN: manifest.run, MEMORY_SMOKE_LEGACY: '1',
|
||||
MEMORY_SMOKE_OWNER_AUTH: path.join(directory, 'owner-auth.json'),
|
||||
MEMORY_SMOKE_OUTSIDER_AUTH: path.join(directory, 'outsider-auth.json'),
|
||||
MEMORY_SMOKE_MANIFEST: path.join(directory, 'smoke.json') },
|
||||
stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
|
||||
})
|
||||
manifest.smoke_pid = child.pid
|
||||
persist()
|
||||
const timer = setTimeout(() => child.kill('SIGTERM'), 300000)
|
||||
child.on('message', message => {
|
||||
if (message?.type !== 'seed-legacy-memory') return
|
||||
try {
|
||||
validateManifest()
|
||||
assert.equal(message.run, manifest.run)
|
||||
assert.match(message.agent_id, /^[a-f0-9-]{36}$/)
|
||||
assert.ok(!manifest.legacy_candidate, 'Only one synthetic legacy candidate is permitted')
|
||||
const ownerID = manifest.users[0].id
|
||||
const id = crypto.randomUUID(), historyID = crypto.randomUUID()
|
||||
const original = `${manifest.run}: original legacy advice`
|
||||
const source = `${manifest.run}/legacy/${historyID}`
|
||||
// Seed only a synthetic accepted history row in this run's empty workspace.
|
||||
// Migration provenance matching itself is verified by PostgreSQL unit tests.
|
||||
sql(`BEGIN;
|
||||
DO $seed$ BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM tenants WHERE id=${manifest.tenant_id} AND name=${literal(manifest.run)} AND business='native-memory-smoke-fixture') OR
|
||||
NOT EXISTS (SELECT 1 FROM custom_agents WHERE id=${literal(message.agent_id)} AND tenant_id=${manifest.tenant_id} AND created_by=${literal(ownerID)} AND name=${literal(manifest.run)} AND deleted_at IS NULL AND config->>'memory_enabled'='true')
|
||||
THEN RAISE EXCEPTION 'Legacy fixture scope mismatch'; END IF;
|
||||
END $seed$;
|
||||
INSERT INTO workbench_history(id,tenant_id,owner_id,source_kind,source_id,payload)
|
||||
VALUES(${literal(historyID)},${manifest.tenant_id},${literal(ownerID)},'pendingmemory',${literal(id)},${literal(JSON.stringify({ fixture: manifest.run, content: original }))}::jsonb);
|
||||
INSERT INTO workbench_pending_memories(id,tenant_id,owner_id,agent_id,content,source_ref,status,reviewed_by,legacy_history_id)
|
||||
VALUES(${literal(id)},${manifest.tenant_id},${literal(ownerID)},${literal(message.agent_id)},${literal(original)},${literal(source)},'accepted',${literal(ownerID)},${literal(historyID)});
|
||||
COMMIT;`)
|
||||
manifest.legacy_candidate = { id, history_id: historyID, agent_id: message.agent_id }
|
||||
persist()
|
||||
child.send({ type: 'legacy-memory-seeded', id, history_id: historyID })
|
||||
} catch (error) {
|
||||
manifest.legacy_seed_error = error.message
|
||||
persist()
|
||||
if (child.connected) child.send({ type: 'legacy-memory-seeded', error: error.message })
|
||||
}
|
||||
})
|
||||
child.once('error', error => { clearTimeout(timer); reject(error) })
|
||||
child.once('close', (code, signal) => {
|
||||
clearTimeout(timer)
|
||||
manifest.smoke_exit_code = code
|
||||
manifest.smoke_signal = signal
|
||||
manifest.smoke_finished_at = new Date().toISOString()
|
||||
manifest.passed = code === 0
|
||||
persist()
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function cleanup() {
|
||||
validateManifest()
|
||||
if (manifest.smoke_pid && !manifest.smoke_finished_at) {
|
||||
let running = true
|
||||
try { process.kill(manifest.smoke_pid, 0) } catch (error) {
|
||||
if (error.code === 'ESRCH') running = false
|
||||
}
|
||||
assert.equal(running, false, 'Fixture smoke process is still alive; wait for it to finish before cleanup')
|
||||
}
|
||||
const discovered = sql(`SELECT id FROM tenants WHERE name=${literal(manifest.run)} AND business='native-memory-smoke-fixture';`)
|
||||
if (discovered) {
|
||||
const tenant = Number(discovered)
|
||||
assert.ok(Number.isSafeInteger(tenant) && tenant > 0 && tenant !== manifest.source_tenant_id)
|
||||
if (manifest.tenant_id) assert.equal(tenant, manifest.tenant_id)
|
||||
manifest.tenant_id = tenant
|
||||
}
|
||||
const tenant = manifest.tenant_id || 0
|
||||
const identities = manifest.users.map(user => `(username=${literal(user.username)} AND email=${literal(user.email)})`).join(' OR ')
|
||||
const rows = JSON.parse(sql(`SELECT COALESCE(json_agg(json_build_object('id',id,'username',username,'tenant_id',tenant_id)),'[]') FROM users WHERE ${identities};`))
|
||||
for (const row of rows) {
|
||||
const user = manifest.users.find(value => value.username === row.username)
|
||||
if (user.id) assert.equal(user.id, row.id)
|
||||
assert.ok([manifest.source_tenant_id, tenant].includes(Number(row.tenant_id)), 'Fixture user moved to an unexpected workspace')
|
||||
user.id = row.id
|
||||
}
|
||||
persist()
|
||||
const userIDs = manifest.users.filter(user => user.id).map(user => literal(user.id)).join(',') || "''"
|
||||
// Unknown nonempty tables stop cleanup. This prevents silently orphaning new
|
||||
// fixture dependencies when future API changes add another storage surface.
|
||||
const tables = [
|
||||
'workbench_pending_memories', 'workbench_revisions', 'workbench_events', 'workbench_jobs',
|
||||
'workbench_resources', 'workbench_balance_holds', 'workbench_usage', 'workbench_transactions',
|
||||
'workbench_balances', 'workbench_operation_logs', 'workbench_consumption', 'workbench_history',
|
||||
'workbench_usage_quotas', 'workbench_resource_forks', 'workbench_resource_submissions',
|
||||
'workbench_resource_hidden', 'workbench_resource_version_history',
|
||||
'workbench_access_grants', 'workbench_access_publications', 'workbench_access_assignments', 'workbench_access_roles',
|
||||
'memory_item_embeddings', 'memory_tombstones', 'memory_items', 'memory_topic_stats', 'memory_doc_affinity', 'memory_subjects',
|
||||
'custom_agents', 'models', 'audit_logs', 'tenant_members',
|
||||
]
|
||||
const tableList = tables.map(literal).join(',')
|
||||
const query = `BEGIN;
|
||||
SET LOCAL lock_timeout = '3s';
|
||||
SET LOCAL statement_timeout = '20s';
|
||||
DO $fixture$
|
||||
DECLARE entry record; count_rows bigint;
|
||||
BEGIN
|
||||
IF ${tenant} <> 0 AND EXISTS (SELECT 1 FROM tenants WHERE id=${tenant} AND (name<>${literal(manifest.run)} OR business<>'native-memory-smoke-fixture')) THEN RAISE EXCEPTION 'Fixture tenant identity mismatch'; END IF;
|
||||
IF EXISTS (SELECT 1 FROM users WHERE id IN (${userIDs}) AND (is_system_admin OR can_access_all_tenants OR NOT (${identities}))) THEN RAISE EXCEPTION 'Unexpected user identity'; END IF;
|
||||
IF ${tenant} > 0 AND (EXISTS (SELECT 1 FROM users WHERE tenant_id=${tenant} AND id NOT IN (${userIDs})) OR EXISTS (SELECT 1 FROM tenant_members WHERE tenant_id=${tenant} AND user_id NOT IN (${userIDs}))) THEN RAISE EXCEPTION 'Fixture has other users'; END IF;
|
||||
IF EXISTS (SELECT 1 FROM tenant_members WHERE user_id IN (${userIDs}) AND tenant_id NOT IN (${tenant},${manifest.source_tenant_id})) THEN RAISE EXCEPTION 'Fixture user has foreign memberships'; END IF;
|
||||
IF ${tenant} > 0 AND EXISTS (SELECT 1 FROM workbench_jobs WHERE tenant_id=${tenant} AND status IN ('queued','running')) THEN RAISE EXCEPTION 'Fixture jobs are still active'; END IF;
|
||||
FOR entry IN SELECT table_name FROM information_schema.columns WHERE table_schema='public' AND column_name='tenant_id' AND table_name NOT IN (${tableList},'users') LOOP
|
||||
EXECUTE format('SELECT count(*) FROM %I WHERE tenant_id=$1 AND $1>0',entry.table_name) INTO count_rows USING ${tenant};
|
||||
IF count_rows > 0 THEN RAISE EXCEPTION 'Unknown fixture dependency: %',entry.table_name; END IF;
|
||||
END LOOP;
|
||||
${tables.map(table => `IF ${tenant} > 0 AND to_regclass('public.${table}') IS NOT NULL THEN DELETE FROM ${table} WHERE tenant_id=${tenant}; END IF;`).join('\n ')}
|
||||
DELETE FROM auth_tokens WHERE user_id IN (${userIDs});
|
||||
DELETE FROM workbench_access_assignments WHERE user_id IN (${userIDs}) AND tenant_id=${manifest.source_tenant_id};
|
||||
DELETE FROM workbench_access_grants WHERE user_id IN (${userIDs}) AND tenant_id=${manifest.source_tenant_id};
|
||||
DELETE FROM tenant_members WHERE user_id IN (${userIDs}) AND tenant_id=${manifest.source_tenant_id};
|
||||
DELETE FROM users WHERE id IN (${userIDs}) AND (${identities}) AND tenant_id IN (${tenant},${manifest.source_tenant_id});
|
||||
DELETE FROM tenants WHERE id=${tenant} AND name=${literal(manifest.run)} AND business='native-memory-smoke-fixture';
|
||||
END $fixture$;
|
||||
COMMIT;`
|
||||
fs.writeFileSync(path.join(path.dirname(manifestPath), 'fixture-cleanup.sql'), query + '\n', { mode: 0o600 })
|
||||
sql(query)
|
||||
assert.equal(sql(`SELECT (SELECT count(*) FROM tenants WHERE id=${tenant} AND ${tenant}>0) + (SELECT count(*) FROM users WHERE id IN (${userIDs}));`), '0')
|
||||
for (const name of ['owner-auth.json', 'outsider-auth.json']) fs.rmSync(path.join(path.dirname(manifestPath), name), { force: true })
|
||||
manifest.cleaned_at = new Date().toISOString()
|
||||
manifest.cleanup_error = null
|
||||
manifest.cleanup_note = 'Dedicated tenant, fixture users, tokens, balances, model, Agent and fixture data removed. Source workspace administrator audit entries are retained.'
|
||||
persist()
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (process.argv[2] === '--cleanup') {
|
||||
assert.ok(process.argv[3], '--cleanup requires the fixture manifest path')
|
||||
manifestPath = path.resolve(process.argv[3])
|
||||
manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'))
|
||||
cleanup()
|
||||
return
|
||||
}
|
||||
if (process.argv[2] !== '--run') {
|
||||
console.log('Use --run to create an isolated fixture, run native_memory_api_smoke.cjs and remove all fixture data; use --cleanup /absolute/path/fixture.json to retry cleanup. See native_memory_api_smoke.md.')
|
||||
return
|
||||
}
|
||||
const run = `memory-smoke-${crypto.randomBytes(8).toString('hex')}`
|
||||
const directory = path.join(acceptance, run)
|
||||
fs.mkdirSync(directory, { mode: 0o700, recursive: true })
|
||||
manifestPath = path.join(directory, 'fixture.json')
|
||||
manifest = { format: 'native-memory-fixture-v1', run, database,
|
||||
base: (process.env.MEMORY_SMOKE_BASE || 'http://127.0.0.1:18080/api/v1').replace(/\/$/, ''),
|
||||
source_tenant_id: Number(process.env.MEMORY_SMOKE_SOURCE_TENANT || 10059),
|
||||
started_at: new Date().toISOString(), passed: false,
|
||||
users: ['owner', 'outsider'].map(label => ({ username: `${run}-${label}`, email: `${run}-${label}@example.test` })) }
|
||||
validateManifest()
|
||||
persist()
|
||||
try {
|
||||
await provision()
|
||||
await runSmoke()
|
||||
} catch (error) {
|
||||
manifest.error = error.message
|
||||
process.exitCode = 1
|
||||
persist()
|
||||
} finally {
|
||||
try { cleanup() } catch (error) { manifest.cleanup_error = error.message; process.exitCode = 1; persist() }
|
||||
}
|
||||
if (!manifest.passed || !manifest.cleaned_at) process.exitCode = 1
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
process.exitCode = 1
|
||||
console.error(error.message)
|
||||
}).finally(() => {
|
||||
if (manifest) console.log(JSON.stringify({ passed: manifest.passed, cleaned: !!manifest.cleaned_at, error: manifest.error, cleanup_error: manifest.cleanup_error, manifest: manifestPath }, null, 2))
|
||||
})
|
||||
Executable
+97
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
#
|
||||
# 本地构建 + 打包 WeKnora Lite 发行包
|
||||
#
|
||||
# 用法:
|
||||
# ./scripts/package-lite.sh # 自动检测版本
|
||||
# ./scripts/package-lite.sh v0.2.0 # 指定版本号
|
||||
# SKIP_FRONTEND=1 ./scripts/package-lite.sh # 跳过前端构建(使用已有 web/)
|
||||
#
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
cd "${ROOT_DIR}"
|
||||
|
||||
# Resolve version
|
||||
if [ -n "${1:-}" ]; then
|
||||
VERSION="$1"
|
||||
elif command -v git >/dev/null 2>&1; then
|
||||
VERSION=$(git describe --tags --abbrev=0 2>/dev/null || echo "dev")
|
||||
else
|
||||
VERSION="dev"
|
||||
fi
|
||||
|
||||
GOOS=$(go env GOOS)
|
||||
GOARCH=$(go env GOARCH)
|
||||
ARCHIVE="WeKnora-lite_${VERSION}_${GOOS}_${GOARCH}"
|
||||
DIST_DIR="dist/${ARCHIVE}"
|
||||
|
||||
echo "=== WeKnora Lite Packager ==="
|
||||
echo " Version : ${VERSION}"
|
||||
echo " Platform: ${GOOS}/${GOARCH}"
|
||||
echo " Output : dist/${ARCHIVE}.tar.gz"
|
||||
echo ""
|
||||
|
||||
# ── Step 1: Build frontend (if not skipped) ──
|
||||
if [ "${SKIP_FRONTEND:-}" != "1" ]; then
|
||||
if [ -f frontend/package.json ]; then
|
||||
echo ">> Building frontend..."
|
||||
(cd frontend && npm ci --prefer-offline && npm run build)
|
||||
rm -rf web
|
||||
cp -r frontend/dist web
|
||||
else
|
||||
echo ">> No frontend/package.json found, skipping frontend build"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f web/index.html ]; then
|
||||
echo "WARNING: web/index.html not found — package will not include frontend"
|
||||
fi
|
||||
|
||||
# ── Step 2: Build Go binary ──
|
||||
echo ">> Building WeKnora-lite binary..."
|
||||
export EDITION=lite
|
||||
eval "$(./scripts/get_version.sh env)"
|
||||
LDFLAGS="-w -s $(./scripts/get_version.sh ldflags)"
|
||||
export CGO_CFLAGS="-Wno-deprecated-declarations"
|
||||
if [ "$(uname)" = "Darwin" ]; then
|
||||
export CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries"
|
||||
fi
|
||||
CGO_ENABLED=1 go build -tags "sqlite_fts5" -ldflags="${LDFLAGS}" \
|
||||
-o WeKnora-lite ./cmd/server
|
||||
|
||||
# ── Step 3: Assemble package ──
|
||||
echo ">> Assembling package..."
|
||||
rm -rf "${DIST_DIR}"
|
||||
mkdir -p "${DIST_DIR}/web"
|
||||
|
||||
cp WeKnora-lite "${DIST_DIR}/"
|
||||
if [ -d web ] && [ -f web/index.html ]; then
|
||||
cp -r web/* "${DIST_DIR}/web/"
|
||||
fi
|
||||
cp .env.lite.example "${DIST_DIR}/"
|
||||
cp docs/LITE.md "${DIST_DIR}/README.md"
|
||||
if [ -d config ]; then
|
||||
cp -r config "${DIST_DIR}/config"
|
||||
fi
|
||||
if [ -d migrations/sqlite ]; then
|
||||
mkdir -p "${DIST_DIR}/migrations/sqlite"
|
||||
cp -r migrations/sqlite/* "${DIST_DIR}/migrations/sqlite/"
|
||||
fi
|
||||
if [ -f deploy/weknora-lite.service ]; then
|
||||
cp deploy/weknora-lite.service "${DIST_DIR}/"
|
||||
fi
|
||||
|
||||
# ── Step 4: Create tarball ──
|
||||
echo ">> Creating tarball..."
|
||||
(cd dist && tar czf "${ARCHIVE}.tar.gz" "${ARCHIVE}")
|
||||
(cd dist && shasum -a 256 "${ARCHIVE}.tar.gz" > "${ARCHIVE}.tar.gz.sha256")
|
||||
|
||||
echo ""
|
||||
echo "=== Done ==="
|
||||
echo " dist/${ARCHIVE}.tar.gz"
|
||||
echo " dist/${ARCHIVE}.tar.gz.sha256"
|
||||
SIZE=$(du -h "dist/${ARCHIVE}.tar.gz" | cut -f1)
|
||||
echo " Size: ${SIZE}"
|
||||
Executable
+101
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
#
|
||||
# 本地构建 + 打包 WeKnora macOS 桌面应用 (.app)
|
||||
#
|
||||
# 用法:
|
||||
# ./scripts/package-mac-app.sh
|
||||
# SKIP_FRONTEND=1 ./scripts/package-mac-app.sh # 跳过前端构建
|
||||
#
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
cd "${ROOT_DIR}"
|
||||
|
||||
APP_NAME="WeKnora Lite"
|
||||
APP_BUNDLE="${APP_NAME}.app"
|
||||
DIST_DIR="dist/${APP_BUNDLE}"
|
||||
|
||||
echo "=== WeKnora Mac App Packager ==="
|
||||
echo " Output: dist/${APP_BUNDLE}"
|
||||
echo ""
|
||||
|
||||
# ── Step 1: Build frontend (if not skipped) ──
|
||||
if [ "${SKIP_FRONTEND:-}" != "1" ]; then
|
||||
if [ -f frontend/package.json ]; then
|
||||
echo ">> Building frontend..."
|
||||
(cd frontend && npm ci --prefer-offline && npm run build)
|
||||
# Lite 后端从 ./web 提供 SPA(见 internal/router serveFrontendStatic),与 package-lite.sh 一致须用 dist 更新 web
|
||||
echo ">> Sync frontend/dist -> web/"
|
||||
rm -rf web
|
||||
cp -r frontend/dist web
|
||||
else
|
||||
echo ">> No frontend/package.json found, skipping frontend build"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f web/index.html ]; then
|
||||
echo "WARNING: web/index.html not found. Lite 桌面会从 Resources/web 提供前端;请先完整构建前端(勿使用 SKIP_FRONTEND),或手动: cp -r frontend/dist web"
|
||||
fi
|
||||
|
||||
# ── Step 2: Build with Wails ──
|
||||
echo ">> Building Wails Desktop App..."
|
||||
|
||||
# 如果没有 wails 命令行工具,提醒安装
|
||||
if ! command -v wails >/dev/null 2>&1; then
|
||||
echo "Wails CLI not found. Please install it first:"
|
||||
echo "go install github.com/wailsapp/wails/v2/cmd/wails@latest"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 使用 Wails 打包 (需要先处理依赖代理问题)
|
||||
export GONOSUMDB="git.sr.ht/*"
|
||||
export GOPROXY="https://goproxy.cn,direct"
|
||||
export CGO_CFLAGS="-Wno-deprecated-declarations"
|
||||
export CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries"
|
||||
export EDITION=lite
|
||||
|
||||
# Milvus 与 Qdrant 的 gRPC 生成代码均注册名为 "common.proto" 的描述符,同一进程内会冲突。
|
||||
# -ldflags -X conflictPolicy=warn 只作用于最终链接,无法覆盖 Wails「生成绑定」时单独启动的 go 子进程。
|
||||
# 官方做法:见 https://protobuf.dev/reference/go/faq#namespace-conflict
|
||||
export GOLANG_PROTOBUF_REGISTRATION_CONFLICT=warn
|
||||
|
||||
# 获取版本号并配置 LDFLAGS
|
||||
eval "$(./scripts/get_version.sh env)"
|
||||
LDFLAGS="$(./scripts/get_version.sh ldflags) -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"
|
||||
|
||||
# 默认生成 Wails 绑定(供 window.go.main.App.* 等);若加 -skipbindings 则需在 OnDomReady 注入 __WEKNORA_API_BASE__
|
||||
(cd cmd/desktop && wails build -clean -tags "sqlite_fts5" -ldflags="$LDFLAGS" -o "${APP_NAME}")
|
||||
|
||||
# ── Step 3: Copy generated .app to dist ──
|
||||
echo ">> Assembling package..."
|
||||
mkdir -p dist
|
||||
rm -rf "${DIST_DIR}"
|
||||
cp -R "cmd/desktop/build/bin/${APP_BUNDLE}" "dist/"
|
||||
|
||||
# 将配置文件和初始数据库迁移脚本塞进 .app 内部资源里
|
||||
RESOURCES_DIR="${DIST_DIR}/Contents/Resources"
|
||||
mkdir -p "${RESOURCES_DIR}/config"
|
||||
mkdir -p "${RESOURCES_DIR}/migrations/sqlite"
|
||||
|
||||
if [ -f .env.lite.example ]; then
|
||||
cp .env.lite.example "${RESOURCES_DIR}/.env"
|
||||
fi
|
||||
if [ -d migrations/sqlite ]; then
|
||||
cp -r migrations/sqlite/* "${RESOURCES_DIR}/migrations/sqlite/"
|
||||
fi
|
||||
if [ -d config ]; then
|
||||
cp -r config/* "${RESOURCES_DIR}/config/"
|
||||
fi
|
||||
if [ -d web ]; then
|
||||
cp -r web "${RESOURCES_DIR}/"
|
||||
fi
|
||||
|
||||
# 注意:Wails build 生成的二进制文件工作目录默认是 app 的 Contents/MacOS 目录
|
||||
# 后续可能需要调整代码中对配置文件的路径读取逻辑。
|
||||
|
||||
echo ""
|
||||
echo "=== Done ==="
|
||||
echo " App generated at: dist/${APP_BUNDLE}"
|
||||
echo " You can double click it to run!"
|
||||
@@ -0,0 +1,49 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
|
||||
// Snapshot only this build, before adding older hashed chunks for open tabs.
|
||||
function preserveFrontendAssets(root, revision) {
|
||||
if (!/^[a-zA-Z0-9-]+$/.test(revision)) throw new Error('Invalid asset revision')
|
||||
const assets = path.join(root, 'src/frontend/dist/assets')
|
||||
const archive = path.join(root, 'frontend-asset-archives')
|
||||
fs.mkdirSync(archive, { recursive: true })
|
||||
const previous = fs.readdirSync(archive).filter(name => fs.statSync(path.join(archive, name)).isDirectory())
|
||||
.sort((a, b) => fs.statSync(path.join(archive, b)).mtimeMs - fs.statSync(path.join(archive, a)).mtimeMs).slice(0, 3)
|
||||
const snapshot = path.join(archive, revision)
|
||||
if (fs.existsSync(snapshot)) throw new Error('Asset snapshot already exists')
|
||||
fs.cpSync(assets, snapshot, { recursive: true, errorOnExist: true, force: false })
|
||||
if (!previous.length) {
|
||||
const docker = (...args) => execFileSync('docker', ['--context', 'colima-yawan', ...args], { encoding: 'utf8' }).trim()
|
||||
const containers = docker('ps', '-a', '--filter', 'name=yawan-public-frontend', '--format', '{{.Names}}').split('\n').filter(Boolean)
|
||||
for (const container of containers) {
|
||||
if (!/^yawan-public-frontend(?:-before-[a-zA-Z0-9-]+)?$/.test(container)) continue
|
||||
const name = `bootstrap-${container}`
|
||||
const destination = path.join(archive, name)
|
||||
if (!fs.existsSync(destination)) docker('cp', `${container}:/usr/share/nginx/html/assets`, destination)
|
||||
previous.push(name)
|
||||
}
|
||||
}
|
||||
let retained = 0
|
||||
for (const name of previous) retained += mergeAssets(path.join(archive, name), assets)
|
||||
const now = new Date()
|
||||
fs.utimesSync(snapshot, now, now)
|
||||
return { retained, previous_builds: previous.length }
|
||||
}
|
||||
|
||||
function mergeAssets(source, destination) {
|
||||
let copied = 0
|
||||
for (const entry of fs.readdirSync(source, { withFileTypes: true })) {
|
||||
const from = path.join(source, entry.name), to = path.join(destination, entry.name)
|
||||
if (entry.isSymbolicLink()) throw new Error('Asset symlinks are not supported')
|
||||
if (entry.isDirectory()) {
|
||||
fs.mkdirSync(to, { recursive: true })
|
||||
copied += mergeAssets(from, to)
|
||||
} else if (entry.isFile() && !fs.existsSync(to)) {
|
||||
fs.copyFileSync(from, to, fs.constants.COPYFILE_EXCL)
|
||||
copied++
|
||||
}
|
||||
}
|
||||
return copied
|
||||
}
|
||||
module.exports = { preserveFrontendAssets, mergeAssets }
|
||||
@@ -0,0 +1,21 @@
|
||||
const { test } = require('node:test')
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const os = require('node:os')
|
||||
const path = require('node:path')
|
||||
const { mergeAssets } = require('./preserve_frontend_assets.cjs')
|
||||
|
||||
test('keeps old lazy chunks without replacing current build files', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'assets-test-'))
|
||||
try {
|
||||
const old = path.join(dir, 'old'), current = path.join(dir, 'current')
|
||||
fs.mkdirSync(old); fs.mkdirSync(current)
|
||||
fs.writeFileSync(path.join(old, 'Agent-oldhash.js'), 'old import')
|
||||
fs.writeFileSync(path.join(old, 'shared.css'), 'old style')
|
||||
fs.writeFileSync(path.join(current, 'shared.css'), 'current style')
|
||||
assert.equal(mergeAssets(old, current), 1)
|
||||
assert.equal(fs.readFileSync(path.join(current, 'Agent-oldhash.js'), 'utf8'), 'old import')
|
||||
assert.equal(fs.readFileSync(path.join(current, 'shared.css'), 'utf8'), 'current style')
|
||||
assert.equal(mergeAssets(old, current), 0)
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }) }
|
||||
})
|
||||
@@ -0,0 +1,109 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const test = require('node:test')
|
||||
const fs = require('node:fs')
|
||||
const os = require('node:os')
|
||||
const path = require('node:path')
|
||||
const vm = require('node:vm')
|
||||
const { stagePublicAPIContext } = require('./stage_public_api_context.cjs')
|
||||
|
||||
const release = 'public-20260921-040700'
|
||||
|
||||
function publisher(t, action, failBuild = false) {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'public-api-publisher-'))
|
||||
t.after(() => fs.rmSync(root, { recursive: true, force: true }))
|
||||
const source = path.join(root, 'src')
|
||||
for (const directory of ['scripts', 'migrations', 'frontend/dist']) fs.mkdirSync(path.join(source, directory), { recursive: true })
|
||||
fs.mkdirSync(path.join(root, 'acceptance'))
|
||||
fs.writeFileSync(path.join(source, 'scripts/Dockerfile.public-api'), fs.readFileSync(path.join(__dirname, 'Dockerfile.public-api')))
|
||||
fs.writeFileSync(path.join(source, 'WeKnora-native-acceptance-next'), 'reviewed-api', { mode: 0o755 })
|
||||
fs.writeFileSync(path.join(source, 'migrations/001.sql'), 'SELECT 1;\n')
|
||||
fs.writeFileSync(path.join(root, 'acceptance/runtime.env'), 'DB_NAME=yawan_native_acceptance_20260915\n')
|
||||
const directory = path.join(root, 'releases', release)
|
||||
if (action === 'refreshAPI') {
|
||||
fs.mkdirSync(directory, { recursive: true })
|
||||
fs.writeFileSync(path.join(directory, 'manifest.json'), JSON.stringify({ release, stage: 'origin-ready',
|
||||
api_image: 'yawan-api:previous', api_image_id: 'sha256:previous' }))
|
||||
}
|
||||
const calls = []
|
||||
const module = { exports: {} }
|
||||
const injectedRequire = name => {
|
||||
if (name === 'node:fs') return { ...fs, copyFileSync(from, to, flags) {
|
||||
if (from === '/Users/macclaw/.cloudflared/config.yml') {
|
||||
calls.push({ kind: 'tunnel-backup', to })
|
||||
return fs.writeFileSync(to, 'previous tunnel configuration\n')
|
||||
}
|
||||
return fs.copyFileSync(from, to, flags)
|
||||
} }
|
||||
if (name === 'node:child_process') return { execFileSync(binary, args) {
|
||||
assert.equal(binary, 'docker')
|
||||
assert.deepEqual(Array.from(args.slice(0, 2)), ['--context', 'colima-yawan'])
|
||||
const command = Array.from(args.slice(2))
|
||||
calls.push({ kind: 'docker', command })
|
||||
if (command[0] === 'build' && command[2].startsWith('yawan-api:')) {
|
||||
if (failBuild) throw new Error('stubbed API build failure')
|
||||
assert.deepEqual(fs.readdirSync(command.at(-1)).sort(), ['Dockerfile', 'WeKnora-native-acceptance-next', 'migrations'])
|
||||
}
|
||||
return command[0] === 'image' ? 'sha256:built-image\n' : ''
|
||||
} }
|
||||
if (name === './preserve_frontend_assets.cjs') return { preserveFrontendAssets() { calls.push({ kind: 'retain-assets' }); return { retained: 7 } } }
|
||||
if (name === './stage_public_api_context.cjs') return { stagePublicAPIContext }
|
||||
return require(name)
|
||||
}
|
||||
const context = { require: injectedRequire, module, __dirname: path.join(source, 'scripts'), URL, AbortSignal,
|
||||
fetch: async () => ({ ok: true }), console: { log() {} }, recordIdle: () => calls.push({ kind: 'idle' }) }
|
||||
vm.runInNewContext(fs.readFileSync(path.join(__dirname, 'publish_public_runtime.cjs'), 'utf8')
|
||||
+ '\n;idle = recordIdle; globalThis.actions = { prepare, refreshAPI };', context)
|
||||
return { root, source, directory, calls, run: () => context.actions[action](release) }
|
||||
}
|
||||
|
||||
test('prepare builds its API from the release snapshot after backups and retains frontend behavior', t => {
|
||||
const value = publisher(t, 'prepare')
|
||||
value.run()
|
||||
const builds = value.calls.filter(call => call.kind === 'docker' && call.command[0] === 'build')
|
||||
assert.deepEqual(builds.map(call => call.command), [
|
||||
['build', '-t', `yawan-api:${release}`, path.join(value.directory, 'api-context')],
|
||||
['build', '-t', `yawan-frontend:${release}`, path.join(value.source, 'frontend')],
|
||||
])
|
||||
const firstBuild = value.calls.indexOf(builds[0])
|
||||
const backups = value.calls.filter(call => call.kind === 'docker' && call.command.includes('pg_dump'))
|
||||
assert.equal(backups.length, 2)
|
||||
assert.ok(backups.every(call => value.calls.indexOf(call) < firstBuild))
|
||||
assert.ok(value.calls.find(call => call.kind === 'tunnel-backup'))
|
||||
assert.ok(value.calls.find(call => call.kind === 'retain-assets'))
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(value.directory, 'manifest.json')))
|
||||
assert.equal(manifest.stage, 'prepared')
|
||||
assert.equal(manifest.api_image_id, 'sha256:built-image')
|
||||
assert.equal(manifest.frontend_image_id, 'sha256:built-image')
|
||||
assert.equal(manifest.retained_assets.retained, 7)
|
||||
})
|
||||
|
||||
test('refresh-api uses its immutable revision snapshot and preserves idle and image identity checks', async t => {
|
||||
const value = publisher(t, 'refreshAPI')
|
||||
await value.run()
|
||||
const buildIndex = value.calls.findIndex(call => call.kind === 'docker' && call.command[0] === 'build')
|
||||
const build = value.calls[buildIndex].command
|
||||
const revision = build[2].match(/-api-(\d{17})$/)[1]
|
||||
assert.deepEqual(build, ['build', '-t', `yawan-api:${release}-api-${revision}`, path.join(value.directory, `api-context-${revision}`)])
|
||||
const idleIndexes = value.calls.flatMap((call, index) => call.kind === 'idle' ? [index] : [])
|
||||
const stopIndex = value.calls.findIndex(call => call.kind === 'docker' && call.command[0] === 'stop')
|
||||
assert.equal(idleIndexes.length, 2)
|
||||
assert.ok(idleIndexes[0] < buildIndex && buildIndex < idleIndexes[1] && idleIndexes[1] < stopIndex)
|
||||
const start = value.calls.find(call => call.kind === 'docker' && call.command[0] === 'run').command
|
||||
assert.equal(start.at(-1), 'sha256:built-image')
|
||||
assert.ok(start.includes('WEKNORA_REVIEWED_API_IMAGE=sha256:built-image'))
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(value.directory, 'manifest.json')))
|
||||
assert.equal(manifest.api_image_id, 'sha256:built-image')
|
||||
assert.deepEqual(manifest.previous_apis, [{ image: 'yawan-api:previous', image_id: 'sha256:previous',
|
||||
container: `yawan-native-acceptance-before-${revision}` }])
|
||||
})
|
||||
|
||||
test('an API build failure retains the staged context and never stops the running API', async t => {
|
||||
const value = publisher(t, 'refreshAPI', true)
|
||||
await assert.rejects(value.run(), /stubbed API build failure/)
|
||||
assert.equal(value.calls.some(call => call.kind === 'docker' && ['stop', 'rename', 'run'].includes(call.command[0])), false)
|
||||
const build = value.calls.find(call => call.kind === 'docker' && call.command[0] === 'build').command
|
||||
assert.equal(fs.readFileSync(path.join(build.at(-1), 'WeKnora-native-acceptance-next'), 'utf8'), 'reviewed-api')
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(value.directory, 'manifest.json')))
|
||||
assert.equal(manifest.api_image_id, 'sha256:previous')
|
||||
assert.equal(manifest.stage, 'origin-ready')
|
||||
})
|
||||
@@ -0,0 +1,447 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
const { parseEnvironment } = require('./workbench_acceptance_runtime.cjs')
|
||||
const { preserveFrontendAssets } = require('./preserve_frontend_assets.cjs')
|
||||
const { stagePublicAPIContext } = require('./stage_public_api_context.cjs')
|
||||
|
||||
const root = path.resolve(__dirname, '../..')
|
||||
const database = 'yawan_native_acceptance_20260915'
|
||||
const apiName = 'yawan-native-acceptance'
|
||||
const network = 'weknora_WeKnora-network'
|
||||
const publicVerificationReports = ['public-release-smoke', 'public-navigation', 'public-access-control', 'public-resource-isolation', 'public-resource-files', 'public-tenant-isolation', 'public-indexed-retrieval-isolation']
|
||||
const fixtureReconciliationCounts = ['active_users', 'active_memberships', 'roles', 'assignments', 'grants',
|
||||
'unrevoked_tokens', 'active_agents', 'active_skills', 'active_digital_humans', 'workbench_resources',
|
||||
'active_sessions', 'synthetic_messages', 'file_resources', 'file_bindings', 'file_access_grants',
|
||||
'active_knowledge_bases', 'active_documents', 'active_chunks', 'tags', 'tag_relations', 'publications',
|
||||
'secondary_tenant', 'secondary_memberships', 'memory_item_embeddings', 'memory_items', 'memory_tombstones',
|
||||
'memory_topic_stats', 'memory_doc_affinity', 'memory_subjects']
|
||||
const docker = (...args) => execFileSync('docker', ['--context', 'colima-yawan', ...args], {
|
||||
encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'],
|
||||
}).trim()
|
||||
const writeJSON = (file, value) => fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n', { mode: 0o600 })
|
||||
|
||||
// Include every native queue, plus queues discovered in Redis. Read all states
|
||||
// in one read-only Lua snapshot: an active task can otherwise move to retry
|
||||
// between separate commands and appear idle. Archived/completed history does
|
||||
// not block publication; waiting aggregation work does.
|
||||
const nativeQueues = ['default', 'chat_attachment', 'postprocess', 'summary', 'multimodal',
|
||||
'graph', 'question', 'sync', 'low', 'wiki', 'memory']
|
||||
const queueStates = ['active', 'pending', 'scheduled', 'retry', 'aggregating', 'aggregation_sets']
|
||||
const queueSnapshotScript = `
|
||||
local names = {}
|
||||
for _, name in ipairs(ARGV) do names[name] = true end
|
||||
for _, name in ipairs(redis.call('SMEMBERS', KEYS[1])) do names[name] = true end
|
||||
local result = {}
|
||||
for name, _ in pairs(names) do
|
||||
local prefix = 'asynq:{' .. name .. '}:'
|
||||
local aggregating = 0
|
||||
for _, group in ipairs(redis.call('SMEMBERS', prefix .. 'groups')) do
|
||||
aggregating = aggregating + redis.call('ZCARD', prefix .. 'g:' .. group)
|
||||
end
|
||||
result[name] = {
|
||||
active = redis.call('LLEN', prefix .. 'active'),
|
||||
pending = redis.call('LLEN', prefix .. 'pending'),
|
||||
scheduled = redis.call('ZCARD', prefix .. 'scheduled'),
|
||||
retry = redis.call('ZCARD', prefix .. 'retry'),
|
||||
aggregating = aggregating,
|
||||
aggregation_sets = redis.call('ZCARD', prefix .. 'aggregation_sets')
|
||||
}
|
||||
end
|
||||
return cjson.encode(result)
|
||||
`
|
||||
|
||||
function idleCount(value) {
|
||||
if (!Number.isSafeInteger(value) || value < 0) throw new Error('Cannot verify background task counts')
|
||||
return value
|
||||
}
|
||||
|
||||
function assertIdleSnapshot(snapshot) {
|
||||
if (!snapshot || idleCount(snapshot.workbench) !== 0) throw new Error('Background jobs, uploads or reserved balances are still active')
|
||||
if (idleCount(snapshot.pending_ops) !== 0) throw new Error('Wiki or other persisted background operations are still pending')
|
||||
const queues = snapshot.queues
|
||||
if (!queues || typeof queues !== 'object' || Array.isArray(queues)
|
||||
|| nativeQueues.some(name => !Object.hasOwn(queues, name))) throw new Error('Cannot verify native background queues')
|
||||
for (const state of Object.values(queues)) {
|
||||
if (!state || typeof state !== 'object' || queueStates.some(name => idleCount(state[name]) !== 0)) {
|
||||
throw new Error('Native background queues still have active or waiting tasks')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function idle(run = execFileSync) {
|
||||
const invoke = (args, env = process.env) => run('docker', ['--context', 'colima-yawan', ...args], {
|
||||
encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, timeout: 15000,
|
||||
env, stdio: ['ignore', 'pipe', 'pipe'],
|
||||
}).trim()
|
||||
let snapshot
|
||||
try {
|
||||
snapshot = JSON.parse(invoke(['exec', 'WeKnora-postgres', 'psql', '-X', '-qAt',
|
||||
'-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', database, '-c',
|
||||
"SELECT json_build_object('workbench', (SELECT count(*) FROM workbench_jobs WHERE status IN ('queued','running')) + (SELECT count(*) FROM digital_human_jobs WHERE status IN ('queued','running')) + (SELECT count(*) FROM workbench_knowledge_version_jobs WHERE status IN ('queued','running')) + (SELECT count(*) FROM knowledge_classification_jobs WHERE status='running') + (SELECT count(*) FROM resumable_uploads WHERE state NOT IN ('completed','cancelled','failed','expired')) + (SELECT count(*) FROM workbench_balance_holds WHERE status='reserved'), 'pending_ops', (SELECT count(*) FROM task_pending_ops));"]))
|
||||
const environment = JSON.parse(invoke(['inspect', apiName, '--format', '{{json .Config.Env}}']))
|
||||
if (!Array.isArray(environment) || environment.some(value => typeof value !== 'string')) throw new Error()
|
||||
const runtime = parseEnvironment(environment.join('\n'))
|
||||
// Use the running API's Redis host, ACL identity and database, not the old
|
||||
// deployment or the redis-cli defaults. Secrets are passed only in env.
|
||||
const address = /^(?:\[([^\]]+)\]|([^:]+)):(\d+)$/.exec(runtime.REDIS_ADDR || '')
|
||||
if (!address || !/^\d+$/.test(runtime.REDIS_DB || '') || runtime.DB_NAME !== database) throw new Error()
|
||||
const redisArgs = ['redis-cli', '--raw', '--no-auth-warning', '-h', address[1] || address[2],
|
||||
'-p', address[3], '-n', runtime.REDIS_DB]
|
||||
if (runtime.REDIS_USERNAME) redisArgs.push('--user', runtime.REDIS_USERNAME)
|
||||
if (/^true$/i.test((runtime.REDIS_USE_TLS || '').trim())) {
|
||||
redisArgs.push('--tls')
|
||||
if (runtime.REDIS_TLS_SERVER_NAME) redisArgs.push('--sni', runtime.REDIS_TLS_SERVER_NAME)
|
||||
if (/^true$/i.test((runtime.REDIS_TLS_INSECURE_SKIP_VERIFY || '').trim())) redisArgs.push('--insecure')
|
||||
}
|
||||
redisArgs.push('EVAL_RO', queueSnapshotScript, '1', 'asynq:queues', ...nativeQueues)
|
||||
snapshot.queues = JSON.parse(invoke(['exec', '-e', 'REDISCLI_AUTH', 'WeKnora-redis', ...redisArgs],
|
||||
{ ...process.env, REDISCLI_AUTH: runtime.REDIS_PASSWORD || '' }))
|
||||
} catch {
|
||||
// Docker, Redis and JSON errors may embed credentials or server output.
|
||||
// An unavailable or unparseable queue is never considered empty.
|
||||
throw new Error('Cannot verify background task idleness; publication stopped')
|
||||
}
|
||||
assertIdleSnapshot(snapshot)
|
||||
return snapshot
|
||||
}
|
||||
|
||||
function prepare(release) {
|
||||
if (!/^public-\d{8}-\d{6}$/.test(release || '')) throw new Error('Expected public-YYYYMMDD-HHMMSS release identifier')
|
||||
const directory = path.join(root, 'releases', release)
|
||||
if (fs.existsSync(directory)) throw new Error('Release already exists; do not overwrite immutable artifacts')
|
||||
fs.mkdirSync(directory, { recursive: true, mode: 0o700 })
|
||||
const env = parseEnvironment(fs.readFileSync(path.join(root, 'acceptance/runtime.env'), 'utf8'))
|
||||
if (env.DB_NAME !== database) throw new Error('Unexpected source database')
|
||||
Object.assign(env, {
|
||||
GIN_MODE: 'release', FRONTEND_BASE_URL: 'https://agent.pducv.cn',
|
||||
DISABLE_REGISTRATION: 'true', WEKNORA_AUTH_REGISTRATION_MODE: 'invite_only',
|
||||
WEKNORA_TENANT_ENABLE_RBAC: 'true', WEKNORA_TENANT_ENABLE_CROSS_TENANT_ACCESS: 'false',
|
||||
WEKNORA_TENANT_SELF_SERVICE_CREATION_ENABLED: 'false',
|
||||
WEKNORA_TRUSTED_PROXIES: '127.0.0.1,::1,172.19.0.0/16',
|
||||
})
|
||||
fs.writeFileSync(path.join(directory, 'runtime.env'), Object.entries(env).map(([key, value]) => `${key}=${value}`).join('\n') + '\n', { mode: 0o600 })
|
||||
fs.copyFileSync(path.join(root, 'acceptance/runtime.env'), path.join(directory, 'previous-runtime.env'))
|
||||
fs.copyFileSync('/Users/macclaw/.cloudflared/config.yml', path.join(directory, 'previous-tunnel.yml'))
|
||||
for (const db of [database, 'WeKnora']) {
|
||||
const fd = fs.openSync(path.join(directory, `${db}.dump`), 'wx', 0o600)
|
||||
try { execFileSync('docker', ['--context', 'colima-yawan', 'exec', 'WeKnora-postgres', 'pg_dump', '-U', 'postgres', '-Fc', db], { stdio: ['ignore', fd, 'pipe'] }) }
|
||||
finally { fs.closeSync(fd) }
|
||||
}
|
||||
const manifest = { release, prepared_at: new Date().toISOString(), database, api_image: `yawan-api:${release}`, frontend_image: `yawan-frontend:${release}`, previous_api: `yawan-native-before-${release}`, public_origin: 'http://127.0.0.1:18088', url: 'https://agent.pducv.cn', stage: 'prepared' }
|
||||
writeJSON(path.join(directory, 'manifest.json'), manifest)
|
||||
manifest.retained_assets = preserveFrontendAssets(root, release)
|
||||
fs.writeFileSync(path.join(root, 'src/frontend/dist/release.json'), JSON.stringify({ release, built_at: manifest.prepared_at }))
|
||||
const apiContext = stagePublicAPIContext(root, release)
|
||||
docker('build', '-t', manifest.api_image, apiContext)
|
||||
docker('build', '-t', manifest.frontend_image, path.join(root, 'src/frontend'))
|
||||
for (const [key, image] of [['api_image_id', manifest.api_image], ['frontend_image_id', manifest.frontend_image]]) {
|
||||
manifest[key] = docker('image', 'inspect', image, '--format', '{{.Id}}')
|
||||
}
|
||||
writeJSON(path.join(directory, 'manifest.json'), manifest)
|
||||
console.log(JSON.stringify({ release, stage: 'prepared', backups: 2, immutable_images: true }))
|
||||
}
|
||||
|
||||
function isActivatedRelease(runtime, manifest) {
|
||||
return !!manifest.release && !!manifest.api_image_id
|
||||
&& runtime.image === manifest.api_image_id && runtime.release === manifest.release
|
||||
}
|
||||
|
||||
async function activate(release) {
|
||||
if (!/^public-\d{8}-\d{6}$/.test(release || '')) throw new Error('Invalid release identifier')
|
||||
const directory = path.join(root, 'releases', release)
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(directory, 'manifest.json'), 'utf8'))
|
||||
if (manifest.stage !== 'prepared') throw new Error('Release has already been activated')
|
||||
idle()
|
||||
const frontendName = 'yawan-public-frontend'
|
||||
const previousFrontend = `${frontendName}-before-${release}`
|
||||
const hasFrontend = docker('ps', '-a', '--filter', `name=^/${frontendName}$`, '--format', '{{.Names}}') === frontendName
|
||||
const runtime = JSON.parse(docker('inspect', apiName, '--format', '{"image":{{json .Image}},"release":{{json (index .Config.Labels "yawan.release")}}}'))
|
||||
const apiAlreadyStarted = isActivatedRelease(runtime, manifest)
|
||||
if (apiAlreadyStarted && docker('ps', '-a', '--filter', `name=^/${manifest.previous_api}$`, '--format', '{{.Names}}') !== manifest.previous_api) {
|
||||
throw new Error('Cannot resume activation without the previous API rollback container')
|
||||
}
|
||||
let frontendRenamed = false
|
||||
let frontendCreated = false
|
||||
if (!apiAlreadyStarted) {
|
||||
docker('stop', '--time', '40', apiName)
|
||||
docker('rename', apiName, manifest.previous_api)
|
||||
}
|
||||
try {
|
||||
if (!apiAlreadyStarted) docker('run', '-d', '--name', apiName, '--restart', 'unless-stopped', '--network', network,
|
||||
'--env-file', path.join(directory, 'runtime.env'), '-p', '127.0.0.1:18080:8080',
|
||||
'-e', `WEKNORA_REVIEWED_RELEASE=${release}`, '-e', `WEKNORA_REVIEWED_API_IMAGE=${manifest.api_image_id}`,
|
||||
'--label', `yawan.release=${release}`,
|
||||
'-v', `${path.join(root, 'acceptance/files')}:/data/files`,
|
||||
'-v', `${path.join(root, 'acceptance/resumable-uploads')}:/data/uploads`, manifest.api_image_id)
|
||||
let healthy = false
|
||||
for (let attempt = 0; attempt < 45; attempt++) {
|
||||
try { healthy = (await fetch('http://127.0.0.1:18080/health', { signal: AbortSignal.timeout(1500) })).ok } catch {}
|
||||
if (healthy) break
|
||||
await new Promise(resolve => setTimeout(resolve, 1000))
|
||||
}
|
||||
if (!healthy) throw new Error('New API did not become healthy')
|
||||
if (hasFrontend) {
|
||||
docker('stop', '--time', '20', frontendName)
|
||||
docker('rename', frontendName, previousFrontend)
|
||||
frontendRenamed = true
|
||||
}
|
||||
docker('run', '-d', '--name', 'yawan-public-frontend', '--restart', 'unless-stopped', '--network', network,
|
||||
'-p', '127.0.0.1:18088:80', '-e', `APP_HOST=${apiName}`, '-e', 'APP_PORT=8080',
|
||||
'-e', 'DEFAULT_LOCALE=zh-CN', '-e', 'MAX_FILE_SIZE_MB=50', '--label', `yawan.release=${release}`, manifest.frontend_image_id)
|
||||
frontendCreated = true
|
||||
docker('exec', 'yawan-public-frontend', 'nginx', '-t')
|
||||
const frontendHealth = await fetch('http://127.0.0.1:18088/api/v1/access/me', { signal: AbortSignal.timeout(10000) })
|
||||
if (frontendHealth.status !== 401) throw new Error('Frontend proxy health failed')
|
||||
} catch (error) {
|
||||
if (frontendCreated || frontendRenamed) {
|
||||
try { docker('rm', '-f', frontendName) } catch {}
|
||||
}
|
||||
try { docker('rm', '-f', apiName) } catch {}
|
||||
docker('rename', manifest.previous_api, apiName)
|
||||
docker('start', apiName)
|
||||
if (frontendRenamed) {
|
||||
docker('rename', previousFrontend, frontendName)
|
||||
docker('start', frontendName)
|
||||
} else if (hasFrontend) {
|
||||
docker('restart', frontendName)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
if (frontendRenamed) manifest.previous_frontend = previousFrontend
|
||||
manifest.stage = 'origin-ready'
|
||||
manifest.activated_at = new Date().toISOString()
|
||||
writeJSON(path.join(directory, 'manifest.json'), manifest)
|
||||
writeJSON(path.join(root, 'public-release.json'), manifest)
|
||||
console.log(JSON.stringify({ release, stage: manifest.stage, origin: manifest.public_origin, tunnel_changed: false }))
|
||||
}
|
||||
|
||||
async function refreshFrontend(release) {
|
||||
if (!/^public-\d{8}-\d{6}$/.test(release || '')) throw new Error('Invalid release identifier')
|
||||
const directory = path.join(root, 'releases', release)
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(directory, 'manifest.json'), 'utf8'))
|
||||
if (manifest.stage !== 'origin-ready') throw new Error('Only refresh an unpublished origin')
|
||||
const builtAt = new Date().toISOString()
|
||||
const revision = builtAt.replace(/[^0-9]/g, '')
|
||||
const image = `yawan-frontend:${release}-ui-${revision}`
|
||||
manifest.retained_assets = preserveFrontendAssets(root, `${release}-ui-${revision}`)
|
||||
fs.writeFileSync(path.join(root, 'src/frontend/dist/release.json'), JSON.stringify({ release, built_at: builtAt, frontend_revision: revision }))
|
||||
docker('build', '-t', image, path.join(root, 'src/frontend'))
|
||||
const imageID = docker('image', 'inspect', image, '--format', '{{.Id}}')
|
||||
const previous = `yawan-public-frontend-before-${revision}`
|
||||
docker('stop', '--time', '20', 'yawan-public-frontend')
|
||||
docker('rename', 'yawan-public-frontend', previous)
|
||||
try {
|
||||
docker('run', '-d', '--name', 'yawan-public-frontend', '--restart', 'unless-stopped', '--network', network,
|
||||
'-p', '127.0.0.1:18088:80', '-e', `APP_HOST=${apiName}`, '-e', 'APP_PORT=8080',
|
||||
'-e', 'DEFAULT_LOCALE=zh-CN', '-e', 'MAX_FILE_SIZE_MB=50', '--label', `yawan.release=${release}`, imageID)
|
||||
docker('exec', 'yawan-public-frontend', 'nginx', '-t')
|
||||
if (!(await fetch('http://127.0.0.1:18088/login', { signal: AbortSignal.timeout(10000) })).ok) throw new Error('Frontend health failed')
|
||||
} catch (error) {
|
||||
try { docker('rm', '-f', 'yawan-public-frontend') } catch {}
|
||||
docker('rename', previous, 'yawan-public-frontend')
|
||||
docker('start', 'yawan-public-frontend')
|
||||
throw error
|
||||
}
|
||||
manifest.previous_frontends = [...(manifest.previous_frontends || []), { image: manifest.frontend_image, image_id: manifest.frontend_image_id, container: previous }]
|
||||
manifest.frontend_image = image
|
||||
manifest.frontend_image_id = imageID
|
||||
manifest.frontend_built_at = builtAt
|
||||
manifest.frontend_activated_at = new Date().toISOString()
|
||||
writeJSON(path.join(directory, 'manifest.json'), manifest)
|
||||
writeJSON(path.join(root, 'public-release.json'), manifest)
|
||||
console.log(JSON.stringify({ release, stage: manifest.stage, frontend_revision: revision, tunnel_changed: false }))
|
||||
}
|
||||
|
||||
async function refreshAPI(release) {
|
||||
if (!/^public-\d{8}-\d{6}$/.test(release || '')) throw new Error('Invalid release identifier')
|
||||
const directory = path.join(root, 'releases', release)
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(directory, 'manifest.json'), 'utf8'))
|
||||
if (manifest.stage !== 'origin-ready') throw new Error('Only refresh an unverified origin')
|
||||
idle()
|
||||
const revision = new Date().toISOString().replace(/[^0-9]/g, '')
|
||||
const image = `yawan-api:${release}-api-${revision}`
|
||||
const apiContext = stagePublicAPIContext(root, release, revision)
|
||||
docker('build', '-t', image, apiContext)
|
||||
const imageID = docker('image', 'inspect', image, '--format', '{{.Id}}')
|
||||
idle()
|
||||
const previous = `${apiName}-before-${revision}`
|
||||
docker('stop', '--time', '40', apiName)
|
||||
docker('rename', apiName, previous)
|
||||
try {
|
||||
docker('run', '-d', '--name', apiName, '--restart', 'unless-stopped', '--network', network,
|
||||
'--env-file', path.join(directory, 'runtime.env'), '-p', '127.0.0.1:18080:8080',
|
||||
'-e', `WEKNORA_REVIEWED_RELEASE=${release}`, '-e', `WEKNORA_REVIEWED_API_IMAGE=${imageID}`,
|
||||
'--label', `yawan.release=${release}`,
|
||||
'-v', `${path.join(root, 'acceptance/files')}:/data/files`,
|
||||
'-v', `${path.join(root, 'acceptance/resumable-uploads')}:/data/uploads`, imageID)
|
||||
let healthy = false
|
||||
for (let attempt = 0; attempt < 45; attempt++) {
|
||||
try { healthy = (await fetch('http://127.0.0.1:18080/health', { signal: AbortSignal.timeout(1500) })).ok } catch {}
|
||||
if (healthy) break
|
||||
await new Promise(resolve => setTimeout(resolve, 1000))
|
||||
}
|
||||
if (!healthy) throw new Error('Refreshed API health failed')
|
||||
docker('exec', 'yawan-public-frontend', 'nginx', '-s', 'reload')
|
||||
} catch (error) {
|
||||
try { docker('rm', '-f', apiName) } catch {}
|
||||
docker('rename', previous, apiName)
|
||||
docker('start', apiName)
|
||||
docker('exec', 'yawan-public-frontend', 'nginx', '-s', 'reload')
|
||||
throw error
|
||||
}
|
||||
manifest.previous_apis = [...(manifest.previous_apis || []), {image: manifest.api_image, image_id: manifest.api_image_id, container: previous}]
|
||||
manifest.api_image = image
|
||||
manifest.api_image_id = imageID
|
||||
manifest.api_activated_at = new Date().toISOString()
|
||||
writeJSON(path.join(directory, 'manifest.json'), manifest)
|
||||
writeJSON(path.join(root, 'public-release.json'), manifest)
|
||||
console.log(JSON.stringify({release, stage: manifest.stage, api_revision: revision}))
|
||||
}
|
||||
|
||||
function publicationTimestamp(value, field, now) {
|
||||
const parsed = typeof value === 'string' ? Date.parse(value) : NaN
|
||||
if (!Number.isFinite(parsed) || new Date(parsed).toISOString() !== value || parsed > now) {
|
||||
throw new Error(`Invalid publication timestamp: ${field}`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
function assertSuccessfulVerification(result, report) {
|
||||
if (!result || result.passed !== true || !Array.isArray(result.checks) || !result.checks.length
|
||||
|| result.checks.some(check => check?.passed !== true)
|
||||
|| !Array.isArray(result.failures) || result.failures.length) {
|
||||
throw new Error(`Public verification did not pass: ${report}`)
|
||||
}
|
||||
if (Object.hasOwn(result, 'cleanup') && (!Array.isArray(result.cleanup) || !result.cleanup.length
|
||||
|| result.cleanup.some(item => !item || typeof item !== 'object' || Array.isArray(item)
|
||||
|| typeof item.name !== 'string' || !item.name.trim() || item.passed !== true
|
||||
|| (Object.hasOwn(item, 'error') && item.error !== undefined && item.error !== null && item.error !== '')))) {
|
||||
throw new Error(`Public verification cleanup did not pass: ${report}`)
|
||||
}
|
||||
}
|
||||
|
||||
function assertPublicVerification(manifest, result, report, now = Date.now()) {
|
||||
const activated = publicationTimestamp(manifest.activated_at, 'activated_at', now)
|
||||
const changes = [activated]
|
||||
for (const field of ['api_activated_at', 'frontend_built_at', 'frontend_activated_at']) {
|
||||
if (Object.hasOwn(manifest, field)) {
|
||||
const changed = publicationTimestamp(manifest[field], field, now)
|
||||
if (changed < activated) throw new Error(`Publication timestamp predates activation: ${field}`)
|
||||
changes.push(changed)
|
||||
}
|
||||
}
|
||||
if (manifest.previous_apis?.length && !Object.hasOwn(manifest, 'api_activated_at')) {
|
||||
throw new Error('Missing API activation timestamp')
|
||||
}
|
||||
const frontendRefreshed = Object.hasOwn(manifest, 'frontend_built_at')
|
||||
|| Object.hasOwn(manifest, 'frontend_activated_at') || manifest.previous_frontends?.length
|
||||
if (frontendRefreshed) {
|
||||
const built = publicationTimestamp(manifest.frontend_built_at, 'frontend_built_at', now)
|
||||
const enabled = publicationTimestamp(manifest.frontend_activated_at, 'frontend_activated_at', now)
|
||||
if (enabled < built) throw new Error('Frontend activation predates its build')
|
||||
}
|
||||
assertSuccessfulVerification(result, report)
|
||||
if (report === 'public-indexed-retrieval-isolation') {
|
||||
const knowledgeIDs = result.fixtures?.kbs
|
||||
if (!Array.isArray(knowledgeIDs) || !knowledgeIDs.length
|
||||
|| knowledgeIDs.some(id => typeof id !== 'string' || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id))
|
||||
|| new Set(knowledgeIDs).size !== knowledgeIDs.length) {
|
||||
throw new Error(`Invalid indexed knowledge fixtures: ${report}`)
|
||||
}
|
||||
// The final SQL reconciliation relies on these entries to attest Milvus cleanup.
|
||||
const knowledgeCleanup = result.cleanup?.filter(item => item.name.startsWith('knowledge '))
|
||||
if (!knowledgeCleanup || knowledgeCleanup.length !== knowledgeIDs.length
|
||||
|| knowledgeIDs.some(id => knowledgeCleanup.filter(item => item.name === `knowledge ${id}`).length !== 1)) {
|
||||
throw new Error(`Incomplete indexed knowledge cleanup: ${report}`)
|
||||
}
|
||||
}
|
||||
const started = publicationTimestamp(result.started_at, `${report}.started_at`, now)
|
||||
const completedField = Object.hasOwn(result, 'checked_at') ? 'checked_at' : 'time'
|
||||
const completed = publicationTimestamp(result[completedField], `${report}.${completedField}`, now)
|
||||
if (completed < started) throw new Error(`Public verification finished before it started: ${report}`)
|
||||
if (started < Math.max(...changes)) throw new Error(`Public verification started before this runtime: ${report}`)
|
||||
if (new URL(result.origin || result.base).origin !== manifest.url) throw new Error(`Wrong verification origin: ${report}`)
|
||||
if (['public-navigation', 'public-access-control'].includes(report)) {
|
||||
let apiOrigin
|
||||
try { apiOrigin = new URL(result.api_origin).origin } catch {}
|
||||
if (apiOrigin !== manifest.url) throw new Error(`Wrong verification API origin: ${report}`)
|
||||
}
|
||||
}
|
||||
|
||||
function assertFixtureReconciliation(manifest, result, reports, now = Date.now()) {
|
||||
const name = 'public-fixture-reconciliation'
|
||||
assertSuccessfulVerification(result, name)
|
||||
if (result.release !== manifest.release || result.origin !== manifest.url || result.read_only !== true) {
|
||||
throw new Error('Fixture reconciliation identity does not match this release')
|
||||
}
|
||||
const checked = publicationTimestamp(result.checked_at, `${name}.checked_at`, now)
|
||||
if (!result.reports || Array.isArray(result.reports)
|
||||
|| Object.keys(result.reports).length !== publicVerificationReports.length) {
|
||||
throw new Error('Fixture reconciliation must cover all seven source reports')
|
||||
}
|
||||
for (const sourceName of publicVerificationReports) {
|
||||
const source = reports?.[sourceName]
|
||||
assertPublicVerification(manifest, source, sourceName, now)
|
||||
const completed = Object.hasOwn(source, 'checked_at') ? source.checked_at : source.time
|
||||
const reconciled = result.reports[sourceName]
|
||||
if (!reconciled || reconciled.started_at !== source.started_at || reconciled.completed_at !== completed) {
|
||||
throw new Error(`Fixture reconciliation does not match current report: ${sourceName}`)
|
||||
}
|
||||
if (checked < Date.parse(completed)) throw new Error(`Fixture reconciliation predates source report: ${sourceName}`)
|
||||
}
|
||||
const expected = [...fixtureReconciliationCounts, 'file_fixture_directory_removed']
|
||||
const byName = new Map(result.checks.map(check => [check.name, check]))
|
||||
if (result.checks.length !== expected.length || byName.size !== expected.length
|
||||
|| expected.some(check => !byName.has(check))) {
|
||||
throw new Error('Fixture reconciliation is missing required residual checks')
|
||||
}
|
||||
for (const check of fixtureReconciliationCounts) {
|
||||
if (byName.get(check).count !== 0) throw new Error(`Fixture reconciliation has residual rows: ${check}`)
|
||||
}
|
||||
const prefix = reports['public-resource-files'].prefix
|
||||
if (!/^file-isolation-[0-9]+$/.test(prefix || '')
|
||||
|| byName.get('file_fixture_directory_removed').path !== path.join(root, 'acceptance', 'files', '10059', 'exports', prefix)) {
|
||||
throw new Error('Fixture reconciliation checked the wrong attachment directory')
|
||||
}
|
||||
}
|
||||
|
||||
async function markPublished(release) {
|
||||
if (!/^public-\d{8}-\d{6}$/.test(release || '')) throw new Error('Invalid release identifier')
|
||||
const directory = path.join(root, 'releases', release)
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(directory, 'manifest.json'), 'utf8'))
|
||||
if (manifest.release !== release) throw new Error('Manifest release does not match the requested release')
|
||||
if (manifest.stage !== 'origin-ready') throw new Error('Expected verified origin')
|
||||
const response = await fetch(`${manifest.url}/release.json`, { signal: AbortSignal.timeout(30000) })
|
||||
if (!response.ok || (await response.json()).release !== release) throw new Error('Public origin does not serve this release')
|
||||
const reports = Object.fromEntries(publicVerificationReports.map(name => [name,
|
||||
JSON.parse(fs.readFileSync(path.join(root, 'acceptance', `${name}.json`), 'utf8'))]))
|
||||
const reconciliation = JSON.parse(fs.readFileSync(path.join(root, 'acceptance', 'public-fixture-reconciliation.json'), 'utf8'))
|
||||
assertFixtureReconciliation(manifest, reconciliation, reports)
|
||||
manifest.stage = 'published'
|
||||
manifest.published_at = new Date().toISOString()
|
||||
manifest.public_verification_reports = publicVerificationReports
|
||||
manifest.fixture_reconciliation_checked_at = reconciliation.checked_at
|
||||
writeJSON(path.join(directory, 'manifest.json'), manifest)
|
||||
writeJSON(path.join(root, 'public-release.json'), manifest)
|
||||
console.log(JSON.stringify({ release, stage: manifest.stage, url: manifest.url }))
|
||||
}
|
||||
|
||||
module.exports = { idle, assertIdleSnapshot, nativeQueues, queueStates, queueSnapshotScript, isActivatedRelease, assertPublicVerification,
|
||||
assertFixtureReconciliation, publicVerificationReports, fixtureReconciliationCounts }
|
||||
|
||||
if (require.main === module) {
|
||||
const [action, release] = process.argv.slice(2)
|
||||
Promise.resolve().then(() => {
|
||||
if (action === 'prepare') return prepare(release)
|
||||
if (action === 'activate') return activate(release)
|
||||
if (action === 'refresh-frontend') return refreshFrontend(release)
|
||||
if (action === 'refresh-api') return refreshAPI(release)
|
||||
if (action === 'mark-published') return markPublished(release)
|
||||
throw new Error('Use prepare, activate, refresh-api, refresh-frontend or mark-published, followed by release identifier')
|
||||
}).catch(() => {
|
||||
// Docker diagnostics may contain environment secrets; keep console output bounded.
|
||||
console.error('Public release command failed; inspect the private release directory and container health. This command does not change the tunnel.')
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,420 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const test = require('node:test')
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const vm = require('node:vm')
|
||||
const { idle, assertIdleSnapshot, nativeQueues, queueStates, isActivatedRelease, assertPublicVerification,
|
||||
assertFixtureReconciliation, publicVerificationReports, fixtureReconciliationCounts } = require('./publish_public_runtime.cjs')
|
||||
|
||||
test('a frontend-only release with an unchanged API image is a new activation', () => {
|
||||
const manifest = { release: 'public-20260920-165100', api_image_id: 'sha256:same-image' }
|
||||
assert.equal(isActivatedRelease({ image: manifest.api_image_id, release: 'public-20260920-160500' }, manifest), false)
|
||||
assert.equal(isActivatedRelease({ image: manifest.api_image_id, release: manifest.release }, manifest), true)
|
||||
assert.equal(isActivatedRelease({ image: 'sha256:other-image', release: manifest.release }, manifest), false)
|
||||
assert.equal(isActivatedRelease({ image: manifest.api_image_id, release: null }, manifest), false)
|
||||
})
|
||||
|
||||
function emptySnapshot() {
|
||||
return { workbench: 0, pending_ops: 0,
|
||||
queues: Object.fromEntries(nativeQueues.map(name => [name,
|
||||
Object.fromEntries(queueStates.map(state => [state, 0]))])) }
|
||||
}
|
||||
|
||||
test('permits idle runtime while retaining archived and completed task history', () => {
|
||||
const snapshot = emptySnapshot()
|
||||
Object.assign(snapshot.queues.graph, { archived: 10, completed: 16 })
|
||||
assert.doesNotThrow(() => assertIdleSnapshot(snapshot))
|
||||
})
|
||||
|
||||
test('blocks persisted Wiki operations even when every Redis queue is empty', () => {
|
||||
const snapshot = emptySnapshot()
|
||||
snapshot.pending_ops = 1
|
||||
assert.throws(() => assertIdleSnapshot(snapshot), /persisted background/)
|
||||
})
|
||||
|
||||
test('blocks workbench jobs, uploads or reserved balances', () => {
|
||||
const snapshot = emptySnapshot()
|
||||
snapshot.workbench = 1
|
||||
assert.throws(() => assertIdleSnapshot(snapshot), /jobs, uploads or reserved/)
|
||||
})
|
||||
|
||||
for (const queue of nativeQueues) {
|
||||
for (const state of queueStates) {
|
||||
test(`blocks native ${queue} work in ${state} state`, () => {
|
||||
const snapshot = emptySnapshot()
|
||||
snapshot.queues[queue][state] = 1
|
||||
assert.throws(() => assertIdleSnapshot(snapshot), /active or waiting/)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
test('blocks newly discovered queues instead of ignoring future worker types', () => {
|
||||
const snapshot = emptySnapshot()
|
||||
snapshot.queues.future_worker = { ...snapshot.queues.graph, active: 1 }
|
||||
assert.throws(() => assertIdleSnapshot(snapshot), /active or waiting/)
|
||||
})
|
||||
|
||||
test('fails closed on missing queues and invalid count shapes', () => {
|
||||
const snapshot = emptySnapshot()
|
||||
delete snapshot.queues.graph
|
||||
assert.throws(() => assertIdleSnapshot(snapshot), /Cannot verify/)
|
||||
for (const count of [undefined, null, '', '0', -1, NaN, Infinity, 0.1, Number.MAX_SAFE_INTEGER + 1]) {
|
||||
const item = emptySnapshot()
|
||||
item.queues.wiki.active = count
|
||||
assert.throws(() => assertIdleSnapshot(item), /Cannot verify/)
|
||||
assert.throws(() => assertIdleSnapshot({ ...emptySnapshot(), pending_ops: count }), /Cannot verify/)
|
||||
}
|
||||
})
|
||||
|
||||
function fakeRuntime(overrides = {}, transformRedis = value => value) {
|
||||
const runtime = {
|
||||
DB_NAME: 'yawan_native_acceptance_20260915', REDIS_ADDR: 'actual-redis:6379',
|
||||
REDIS_DB: '14', REDIS_USERNAME: 'worker', REDIS_PASSWORD: 'private-test-password', ...overrides,
|
||||
}
|
||||
const calls = []
|
||||
const run = (binary, args, options) => {
|
||||
calls.push({ binary, args, options })
|
||||
if (args.includes('psql')) {
|
||||
assert.match(args.at(-1), /count\(\*\) FROM task_pending_ops/)
|
||||
return JSON.stringify({ workbench: 0, pending_ops: 0 })
|
||||
}
|
||||
if (args.includes('inspect')) return JSON.stringify(Object.entries(runtime).map(([key, value]) => `${key}=${value}`))
|
||||
assert.ok(args.includes('redis-cli'))
|
||||
return transformRedis(JSON.stringify(emptySnapshot().queues))
|
||||
}
|
||||
return { run, calls }
|
||||
}
|
||||
|
||||
test('reads actual Redis identity and DB with no password in argv or parent env', () => {
|
||||
const previous = process.env.REDISCLI_AUTH
|
||||
const { run, calls } = fakeRuntime()
|
||||
assert.doesNotThrow(() => idle(run))
|
||||
const command = calls.at(-1)
|
||||
assert.equal(command.binary, 'docker')
|
||||
assert.ok(command.args.includes('EVAL_RO'))
|
||||
assert.equal(command.args[command.args.indexOf('-h') + 1], 'actual-redis')
|
||||
assert.equal(command.args[command.args.indexOf('-n') + 1], '14')
|
||||
assert.equal(command.args[command.args.indexOf('--user') + 1], 'worker')
|
||||
assert.equal(command.options.env.REDISCLI_AUTH, 'private-test-password')
|
||||
assert.equal(process.env.REDISCLI_AUTH, previous)
|
||||
assert.ok(!command.args.join(' ').includes('private-test-password'))
|
||||
assert.ok(calls.every(call => call.options.timeout === 15000))
|
||||
})
|
||||
|
||||
test('preserves runtime TLS verification and supports IPv6 Redis addresses', () => {
|
||||
const { run, calls } = fakeRuntime({ REDIS_ADDR: '[::1]:6380', REDIS_USE_TLS: 'true', REDIS_TLS_SERVER_NAME: 'redis.example.test' })
|
||||
idle(run)
|
||||
const args = calls.at(-1).args
|
||||
assert.equal(args[args.indexOf('-h') + 1], '::1')
|
||||
assert.equal(args[args.indexOf('-p') + 1], '6380')
|
||||
assert.ok(args.includes('--tls'))
|
||||
assert.equal(args[args.indexOf('--sni') + 1], 'redis.example.test')
|
||||
assert.ok(!args.includes('--insecure'))
|
||||
})
|
||||
|
||||
test('does not publish when Redis auth, transport or response validation fails', () => {
|
||||
for (const transform of [() => 'NOAUTH private-test-password', () => '{invalid', () => { throw new Error('connection failed private-test-password') }]) {
|
||||
const { run } = fakeRuntime({}, transform)
|
||||
assert.throws(() => idle(run), error => {
|
||||
assert.match(error.message, /publication stopped/)
|
||||
assert.ok(!error.message.includes('private-test-password'))
|
||||
return true
|
||||
})
|
||||
}
|
||||
for (const overrides of [{ REDIS_ADDR: '' }, { REDIS_DB: '' }, { REDIS_DB: 'broken' }, { DB_NAME: 'old_database' }]) {
|
||||
assert.throws(() => idle(fakeRuntime(overrides).run), /publication stopped/)
|
||||
}
|
||||
})
|
||||
|
||||
test('minimum queue coverage follows the native Go worker topology', () => {
|
||||
const source = fs.readFileSync(path.join(__dirname, '../internal/types/task.go'), 'utf8')
|
||||
const names = [...source.matchAll(/^\s*Queue\w+\s*=\s*"([^"]+)"/gm)].map(match => match[1])
|
||||
assert.deepEqual([...nativeQueues].sort(), names.sort())
|
||||
})
|
||||
|
||||
const clock = Date.parse('2026-09-19T18:00:00.000Z')
|
||||
function publicationFixture() {
|
||||
return {
|
||||
manifest: {
|
||||
url: 'https://agent.pducv.cn', activated_at: '2026-09-19T14:29:38.265Z',
|
||||
api_activated_at: '2026-09-19T14:51:44.349Z',
|
||||
frontend_built_at: '2026-09-19T15:18:58.094Z',
|
||||
frontend_activated_at: '2026-09-19T15:22:00.000Z',
|
||||
},
|
||||
report: {
|
||||
started_at: '2026-09-19T15:23:00.000Z', checked_at: '2026-09-19T15:24:18.110Z',
|
||||
origin: 'https://agent.pducv.cn', api_origin: 'https://agent.pducv.cn', checks: [{passed: true}], failures: [], passed: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
const verifyPublication = (manifest, report) => assertPublicVerification(manifest, report, 'fixture-report', clock)
|
||||
|
||||
for (const name of ['public-navigation', 'public-access-control']) {
|
||||
test(`${name} requires proof its API checks use the public origin`, () => {
|
||||
const {manifest, report} = publicationFixture()
|
||||
assert.doesNotThrow(() => assertPublicVerification(manifest, report, name, clock))
|
||||
for (const api_origin of [undefined, '', 'invalid', 'http://127.0.0.1:18080', 'https://another.example']) {
|
||||
assert.throws(() => assertPublicVerification(manifest, {...report, api_origin}, name, clock), /Wrong verification API origin/)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
test('accepts complete verification begun after all runtime changes with either completion timestamp field', () => {
|
||||
const {manifest, report} = publicationFixture()
|
||||
assert.doesNotThrow(() => verifyPublication(manifest, report))
|
||||
const legacyCompletion = {...report, time: report.checked_at, base: report.origin + '/api/v1'}
|
||||
delete legacyCompletion.checked_at
|
||||
delete legacyCompletion.origin
|
||||
assert.doesNotThrow(() => verifyPublication(manifest, legacyCompletion))
|
||||
})
|
||||
|
||||
test('rejects checks begun before initial, API or UI activation even when completed afterward', () => {
|
||||
const {manifest, report} = publicationFixture()
|
||||
for (const field of ['activated_at', 'api_activated_at', 'frontend_activated_at']) {
|
||||
const start = new Date(Date.parse(manifest[field]) - 1).toISOString()
|
||||
assert.throws(() => verifyPublication(manifest, {...report, started_at: start}), /started before this runtime/)
|
||||
}
|
||||
// Building a new image does not mean it has replaced the served frontend.
|
||||
assert.throws(() => verifyPublication(manifest, {...report, started_at: '2026-09-19T15:20:00.000Z'}), /started before this runtime/)
|
||||
})
|
||||
|
||||
test('initial activation needs no fabricated API or UI refresh timestamps', () => {
|
||||
const {manifest, report} = publicationFixture()
|
||||
const initial = {url: manifest.url, activated_at: manifest.activated_at}
|
||||
assert.doesNotThrow(() => verifyPublication(initial, report))
|
||||
assert.throws(() => verifyPublication({...initial, previous_apis: [{}]}, report), /Missing API/)
|
||||
assert.throws(() => verifyPublication({...initial, previous_frontends: [{}]}, report), /Invalid publication timestamp/)
|
||||
})
|
||||
|
||||
test('requires actual UI activation after every frontend rebuild', () => {
|
||||
const {manifest, report} = publicationFixture()
|
||||
const missingActivation = {...manifest}
|
||||
delete missingActivation.frontend_activated_at
|
||||
assert.throws(() => verifyPublication(missingActivation, report), /frontend_activated_at/)
|
||||
const missingBuild = {...manifest}
|
||||
delete missingBuild.frontend_built_at
|
||||
assert.throws(() => verifyPublication(missingBuild, report), /frontend_built_at/)
|
||||
assert.throws(() => verifyPublication({...manifest, frontend_activated_at: '2026-09-19T15:00:00.000Z'}, report), /activation predates its build/)
|
||||
})
|
||||
|
||||
test('fails closed for every invalid manifest timestamp instead of comparing against NaN', () => {
|
||||
const {manifest, report} = publicationFixture()
|
||||
const invalid = [undefined, null, '', false, 0, 'invalid', '2026-02-30T00:00:00.000Z', '2026-09-20T00:00:00.000Z']
|
||||
for (const field of ['activated_at', 'api_activated_at', 'frontend_built_at', 'frontend_activated_at']) {
|
||||
for (const value of invalid) {
|
||||
assert.throws(() => verifyPublication({...manifest, [field]: value}, report), /Invalid publication timestamp/)
|
||||
}
|
||||
}
|
||||
assert.throws(() => verifyPublication({url: manifest.url}, report), /activated_at/)
|
||||
assert.throws(() => verifyPublication({...manifest, api_activated_at: '2026-09-19T13:00:00.000Z'}, report), /predates activation/)
|
||||
})
|
||||
|
||||
test('rejects missing, invalid, future and reversed report timing rather than using completion as start', () => {
|
||||
const {manifest, report} = publicationFixture()
|
||||
for (const field of ['started_at', 'checked_at']) {
|
||||
for (const value of [undefined, null, '', false, 0, 'invalid', '2026-02-30T00:00:00.000Z', '2026-09-20T00:00:00.000Z']) {
|
||||
assert.throws(() => verifyPublication(manifest, {...report, [field]: value}), /Invalid publication timestamp/)
|
||||
}
|
||||
}
|
||||
assert.throws(() => verifyPublication(manifest, {...report, checked_at: '2026-09-19T15:22:30.000Z'}), /finished before it started/)
|
||||
})
|
||||
|
||||
test('rejects truthy pass flags, empty checks, individual failures and a different origin', () => {
|
||||
const {manifest, report} = publicationFixture()
|
||||
for (const change of [{passed: 'true'}, {checks: []}, {checks: [{passed: false}]}, {checks: [{passed: 'true'}]},
|
||||
{failures: ['cleanup failed']}, {failures: undefined}]) {
|
||||
assert.throws(() => verifyPublication(manifest, {...report, ...change}), /did not pass/)
|
||||
}
|
||||
assert.throws(() => verifyPublication(manifest, {...report, origin: 'http://127.0.0.1:18088'}), /Wrong verification origin/)
|
||||
})
|
||||
|
||||
test('rejects failed, malformed or contradictory cleanup results even when the top-level report passed', () => {
|
||||
const {manifest, report} = publicationFixture()
|
||||
for (const cleanup of [undefined, null, {}, false, 'passed', [], [null], [true],
|
||||
[{name: 'knowledge fixture', passed: false}], [{name: 'knowledge fixture', passed: 'true'}],
|
||||
[{name: '', passed: true}], [{passed: true}], [{name: 'knowledge fixture', passed: true, error: 'cleanup failed'}]]) {
|
||||
assert.throws(() => verifyPublication(manifest, {...report, cleanup}), /cleanup did not pass/)
|
||||
}
|
||||
assert.doesNotThrow(() => verifyPublication(manifest, {...report, cleanup: [{name: 'knowledge fixture', passed: true}]}))
|
||||
})
|
||||
|
||||
function reconciliationFixture() {
|
||||
const {manifest, report} = publicationFixture()
|
||||
Object.assign(manifest, {release: 'public-20260919-223000', stage: 'origin-ready',
|
||||
frontend_built_at: '2026-09-19T16:50:23.481Z', frontend_activated_at: '2026-09-19T16:50:27.512Z'})
|
||||
const reports = Object.fromEntries(publicVerificationReports.map((name, index) => [name, {...structuredClone(report),
|
||||
started_at: `2026-09-19T17:0${index}:00.000Z`, checked_at: `2026-09-19T17:0${index}:30.000Z` }]))
|
||||
reports['public-resource-files'].prefix = 'file-isolation-1758300000000'
|
||||
const indexed = reports['public-indexed-retrieval-isolation']
|
||||
indexed.fixtures = {kbs: ['00000000-0000-4000-8000-000000000001']}
|
||||
indexed.cleanup = indexed.fixtures.kbs.map(id => ({name: `knowledge ${id}`, passed: true}))
|
||||
const reconciliation = {release: manifest.release, origin: manifest.url, read_only: true,
|
||||
checked_at: '2026-09-19T17:07:00.000Z', passed: true, failures: [],
|
||||
reports: Object.fromEntries(publicVerificationReports.map(name => [name, {
|
||||
started_at: reports[name].started_at, completed_at: reports[name].checked_at,
|
||||
}])), checks: [...fixtureReconciliationCounts.map(name => ({name, count: 0, passed: true})), {
|
||||
name: 'file_fixture_directory_removed', passed: true,
|
||||
path: path.resolve(__dirname, '../../acceptance/files/10059/exports/file-isolation-1758300000000'),
|
||||
}]}
|
||||
return {manifest, reports, reconciliation}
|
||||
}
|
||||
|
||||
const invalidIndexedCleanup = [
|
||||
['missing cleanup', report => {delete report.cleanup}, /Incomplete indexed knowledge cleanup/],
|
||||
['empty cleanup', report => {report.cleanup = []}, /cleanup did not pass/],
|
||||
['only account cleanup', report => {report.cleanup = [{name: 'user fixture', passed: true}]}, /Incomplete indexed knowledge cleanup/],
|
||||
['mismatched knowledge cleanup', report => {report.cleanup[0].name = 'knowledge 00000000-0000-4000-8000-000000000002'}, /Incomplete indexed knowledge cleanup/],
|
||||
['duplicate knowledge cleanup', report => {report.cleanup.push({...report.cleanup[0]})}, /Incomplete indexed knowledge cleanup/],
|
||||
['unmatched extra knowledge cleanup', report => {report.cleanup.push({name: 'knowledge 00000000-0000-4000-8000-000000000002', passed: true})}, /Incomplete indexed knowledge cleanup/],
|
||||
['second knowledge fixture without cleanup', report => {report.fixtures.kbs.push('00000000-0000-4000-8000-000000000002')}, /Incomplete indexed knowledge cleanup/],
|
||||
['failed knowledge cleanup', report => {report.cleanup[0].passed = false}, /cleanup did not pass/],
|
||||
['missing fixtures', report => {delete report.fixtures}, /Invalid indexed knowledge fixtures/],
|
||||
['missing knowledge fixtures', report => {delete report.fixtures.kbs}, /Invalid indexed knowledge fixtures/],
|
||||
['empty knowledge fixtures', report => {report.fixtures.kbs = []}, /Invalid indexed knowledge fixtures/],
|
||||
['invalid knowledge fixture', report => {report.fixtures.kbs = ['fixture']; report.cleanup[0].name = 'knowledge fixture'}, /Invalid indexed knowledge fixtures/],
|
||||
['duplicate knowledge fixtures', report => {report.fixtures.kbs.push(report.fixtures.kbs[0])}, /Invalid indexed knowledge fixtures/],
|
||||
]
|
||||
|
||||
for (const [label, mutate, pattern] of invalidIndexedCleanup) {
|
||||
test(`blocks final publication for indexed retrieval with ${label}`, async () => {
|
||||
const value = reconciliationFixture()
|
||||
mutate(value.reports['public-indexed-retrieval-isolation'])
|
||||
assert.throws(() => assertFixtureReconciliation(value.manifest, value.reconciliation, value.reports, clock), pattern)
|
||||
const publisher = injectedPublisher(value)
|
||||
await assert.rejects(publisher.publish(), pattern)
|
||||
assert.deepEqual(publisher.writes, [])
|
||||
})
|
||||
}
|
||||
|
||||
test('indexed retrieval accepts every knowledge cleanup alongside account and role cleanup', () => {
|
||||
const {manifest, reports, reconciliation} = reconciliationFixture()
|
||||
const indexed = reports['public-indexed-retrieval-isolation']
|
||||
indexed.fixtures.kbs.push('00000000-0000-4000-8000-000000000002')
|
||||
indexed.cleanup.push({name: `knowledge ${indexed.fixtures.kbs[1]}`, passed: true},
|
||||
{name: 'user fixture', passed: true}, {name: 'role fixture', passed: true})
|
||||
assert.doesNotThrow(() => assertFixtureReconciliation(manifest, reconciliation, reports, clock))
|
||||
})
|
||||
|
||||
test('accepts complete zero-residue reconciliation tied to all seven reports after the latest activation', () => {
|
||||
const {manifest, reports, reconciliation} = reconciliationFixture()
|
||||
assert.doesNotThrow(() => assertFixtureReconciliation(manifest, reconciliation, reports, clock))
|
||||
const indexed = reports['public-indexed-retrieval-isolation']
|
||||
indexed.time = indexed.checked_at
|
||||
delete indexed.checked_at
|
||||
assert.doesNotThrow(() => assertFixtureReconciliation(manifest, reconciliation, reports, clock))
|
||||
})
|
||||
|
||||
for (const name of publicVerificationReports) {
|
||||
test(`rejects reconciliation that refers to a previous execution of ${name}`, () => {
|
||||
for (const field of ['started_at', 'completed_at']) {
|
||||
const {manifest, reports, reconciliation} = reconciliationFixture()
|
||||
reconciliation.reports[name][field] = '2026-09-19T16:55:00.000Z'
|
||||
assert.throws(() => assertFixtureReconciliation(manifest, reconciliation, reports, clock), /does not match current report/)
|
||||
}
|
||||
})
|
||||
test(`rejects ${name} started before the latest activation even with matching reconciliation metadata`, () => {
|
||||
const {manifest, reports, reconciliation} = reconciliationFixture()
|
||||
reports[name].started_at = '2026-09-19T16:50:27.511Z'
|
||||
reconciliation.reports[name].started_at = reports[name].started_at
|
||||
assert.throws(() => assertFixtureReconciliation(manifest, reconciliation, reports, clock), /started before this runtime/)
|
||||
})
|
||||
}
|
||||
|
||||
for (const [label, mutate, pattern] of [
|
||||
['different release', value => {value.reconciliation.release = 'public-20260919-000000'}, /identity/],
|
||||
['different origin', value => {value.reconciliation.origin = 'http://127.0.0.1:18088'}, /identity/],
|
||||
['non-read-only result', value => {value.reconciliation.read_only = false}, /identity/],
|
||||
['failed result', value => {value.reconciliation.passed = false}, /did not pass/],
|
||||
['reported failure', value => {value.reconciliation.failures.push('residual rows')}, /did not pass/],
|
||||
['future check', value => {value.reconciliation.checked_at = '2026-09-20T00:00:00.000Z'}, /Invalid publication timestamp/],
|
||||
['missing check time', value => {delete value.reconciliation.checked_at}, /Invalid publication timestamp/],
|
||||
['check before final source completes', value => {value.reconciliation.checked_at = '2026-09-19T17:06:29.999Z'}, /predates source report/],
|
||||
['missing source metadata', value => {delete value.reconciliation.reports['public-navigation']}, /all seven/],
|
||||
['renamed source metadata', value => {value.reconciliation.reports.wrong = value.reconciliation.reports['public-navigation']; delete value.reconciliation.reports['public-navigation']}, /does not match/],
|
||||
['extra source metadata', value => {value.reconciliation.reports.extra = {}}, /all seven/],
|
||||
['missing source report', value => {delete value.reports['public-navigation']}, /did not pass/],
|
||||
['failed source report', value => {value.reports['public-navigation'].passed = false}, /did not pass/],
|
||||
['failed source cleanup', value => {value.reports['public-indexed-retrieval-isolation'].cleanup[0].passed = false}, /cleanup did not pass/],
|
||||
['missing residual check', value => {value.reconciliation.checks.pop()}, /missing required residual checks/],
|
||||
['duplicate residual check', value => {value.reconciliation.checks[1] = value.reconciliation.checks[0]}, /missing required residual checks/],
|
||||
['unknown residual check', value => {value.reconciliation.checks[0].name = 'unrelated'}, /missing required residual checks/],
|
||||
['invalid numeric count', value => {value.reconciliation.checks[0].count = '0'}, /residual rows/],
|
||||
['positive count with passed true', value => {value.reconciliation.checks[0].count = 1}, /residual rows/],
|
||||
['missing count', value => {delete value.reconciliation.checks[0].count}, /residual rows/],
|
||||
['wrong directory', value => {value.reconciliation.checks.at(-1).path = '/unrelated/path'}, /wrong attachment directory/],
|
||||
['missing directory', value => {delete value.reconciliation.checks.at(-1).path}, /wrong attachment directory/],
|
||||
['unsafe file prefix', value => {value.reports['public-resource-files'].prefix = '../../private'}, /wrong attachment directory/],
|
||||
]) {
|
||||
test(`blocks publication for ${label}`, () => {
|
||||
const value = reconciliationFixture()
|
||||
mutate(value)
|
||||
assert.throws(() => assertFixtureReconciliation(value.manifest, value.reconciliation, value.reports, clock), pattern)
|
||||
})
|
||||
}
|
||||
|
||||
function injectedPublisher(value, missingReconciliation = false) {
|
||||
const writes = [], reads = []
|
||||
const module = {exports: {}}
|
||||
const injectedRequire = name => {
|
||||
if (name === 'node:fs') return {
|
||||
readFileSync: file => {
|
||||
reads.push(path.basename(file))
|
||||
if (file.endsWith('/manifest.json')) return JSON.stringify(value.manifest)
|
||||
if (file.endsWith('/public-fixture-reconciliation.json')) {
|
||||
if (missingReconciliation) throw new Error('Missing reconciliation file')
|
||||
return JSON.stringify(value.reconciliation)
|
||||
}
|
||||
const report = value.reports[path.basename(file, '.json')]
|
||||
assert.ok(report, 'Unexpected file read')
|
||||
return JSON.stringify(report)
|
||||
},
|
||||
writeFileSync: (file, contents) => writes.push({file, value: JSON.parse(contents)}),
|
||||
}
|
||||
if (name === 'node:child_process') return {execFileSync: () => assert.fail('External command forbidden')}
|
||||
return require(name)
|
||||
}
|
||||
class FrozenDate extends Date {
|
||||
constructor(...args) { super(...(args.length ? args : [clock])) }
|
||||
static now() { return clock }
|
||||
}
|
||||
const context = {require: injectedRequire, module, __dirname, URL, AbortSignal, Date: FrozenDate,
|
||||
fetch: async () => ({ok: true, json: async () => ({release: value.manifest.release})}), console: {log() {}}}
|
||||
vm.runInNewContext(fs.readFileSync(path.join(__dirname, 'publish_public_runtime.cjs'), 'utf8')
|
||||
+ '\n;globalThis.publish = markPublished;', context)
|
||||
return {writes, reads, publish: () => context.publish(value.manifest.release)}
|
||||
}
|
||||
|
||||
test('mark-published cannot write either manifest when reconciliation is absent or failed', async () => {
|
||||
for (const absent of [true, false]) {
|
||||
const value = reconciliationFixture()
|
||||
value.reconciliation.passed = false
|
||||
const publisher = injectedPublisher(value, absent)
|
||||
await assert.rejects(publisher.publish(), /Missing reconciliation file|did not pass/)
|
||||
assert.deepEqual(publisher.writes, [])
|
||||
assert.ok(publisher.reads.includes('public-fixture-reconciliation.json'))
|
||||
}
|
||||
})
|
||||
|
||||
test('mark-published persists only after current reports and their final reconciliation pass', async () => {
|
||||
const value = reconciliationFixture()
|
||||
const publisher = injectedPublisher(value)
|
||||
await publisher.publish()
|
||||
assert.equal(publisher.writes.length, 2)
|
||||
assert.ok(publisher.reads.includes('public-fixture-reconciliation.json'))
|
||||
for (const write of publisher.writes) {
|
||||
assert.equal(write.value.stage, 'published')
|
||||
assert.equal(write.value.fixture_reconciliation_checked_at, value.reconciliation.checked_at)
|
||||
assert.deepEqual(write.value.public_verification_reports, publicVerificationReports)
|
||||
}
|
||||
})
|
||||
|
||||
test('every required producer captures execution start before requests and writes it in its report', () => {
|
||||
for (const name of ['verify_public_release', 'verify_navigation_parity', 'verify_access_control',
|
||||
'verify_resource_isolation_live', 'verify_resource_files_live', 'verify_public_tenant_isolation',
|
||||
'verify_indexed_retrieval_isolation']) {
|
||||
const source = fs.readFileSync(path.join(__dirname, `${name}.cjs`), 'utf8')
|
||||
const start = source.indexOf('const startedAt = new Date().toISOString()')
|
||||
assert.ok(start >= 0 && start < source.indexOf('fetch('), `${name}: capture start before requests`)
|
||||
assert.match(source, /started_at:\s*startedAt/, `${name}: persist captured start`)
|
||||
}
|
||||
})
|
||||
Executable
+123
@@ -0,0 +1,123 @@
|
||||
#!/bin/bash
|
||||
# 快速启动开发环境的一键脚本
|
||||
# 此脚本会在一个终端中启动所有必需的服务
|
||||
|
||||
# 设置颜色
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # 无颜色
|
||||
|
||||
# 获取项目根目录
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
|
||||
|
||||
log_info() {
|
||||
printf "%b\n" "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
printf "%b\n" "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
printf "%b\n" "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
printf "%b\n" "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
echo ""
|
||||
printf "%b\n" "${GREEN}========================================${NC}"
|
||||
printf "%b\n" "${GREEN} WeKnora 快速开发环境启动${NC}"
|
||||
printf "%b\n" "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# 检查是否在项目根目录
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
# 1. 启动基础设施
|
||||
log_info "步骤 1/3: 启动基础设施服务..."
|
||||
./scripts/dev.sh start
|
||||
if [ $? -ne 0 ]; then
|
||||
log_error "基础设施启动失败"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 等待服务就绪
|
||||
log_info "等待服务启动完成..."
|
||||
sleep 5
|
||||
|
||||
# 2. 询问是否启动后端
|
||||
echo ""
|
||||
log_info "步骤 2/3: 启动后端应用"
|
||||
printf "%b" "${YELLOW}是否在当前终端启动后端? (y/N): ${NC}"
|
||||
read -r start_backend
|
||||
|
||||
if [ "$start_backend" = "y" ] || [ "$start_backend" = "Y" ]; then
|
||||
log_info "启动后端..."
|
||||
# 在后台启动后端
|
||||
nohup bash -c 'cd "'$PROJECT_ROOT'" && ./scripts/dev.sh app' > "$PROJECT_ROOT/logs/backend.log" 2>&1 &
|
||||
BACKEND_PID=$!
|
||||
echo $BACKEND_PID > "$PROJECT_ROOT/tmp/backend.pid"
|
||||
log_success "后端已在后台启动 (PID: $BACKEND_PID)"
|
||||
log_info "查看后端日志: tail -f $PROJECT_ROOT/logs/backend.log"
|
||||
else
|
||||
log_warning "跳过后端启动"
|
||||
log_info "稍后在新终端运行: make dev-app 或 ./scripts/dev.sh app"
|
||||
fi
|
||||
|
||||
# 3. 询问是否启动前端
|
||||
echo ""
|
||||
log_info "步骤 3/3: 启动前端应用"
|
||||
printf "%b" "${YELLOW}是否在当前终端启动前端? (y/N): ${NC}"
|
||||
read -r start_frontend
|
||||
|
||||
if [ "$start_frontend" = "y" ] || [ "$start_frontend" = "Y" ]; then
|
||||
log_info "启动前端..."
|
||||
# 在后台启动前端
|
||||
nohup bash -c 'cd "'$PROJECT_ROOT'/frontend" && npm run dev' > "$PROJECT_ROOT/logs/frontend.log" 2>&1 &
|
||||
FRONTEND_PID=$!
|
||||
echo $FRONTEND_PID > "$PROJECT_ROOT/tmp/frontend.pid"
|
||||
log_success "前端已在后台启动 (PID: $FRONTEND_PID)"
|
||||
log_info "查看前端日志: tail -f $PROJECT_ROOT/logs/frontend.log"
|
||||
else
|
||||
log_warning "跳过前端启动"
|
||||
log_info "稍后在新终端运行: make dev-frontend 或 ./scripts/dev.sh frontend"
|
||||
fi
|
||||
|
||||
# 显示总结
|
||||
echo ""
|
||||
printf "%b\n" "${GREEN}========================================${NC}"
|
||||
printf "%b\n" "${GREEN} 启动完成!${NC}"
|
||||
printf "%b\n" "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
log_info "访问地址:"
|
||||
echo " - 前端: http://localhost:5173"
|
||||
echo " - 后端 API: http://localhost:8080"
|
||||
echo " - MinIO Console: http://localhost:9001"
|
||||
echo ""
|
||||
|
||||
log_info "管理命令:"
|
||||
echo " - 查看服务状态: make dev-status"
|
||||
echo " - 查看日志: make dev-logs"
|
||||
echo " - 停止所有服务: make dev-stop"
|
||||
echo ""
|
||||
|
||||
if [ -f "$PROJECT_ROOT/tmp/backend.pid" ] || [ -f "$PROJECT_ROOT/tmp/frontend.pid" ]; then
|
||||
log_warning "停止后台进程:"
|
||||
if [ -f "$PROJECT_ROOT/tmp/backend.pid" ]; then
|
||||
echo " - 停止后端: kill \$(cat $PROJECT_ROOT/tmp/backend.pid)"
|
||||
fi
|
||||
if [ -f "$PROJECT_ROOT/tmp/frontend.pid" ]; then
|
||||
echo " - 停止前端: kill \$(cat $PROJECT_ROOT/tmp/frontend.pid)"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
log_success "开发环境已就绪,开始编码吧!"
|
||||
echo ""
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const assert = require('node:assert/strict')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
const root = path.resolve(__dirname, '../..')
|
||||
const docker = (...args) => execFileSync('docker', ['--context', 'colima-yawan', ...args], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }).trim()
|
||||
const allIDs = ['16c3e59c-4bb6-41cf-ae0a-6d0941b3d032', '20b9280e-ed73-4700-8fa5-944d6ae6dd5f', 'bb86b44a-576b-4bf8-a6bf-8f1baca3a845', 'badd084b-f229-4005-8fb3-196fbe139fe0', '5d28a7e5-e00e-44a7-a0e1-c72342cdcf68', '3ae9257d-928a-4886-8101-7cfd0ed8c27c']
|
||||
const args = process.argv.slice(2)
|
||||
const revisionArg = args.find(arg => arg.startsWith('--backup-revision='))
|
||||
const chunksArg = args.find(arg => arg.startsWith('--chunks='))
|
||||
const chunks = chunksArg ? [...new Set(chunksArg.slice('--chunks='.length).split(','))] : []
|
||||
assert.ok(chunks.length <= 51, 'At most 51 explicitly reviewed chunks may be rebuilt')
|
||||
for (const chunk of chunks) assert.match(chunk, /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/)
|
||||
assert.ok(!chunksArg || (chunks.length > 0 && args.includes('--product-only')), 'Targeted repair requires product-only scope')
|
||||
const revision = revisionArg?.slice('--backup-revision='.length) || '20260919'
|
||||
assert.match(revision, /^[a-z0-9_]{1,32}$/, 'Backup revision must be an SQL-safe identifier')
|
||||
assert.ok(args.every(arg => ['--apply', '--product-only'].includes(arg) || arg === revisionArg || arg === chunksArg), 'Unknown repair option')
|
||||
const ids = args.includes('--product-only') ? allIDs.slice(0, 1) : allIDs
|
||||
const recordsBackup = `native_graph_grounding_backup_${revision}`
|
||||
const sourcesBackup = `native_graph_source_grounding_backup_${revision}`
|
||||
const scope = `tenant_id=10059 AND knowledge_base_id IN (${ids.map(id => `'${id}'`).join(',')})`
|
||||
const chunkValues = chunks.map(id => `'${id}'`).join(',')
|
||||
const recordFilter = chunks.length ? ` AND (source_table='knowledgegraphchunkstate' AND payload->>'native_chunk_id' IN (${chunkValues}) OR source_table='knowledgegraphevidence' AND payload->>'source_chunk_id' IN (${chunks.map(id => `'native:${id}'`).join(',')}))` : ''
|
||||
const sourceFilter = chunks.length ? ` AND native_chunk_id IN (${chunkValues})` : ''
|
||||
const sql = query => docker('exec', 'WeKnora-postgres', 'psql', '-XAt', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-c', query)
|
||||
function queueStatus() {
|
||||
const inspected = JSON.parse(docker('inspect', 'yawan-native-acceptance'))[0]
|
||||
const env = Object.fromEntries(inspected.Config.Env.map(value => { const i = value.indexOf('='); return [value.slice(0, i), value.slice(i + 1)] }))
|
||||
const previous = process.env.REDISCLI_AUTH
|
||||
process.env.REDISCLI_AUTH = env.REDIS_PASSWORD || ''
|
||||
try {
|
||||
const result = {}
|
||||
for (const [state, command] of [['active', 'LLEN'], ['pending', 'LLEN'], ['scheduled', 'ZCARD'], ['retry', 'ZCARD']]) {
|
||||
const value = docker('exec', '-e', 'REDISCLI_AUTH', 'WeKnora-redis', 'redis-cli', '-n', env.REDIS_DB || '0', command, `asynq:{graph}:${state}`)
|
||||
assert.match(value, /^\d+$/)
|
||||
result[state] = Number(value)
|
||||
}
|
||||
return result
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.REDISCLI_AUTH
|
||||
else process.env.REDISCLI_AUTH = previous
|
||||
}
|
||||
}
|
||||
function main() {
|
||||
const queue = queueStatus()
|
||||
const count = Number(sql(`SELECT count(*) FROM native_graph_records WHERE ${scope} AND payload->>'origin'='native_extraction_v1'`))
|
||||
if (chunks.length) {
|
||||
const matched = Number(sql(`SELECT count(*) FROM chunks WHERE ${scope} AND id IN (${chunkValues}) AND deleted_at IS NULL AND is_enabled AND chunk_type='text'`))
|
||||
assert.equal(matched, chunks.length, 'Every selected source must belong to the active product knowledge base')
|
||||
}
|
||||
const directCount = Number(sql(`SELECT count(*) FROM native_graph_records WHERE ${scope} AND payload->>'origin'='native_extraction_v1'${recordFilter}`))
|
||||
if (!process.argv.includes('--apply')) { console.log(JSON.stringify({ queue, derived_records_to_backup: count, direct_records_to_rebuild: directCount, selected_chunks: chunks })); return }
|
||||
assert.equal(docker('inspect', 'yawan-native-graph-recovery', '--format', '{{.State.Running}}'), 'false', 'Stop graph enqueuer before reset')
|
||||
assert.ok(Object.values(queue).every(n => n === 0), 'Wait for graph queue to drain before rebuilding')
|
||||
assert.ok(count > 0 && count < 20000, 'Unexpected graph repair scope')
|
||||
const backup = sql(`SELECT json_build_object('records', (SELECT json_agg(r) FROM native_graph_records r WHERE ${scope} AND payload->>'origin'='native_extraction_v1'), 'sources', (SELECT json_agg(s) FROM native_graph_sources s WHERE ${scope} AND mapping_state='native_extraction'))`)
|
||||
const backupPath = path.join(root, 'acceptance', `native-graph-before-grounding-${Date.now()}.json`)
|
||||
const parsed = JSON.parse(backup)
|
||||
assert.equal(parsed.records.length, count)
|
||||
fs.writeFileSync(backupPath, backup, { flag: 'wx', mode: 0o600 })
|
||||
// A transactional database copy also keeps rollback possible if the host disk disappears.
|
||||
sql(`BEGIN;
|
||||
${ids.map(id => `SELECT pg_advisory_xact_lock(hashtextextended('native-graph:10059:${id}',0));`).join('\n')}
|
||||
CREATE TABLE ${recordsBackup} AS SELECT * FROM native_graph_records WHERE ${scope} AND payload->>'origin'='native_extraction_v1';
|
||||
CREATE TABLE ${sourcesBackup} AS SELECT * FROM native_graph_sources WHERE ${scope} AND mapping_state='native_extraction';
|
||||
DO $$ BEGIN IF (SELECT count(*) FROM ${recordsBackup}) <> ${count} THEN RAISE EXCEPTION 'Graph changed after backup'; END IF; END $$;
|
||||
DELETE FROM native_graph_records WHERE ${scope} AND payload->>'origin'='native_extraction_v1'${recordFilter};
|
||||
DELETE FROM native_graph_sources WHERE ${scope} AND mapping_state='native_extraction'${sourceFilter};
|
||||
${chunks.length ? `DELETE FROM native_graph_records r WHERE ${scope} AND r.payload->>'origin'='native_extraction_v1'
|
||||
AND r.source_table IN ('knowledgeentity','knowledgerelation')
|
||||
AND NOT EXISTS (SELECT 1 FROM native_graph_records e WHERE e.tenant_id=r.tenant_id AND e.knowledge_base_id=r.knowledge_base_id
|
||||
AND e.source_table='knowledgegraphevidence' AND e.payload->>'subject_id'=r.legacy_id::text
|
||||
AND e.payload->>'subject_type'=CASE r.source_table WHEN 'knowledgeentity' THEN 'entity' ELSE 'relation' END);` : ''}
|
||||
COMMIT;`)
|
||||
const remaining = Number(sql(`SELECT count(*) FROM native_graph_records WHERE ${scope} AND payload->>'origin'='native_extraction_v1'`))
|
||||
console.log(JSON.stringify({ removed_derived_records: count - remaining, backed_up_sources: parsed.sources?.length || 0, selected_chunks: chunks, backup: backupPath, database_backups: [recordsBackup, sourcesBackup], source_documents_and_indexes_preserved: true }))
|
||||
}
|
||||
try { main() } catch (error) { console.error(error.message); process.exitCode = 1 }
|
||||
@@ -0,0 +1,301 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const assert = require('node:assert/strict')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
const { attemptOf, latestAttempts, planRecovery, validatePublishedRecoveryFix, validateFinalRecoveryTrigger, supportedReviewedAttempt, describeReadCoverage, durableRecoveryStatus, validateDurableRecoveryReview } = require('./wiki_recovery_policy.cjs')
|
||||
const { recoveryFingerprintsSQL } = require('./wiki_recovery_budget_sql.cjs')
|
||||
const { loadClosures, annotateClosed } = require('./wiki_recovery_closures.cjs')
|
||||
const { validateSixthDiagnosticEvidence, validateSixthReplayEvidence, verifyReplayInputFiles, replayPins, candidateSourceTreeHash, sixthGuardPausedOperation } = require('./wiki_sixth_recovery_review.cjs')
|
||||
const { seventhPins, validateSeventhCorrectionEvidence, verifySeventhBuildFiles, seventhRecoveryEnqueueSQL } = require('./wiki_seventh_recovery_review.cjs')
|
||||
const fingerprints = recoveryFingerprintsSQL()
|
||||
|
||||
const root = path.resolve(__dirname, '../..')
|
||||
const directory = path.join(root, 'acceptance/wiki-deadletter-recovery-20260919')
|
||||
const tenant = 10059
|
||||
const database = 'yawan_native_acceptance_20260915'
|
||||
const table = 'wiki_dead_letter_recoveries'
|
||||
// The service dead-letters at 6. Starting at 5 permits one new recovery
|
||||
// attempt, without repeating the historical six-attempt model-call burst.
|
||||
const recoveryInitialFailCount = 5
|
||||
const args = process.argv.slice(2)
|
||||
const apply = args.includes('--apply')
|
||||
const triggerOnly = args.includes('--trigger')
|
||||
const idsArg = args.find(arg => arg.startsWith('--ids='))?.slice(6)
|
||||
const ids = idsArg ? idsArg.split(',').map(Number) : []
|
||||
const attempt = Number(args.find(arg => arg.startsWith('--attempt='))?.slice(10) || 1)
|
||||
const retryReason = args.find(arg => arg.startsWith('--reason='))?.slice(9) || ''
|
||||
const reviewPath = args.find(arg => arg.startsWith('--review-file='))?.slice(14)
|
||||
assert.ok(ids.every(id => Number.isSafeInteger(id) && id >= 1 && id <= 33), 'Only original dead letters 1..33 are allowed')
|
||||
assert.equal(new Set(ids).size, ids.length, 'Duplicate IDs')
|
||||
assert.ok(!apply || ids.length === 1, 'Apply requires --ids= with exactly one original record')
|
||||
assert.ok(!args.some(arg => !['--apply', '--trigger'].includes(arg) && !['--ids=', '--attempt=', '--reason=', '--review-file='].some(prefix => arg.startsWith(prefix))), 'Unknown argument')
|
||||
assert.ok(attempt === 1 || attempt === 2 || (supportedReviewedAttempt(attempt) && reviewPath), 'Further bounded recovery requires an explicit deployed-fix review file')
|
||||
assert.ok(attempt === 1 || apply, 'Another attempt requires --apply')
|
||||
assert.ok(!apply || reviewPath, 'Every new recovery requires --review-file= with an explicit persistent request limit and deadline')
|
||||
assert.ok(!reviewPath || apply, 'Review files are only valid for a bounded manual recovery')
|
||||
const literal = value => `'${String(value).replaceAll("'", "''")}'`
|
||||
const docker = (args, input) => execFileSync('docker', ['--context', 'colima-yawan', ...args], {
|
||||
input, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], maxBuffer: 32 * 1024 * 1024,
|
||||
})
|
||||
function sql(statement, readOnly = true) {
|
||||
const args = ['exec', '-i']
|
||||
if (readOnly) args.push('-e', 'PGOPTIONS=-c default_transaction_read_only=on')
|
||||
return docker([...args, 'WeKnora-postgres', 'psql', '-X', '-q', '-U', 'postgres', '-d', database, '-At', '-v', 'ON_ERROR_STOP=1'], statement).trim()
|
||||
}
|
||||
const queryJSON = (statement, readOnly = true) => JSON.parse(sql(statement, readOnly))
|
||||
const hasLedger = () => sql(`SELECT to_regclass('public.${table}') IS NOT NULL`) === 't'
|
||||
const writePrivate = (name, value) => fs.writeFileSync(path.join(directory, name), JSON.stringify(value, null, 2) + '\n', { mode: 0o600, flag: 'wx' })
|
||||
const jsonArray = expression => `CASE WHEN jsonb_typeof(${expression})='array' THEN ${expression} ELSE '[]'::jsonb END`
|
||||
const hasSource = (page, knowledge) => `EXISTS (SELECT 1 FROM jsonb_array_elements_text(${jsonArray(`${page}.source_refs`)}) ref WHERE split_part(ref,'|',1)=${knowledge})`
|
||||
|
||||
function inventory(fingerprintIDs = ids.length ? ids : [10]) {
|
||||
return queryJSON(`SELECT coalesce(json_agg(row_to_json(x)), '[]') FROM (
|
||||
SELECT d.id,d.tenant_id,d.related_id AS knowledge_id,d.scope_id AS knowledge_base_id,k.name AS knowledge_base,
|
||||
k.wiki_config->>'synthesis_model_id' AS model_id,
|
||||
k.embedding_model_id,
|
||||
CASE WHEN d.id IN (${fingerprintIDs.join(',')}) THEN ${fingerprints.source} END AS source_fingerprint,
|
||||
CASE WHEN d.id IN (${fingerprintIDs.join(',')}) THEN ${fingerprints.model} END AS model_fingerprint,
|
||||
n.parse_status,n.pending_subtasks_count,
|
||||
(d.payload->>'op'='ingest' AND d.payload->>'knowledge_id'=d.related_id
|
||||
AND d.scope='knowledge_base' AND d.fail_count=6 AND n.deleted_at IS NULL
|
||||
AND k.deleted_at IS NULL AND k.indexing_strategy->>'wiki_enabled'='true'
|
||||
AND n.parse_status='completed' AND n.pending_subtasks_count=0
|
||||
AND EXISTS (SELECT 1 FROM models m WHERE m.id=k.wiki_config->>'synthesis_model_id' AND m.tenant_id=d.tenant_id AND m.type='KnowledgeQA' AND m.deleted_at IS NULL)) AS eligible,
|
||||
(SELECT count(*) FROM chunks c WHERE c.tenant_id=d.tenant_id AND c.knowledge_id=n.id AND c.deleted_at IS NULL) AS chunks,
|
||||
(SELECT coalesce(sum(length(c.content)),0) FROM chunks c WHERE c.tenant_id=d.tenant_id AND c.knowledge_id=n.id AND c.deleted_at IS NULL) AS chunk_chars,
|
||||
(SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=d.tenant_id AND p.knowledge_base_id=d.scope_id AND p.deleted_at IS NULL AND ${hasSource('p', 'd.related_id')}) AS pages,
|
||||
(SELECT count(*) FROM task_pending_ops o WHERE o.tenant_id=d.tenant_id AND o.scope_id=d.scope_id AND o.task_type='wiki:ingest' AND o.dedup_key=d.related_id) AS pending
|
||||
FROM task_dead_letters d
|
||||
JOIN knowledges n ON n.id=d.related_id AND n.tenant_id=d.tenant_id AND n.knowledge_base_id=d.scope_id
|
||||
JOIN knowledge_bases k ON k.id=d.scope_id AND k.tenant_id=d.tenant_id
|
||||
WHERE d.tenant_id=${tenant} AND d.task_type='wiki:ingest' AND d.id BETWEEN 1 AND 33
|
||||
ORDER BY chunk_chars,d.id
|
||||
) x`)
|
||||
}
|
||||
|
||||
function status() {
|
||||
if (!hasLedger()) return []
|
||||
const budgetsAvailable = sql("SELECT to_regclass('public.wiki_recovery_budgets') IS NOT NULL") === 't'
|
||||
const rows = queryJSON(`SELECT coalesce(json_agg(row_to_json(x)), '[]') FROM (
|
||||
SELECT r.*,coalesce((to_jsonb(r)->>'attempt_number')::int,1) AS attempt_number,o.claimed_at,o.fail_count,
|
||||
${budgetsAvailable ? '(SELECT row_to_json(b) FROM wiki_recovery_budgets b WHERE b.tenant_id=r.tenant_id AND b.dead_letter_id=r.dead_letter_id AND b.attempt_number=r.attempt_number AND b.pending_op_id=r.pending_op_id)' : 'NULL::json'} AS recovery_budget,
|
||||
s.status AS span_status,s.error_code,
|
||||
s.output->>'pages_written' AS pages_written,s.output->>'pages_dropped' AS pages_dropped,
|
||||
s.output->>'pages_total' AS pages_total,s.output->>'failed_slug_writes' AS failed_slug_writes,
|
||||
s.output->>'classify_batches' AS classify_batches,
|
||||
s.output->>'read_coverage_version' AS read_coverage_version,
|
||||
(s.output->>'source_read_complete')::boolean AS source_read_complete,
|
||||
(s.output->>'source_runes')::bigint AS source_runes,
|
||||
(s.output->>'source_windows')::int AS source_windows,
|
||||
coalesce((SELECT jsonb_object_agg(c.item_key,c.payload::jsonb) FROM wiki_ingest_checkpoints c
|
||||
WHERE c.tenant_id=r.tenant_id AND c.knowledge_base_id=r.knowledge_base_id AND c.knowledge_id=r.knowledge_id
|
||||
AND c.stage='read_progress' AND c.updated_at>=r.queued_at AND c.updated_at<coalesce(r.next_queued_at,'infinity')
|
||||
AND c.fingerprint=(SELECT latest.fingerprint FROM wiki_ingest_checkpoints latest
|
||||
WHERE latest.tenant_id=r.tenant_id AND latest.knowledge_base_id=r.knowledge_base_id AND latest.knowledge_id=r.knowledge_id
|
||||
AND latest.stage='read_progress' AND latest.updated_at>=r.queued_at AND latest.updated_at<coalesce(r.next_queued_at,'infinity')
|
||||
ORDER BY latest.updated_at DESC,latest.fingerprint LIMIT 1)), '{}'::jsonb) AS read_progress,
|
||||
coalesce((SELECT jsonb_object_agg(c.item_key,c.payload::jsonb) FROM wiki_ingest_checkpoints c
|
||||
WHERE c.tenant_id=r.tenant_id AND c.knowledge_base_id=r.knowledge_base_id AND c.knowledge_id=r.knowledge_id
|
||||
AND c.stage='page_progress' AND c.updated_at>=r.queued_at AND c.updated_at<coalesce(r.next_queued_at,'infinity')
|
||||
AND c.fingerprint=(SELECT latest.fingerprint FROM wiki_ingest_checkpoints latest
|
||||
WHERE latest.tenant_id=r.tenant_id AND latest.knowledge_base_id=r.knowledge_base_id AND latest.knowledge_id=r.knowledge_id
|
||||
AND latest.stage='page_progress' AND latest.updated_at>=r.queued_at AND latest.updated_at<coalesce(r.next_queued_at,'infinity')
|
||||
ORDER BY latest.updated_at DESC,latest.fingerprint LIMIT 1)), '{}'::jsonb) AS page_progress,
|
||||
(SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=r.tenant_id AND p.knowledge_base_id=r.knowledge_base_id AND p.deleted_at IS NULL AND ${hasSource('p', 'r.knowledge_id')}) AS current_pages,
|
||||
(SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=r.tenant_id AND p.knowledge_base_id=r.knowledge_base_id AND p.deleted_at IS NULL AND p.page_type='summary' AND p.status='published' AND length(trim(p.content))>0 AND ${hasSource('p', 'r.knowledge_id')}) AS published_summaries,
|
||||
jsonb_array_length(${jsonArray("s.output->'pages_written_preview'")}) AS expected_pages,
|
||||
(SELECT count(*) FROM jsonb_array_elements(${jsonArray("s.output->'pages_written_preview'")}) expected
|
||||
WHERE NOT EXISTS (SELECT 1 FROM wiki_pages p WHERE p.tenant_id=r.tenant_id AND p.knowledge_base_id=r.knowledge_base_id
|
||||
AND p.slug=expected->>'slug' AND p.deleted_at IS NULL AND p.status='published' AND length(trim(p.content))>0 AND ${hasSource('p', 'r.knowledge_id')})) AS missing_expected_pages,
|
||||
(SELECT count(*) FROM knowledge_processing_spans child WHERE child.knowledge_id=r.knowledge_id AND child.started_at>=r.queued_at AND child.started_at<coalesce(r.next_queued_at,'infinity')
|
||||
AND child.name LIKE 'postprocess.wiki.page[%'
|
||||
AND (child.status='failed' OR child.output->>'addition_failed'='true')) AS failed_page_spans,
|
||||
(SELECT count(*) FROM task_dead_letters d WHERE d.tenant_id=r.tenant_id AND d.task_type='wiki:ingest' AND d.scope_id=r.knowledge_base_id AND d.related_id=r.knowledge_id AND d.id>r.dead_letter_highwater AND d.failed_at<coalesce(r.next_queued_at,'infinity')) AS new_failures,
|
||||
(SELECT count(*) FROM task_pending_ops f WHERE f.tenant_id=r.tenant_id AND f.scope_id=r.knowledge_base_id AND f.task_type='wiki:finalize'
|
||||
AND (coalesce(r.deployed_fix_review->>'budget_version','')<>'1' OR (f.payload->>'recovery_dead_letter_id'=r.dead_letter_id::text AND f.payload->>'recovery_attempt'=r.attempt_number::text AND f.payload->>'recovery_pending_op_id'=r.pending_op_id::text))) AS pending_finalize,
|
||||
o.id IS NOT NULL AS pending
|
||||
FROM (SELECT ledger.*,lead(queued_at) OVER (PARTITION BY tenant_id,dead_letter_id ORDER BY queued_at) AS next_queued_at FROM ${table} ledger) r
|
||||
LEFT JOIN task_pending_ops o ON o.id=r.pending_op_id AND o.tenant_id=r.tenant_id
|
||||
LEFT JOIN LATERAL (SELECT status,error_code,output FROM knowledge_processing_spans s
|
||||
WHERE s.knowledge_id=r.knowledge_id AND s.name='postprocess.wiki' AND s.started_at>=r.queued_at AND s.started_at<coalesce(r.next_queued_at,'infinity')
|
||||
ORDER BY s.started_at DESC,s.id DESC LIMIT 1) s ON true
|
||||
WHERE r.tenant_id=${tenant} ORDER BY r.dead_letter_id,r.queued_at
|
||||
) x`).map(row => ({ ...row, full_read_status: describeReadCoverage(row), recovery_status: durableRecoveryStatus(row) || (row.pending ? (row.claimed_at ? 'running' : 'queued')
|
||||
: row.new_failures > 0 || row.span_status === 'failed' ? 'failed'
|
||||
: row.span_status === 'skipped' ? 'skipped'
|
||||
: Number(row.pages_dropped) > 0 || Number(row.failed_slug_writes) > 0 || row.missing_expected_pages > 0 || row.failed_page_spans > 0 ? 'partial'
|
||||
: row.span_status === 'done' && row.pages_dropped !== null && Number(row.pages_dropped) === 0
|
||||
&& row.current_pages > 0 && row.published_summaries > 0 && row.expected_pages > 0
|
||||
&& (row.deployed_fix_review?.budget_version !== 1 || (row.recovery_budget?.status === 'completed' && describeReadCoverage(row) === 'source_processed'))
|
||||
&& row.expected_pages === Number(row.pages_written) && row.expected_pages === Number(row.pages_total) ? 'wiki_recovered'
|
||||
: 'unverified') }))
|
||||
return annotateClosed(rows, loadClosures(directory, tenant), tenant)
|
||||
}
|
||||
|
||||
function verifyPublishedFix(review) {
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(root, 'public-release.json'), 'utf8'))
|
||||
const runtime = JSON.parse(docker(['inspect', 'yawan-native-acceptance', '--format', '{"running":{{json .State.Running}},"image_id":{{json .Image}},"release":{{json (index .Config.Labels "yawan.release")}}}']))
|
||||
const validated = validatePublishedRecoveryFix(review, manifest, runtime)
|
||||
if (review.max_attempt === 6) {
|
||||
const sourceHash = candidateSourceTreeHash(path.resolve(__dirname, '..'))
|
||||
if (review.candidate_diagnostic.kind === replayPins.kind) {
|
||||
verifyReplayInputFiles(root)
|
||||
const evidence = fs.readFileSync(path.join(root, 'acceptance/wiki-layout-identity-replay-private-20260921/evidence.json'))
|
||||
validateSixthReplayEvidence(validated, evidence, sourceHash)
|
||||
} else {
|
||||
const evidence = fs.readFileSync(path.join(root, 'acceptance/wiki-repair-window-diagnostic-private-20260920/evidence.json'))
|
||||
validateSixthDiagnosticEvidence(validated, evidence, sourceHash)
|
||||
}
|
||||
}
|
||||
if (review.max_attempt === 7) {
|
||||
const evidenceDirectory = path.join(root, seventhPins.evidence_directory)
|
||||
const build = verifySeventhBuildFiles(root)
|
||||
const binaryHash = docker(['exec', 'yawan-native-acceptance', 'sha256sum', '/app/WeKnora']).trim()
|
||||
assert.match(binaryHash, /^[a-f0-9]{64}\s+\/app\/WeKnora$/, 'Running API binary hash unavailable')
|
||||
validateSeventhCorrectionEvidence(validated, fs.readFileSync(path.join(evidenceDirectory, 'evidence.json')),
|
||||
build.sourceTreeHash, binaryHash.slice(0, 64), fs.readFileSync(path.join(evidenceDirectory, 'regression-tests.jsonl')),
|
||||
fs.readFileSync(path.join(evidenceDirectory, 'semantic-review.json')), build.buildManifestBytes, build.frozenBinarySHA)
|
||||
}
|
||||
return validated
|
||||
}
|
||||
|
||||
function trigger(rows) {
|
||||
if (!rows.length) return { triggered_scopes: 0 }
|
||||
assert.ok(rows.every(row => [1, 2].includes(attemptOf(row)) || supportedReviewedAttempt(attemptOf(row))), 'Recovery trigger cannot exceed the attempt limit')
|
||||
assert.ok(rows.every(row => row.deployed_fix_review?.budget_version === 1), 'Legacy recovery receipts cannot execute without a durable reviewed budget')
|
||||
const finalRows = rows
|
||||
if (finalRows.length) {
|
||||
const sources = inventory(rows.map(row => row.dead_letter_id))
|
||||
const history = status()
|
||||
for (const row of finalRows) {
|
||||
const review = verifyPublishedFix(row.deployed_fix_review)
|
||||
validateFinalRecoveryTrigger(row, sources.find(source => source.id === row.dead_letter_id), history, review)
|
||||
}
|
||||
}
|
||||
const scopes = [...new Set(rows.map(row => row.knowledge_base_id))].map(id => ({tenant_id: tenant, knowledge_base_id: id, language: 'zh-CN'}))
|
||||
docker(['cp', path.join(root, 'acceptance/wiki-backfill-trigger'), 'yawan-native-acceptance:/tmp/wiki-backfill-trigger'])
|
||||
return JSON.parse(docker(['exec', '-i', 'yawan-native-acceptance', '/tmp/wiki-backfill-trigger'], JSON.stringify(scopes)))
|
||||
}
|
||||
|
||||
function main() {
|
||||
const before = inventory()
|
||||
const prior = status()
|
||||
let triggered
|
||||
if (apply) {
|
||||
let review
|
||||
if (reviewPath) {
|
||||
review = verifyPublishedFix(JSON.parse(fs.readFileSync(reviewPath, 'utf8')))
|
||||
}
|
||||
const selected = ids.map(id => before.find(row => row.id === id))
|
||||
assert.ok(selected.every(row => row?.eligible && row.chunks > 0), 'Every selected original record must remain eligible')
|
||||
validateDurableRecoveryReview(selected[0], review, attempt)
|
||||
const fresh = planRecovery(selected, prior, attempt, retryReason, review)
|
||||
if (fresh.length) {
|
||||
fs.mkdirSync(directory, { recursive: true, mode: 0o700 })
|
||||
const run = Date.now()
|
||||
const backup = queryJSON(`SELECT coalesce(json_agg(row_to_json(d)), '[]') FROM task_dead_letters d WHERE d.tenant_id=${tenant} AND d.task_type='wiki:ingest' AND d.id IN (${fresh.map(row => row.id).join(',')})`)
|
||||
writePrivate(`original-records-${run}.json`, backup)
|
||||
writePrivate(`before-${run}.json`, { tenant_id: tenant, attempt_number: attempt, retry_reason: retryReason, deployed_fix_review: review, selected: fresh, all_originals: before, prior })
|
||||
// The receipt and pending operation commit together. Re-running after a lost
|
||||
// response cannot enqueue the original dead letter a second time.
|
||||
sql(`BEGIN;
|
||||
SELECT pg_advisory_xact_lock(${tenant}, 20260919);
|
||||
CREATE TABLE IF NOT EXISTS ${table} (
|
||||
tenant_id bigint NOT NULL,dead_letter_id bigint NOT NULL,knowledge_id varchar NOT NULL,
|
||||
knowledge_base_id varchar NOT NULL,model_id varchar NOT NULL,pending_op_id bigint NOT NULL,
|
||||
queued_at timestamptz NOT NULL,baseline_pages bigint NOT NULL,baseline_chunks bigint NOT NULL,
|
||||
chunk_chars bigint NOT NULL,dead_letter_highwater bigint NOT NULL,
|
||||
PRIMARY KEY(tenant_id,dead_letter_id));
|
||||
ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS attempt_number integer NOT NULL DEFAULT 1;
|
||||
ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS retry_reason text NOT NULL DEFAULT '';
|
||||
ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS deployed_fix_review jsonb;
|
||||
ALTER TABLE ${table} DROP CONSTRAINT IF EXISTS ${table}_pkey;
|
||||
ALTER TABLE ${table} ADD PRIMARY KEY(tenant_id,dead_letter_id,attempt_number);
|
||||
LOCK TABLE task_pending_ops IN SHARE ROW EXCLUSIVE MODE;
|
||||
LOCK TABLE knowledges, knowledge_bases, chunks, models IN SHARE MODE;
|
||||
${attempt === 7 ? 'LOCK TABLE wiki_recovery_budgets, task_dead_letters, knowledge_processing_spans, wiki_ingest_checkpoints IN SHARE MODE;' : ''}
|
||||
WITH source AS (
|
||||
SELECT d.*,nextval(pg_get_serial_sequence('task_pending_ops','id')) AS recovery_op_id,
|
||||
k.wiki_config->>'synthesis_model_id' AS model_id
|
||||
FROM task_dead_letters d JOIN knowledges n ON n.id=d.related_id AND n.tenant_id=d.tenant_id AND n.knowledge_base_id=d.scope_id
|
||||
JOIN knowledge_bases k ON k.id=d.scope_id AND k.tenant_id=d.tenant_id
|
||||
WHERE d.tenant_id=${tenant} AND d.task_type='wiki:ingest' AND d.scope='knowledge_base'
|
||||
AND d.id IN (${fresh.map(row => row.id).join(',')}) AND d.payload->>'op'='ingest' AND d.payload->>'knowledge_id'=d.related_id
|
||||
AND n.deleted_at IS NULL AND k.deleted_at IS NULL AND n.parse_status='completed' AND n.pending_subtasks_count=0
|
||||
AND k.indexing_strategy->>'wiki_enabled'='true'
|
||||
${review ? `AND d.related_id=${literal(review.knowledge_id)} AND d.scope_id=${literal(review.knowledge_base_id)} AND k.wiki_config->>'synthesis_model_id'=${literal(review.model_id)}` : ''}
|
||||
AND ${fingerprints.source}=${literal(review.source_fingerprint)} AND ${fingerprints.model}=${literal(review.model_fingerprint)}
|
||||
AND k.embedding_model_id=${literal(review.embedding_model_id)} AND ${literal(review.deadline_at)}::timestamptz>now()
|
||||
${attempt === 5 ? `AND EXISTS (SELECT 1 FROM ${table} p WHERE p.tenant_id=d.tenant_id AND p.dead_letter_id=d.id AND p.attempt_number=4 AND p.pending_op_id=4143)` : ''}
|
||||
${attempt === 6 ? `AND d.id=10 AND d.tenant_id=10059
|
||||
AND ${review.max_provider_requests} BETWEEN 1 AND 1000
|
||||
AND ${literal(review.budget_started_at)}::timestamptz<=now()
|
||||
AND ${literal(review.deadline_at)}::timestamptz>${literal(review.budget_started_at)}::timestamptz
|
||||
AND ${literal(review.deadline_at)}::timestamptz<=${literal(review.budget_started_at)}::timestamptz+interval '120 minutes'
|
||||
AND EXISTS (SELECT 1 FROM ${table} p JOIN wiki_recovery_budgets b
|
||||
ON b.tenant_id=p.tenant_id AND b.dead_letter_id=p.dead_letter_id AND b.attempt_number=p.attempt_number AND b.pending_op_id=p.pending_op_id
|
||||
WHERE p.tenant_id=d.tenant_id AND p.dead_letter_id=d.id AND p.attempt_number=5 AND p.pending_op_id=4144
|
||||
AND p.knowledge_id=d.related_id AND p.knowledge_base_id=d.scope_id AND p.model_id=k.wiki_config->>'synthesis_model_id'
|
||||
AND b.status='stopped_budget' AND b.knowledge_id=d.related_id AND b.knowledge_base_id=d.scope_id
|
||||
AND b.model_id=k.wiki_config->>'synthesis_model_id' AND b.embedding_model_id=k.embedding_model_id
|
||||
AND b.source_fingerprint=${literal(review.source_fingerprint)} AND b.model_fingerprint=${literal(review.model_fingerprint)}
|
||||
AND (EXISTS (SELECT 1 FROM task_dead_letters failure WHERE failure.tenant_id=p.tenant_id
|
||||
AND failure.task_type='wiki:ingest' AND failure.scope_id=p.knowledge_base_id
|
||||
AND failure.related_id=p.knowledge_id AND failure.id>p.dead_letter_highwater)
|
||||
OR (b.stop_reason='guard_rejected' AND b.last_phase='wiki_candidate_slug' AND b.reserved_provider_requests=4
|
||||
AND jsonb_typeof(b.paused_payload::jsonb)='array' AND jsonb_array_length(b.paused_payload::jsonb)=1
|
||||
AND b.paused_payload::jsonb @> ${literal(JSON.stringify([sixthGuardPausedOperation]))}::jsonb
|
||||
AND (SELECT span.status='failed' AND span.error_code='EXTRACT_FAILED' FROM knowledge_processing_spans span
|
||||
WHERE span.knowledge_id=p.knowledge_id AND span.name='postprocess.wiki' AND span.started_at>=p.queued_at
|
||||
ORDER BY span.started_at DESC,span.id DESC LIMIT 1))))
|
||||
AND NOT EXISTS (SELECT 1 FROM ${table} later WHERE later.tenant_id=d.tenant_id AND later.dead_letter_id=d.id AND later.attempt_number>5)` : ''}
|
||||
${attempt === 7 ? seventhRecoveryEnqueueSQL(review, table, literal) : ''}
|
||||
AND NOT EXISTS (SELECT 1 FROM task_pending_ops f WHERE f.tenant_id=d.tenant_id AND f.scope_id=d.scope_id AND f.task_type='wiki:finalize')
|
||||
AND NOT EXISTS (SELECT 1 FROM ${table} r WHERE r.tenant_id=d.tenant_id AND r.dead_letter_id=d.id AND r.attempt_number=${attempt})
|
||||
AND (${attempt}=1 OR EXISTS (SELECT 1 FROM ${table} r WHERE r.tenant_id=d.tenant_id AND r.dead_letter_id=d.id AND r.attempt_number=${attempt - 1}))
|
||||
AND NOT EXISTS (SELECT 1 FROM ${table} r JOIN task_pending_ops o ON o.id=r.pending_op_id AND o.tenant_id=r.tenant_id WHERE r.tenant_id=${tenant})
|
||||
AND NOT EXISTS (SELECT 1 FROM task_pending_ops o WHERE o.tenant_id=d.tenant_id AND o.scope_id=d.scope_id AND o.task_type='wiki:ingest' AND o.dedup_key=d.related_id)
|
||||
), budget AS (
|
||||
INSERT INTO wiki_recovery_budgets(tenant_id,dead_letter_id,attempt_number,pending_op_id,knowledge_base_id,knowledge_id,model_id,embedding_model_id,
|
||||
source_fingerprint,model_fingerprint,reviewed_release,reviewed_api_image,max_provider_requests,reserved_provider_requests,deadline_at,status,paused_payload,verified_progress)
|
||||
SELECT tenant_id,id,${attempt},recovery_op_id,scope_id,related_id,model_id,${literal(review.embedding_model_id)},
|
||||
${literal(review.source_fingerprint)},${literal(review.model_fingerprint)},${literal(review.fix_release)},${literal(review.api_image_id)},
|
||||
${review.max_provider_requests},0,${literal(review.deadline_at)}::timestamptz,'active','[]','{}' FROM source RETURNING pending_op_id
|
||||
), inserted AS (
|
||||
INSERT INTO task_pending_ops(id,tenant_id,task_type,scope,scope_id,op,dedup_key,payload,fail_count,enqueued_at)
|
||||
SELECT recovery_op_id,tenant_id,task_type,scope,scope_id,'ingest',related_id,
|
||||
payload||jsonb_build_object('recovery_dead_letter_id',id,'recovery_attempt',${attempt},'recovery_pending_op_id',recovery_op_id,'recovery_budget_version',1),${recoveryInitialFailCount},now() FROM source
|
||||
WHERE EXISTS (SELECT 1 FROM budget b WHERE b.pending_op_id=source.recovery_op_id) RETURNING id
|
||||
) INSERT INTO ${table} (tenant_id,dead_letter_id,knowledge_id,knowledge_base_id,model_id,pending_op_id,queued_at,baseline_pages,baseline_chunks,chunk_chars,dead_letter_highwater,attempt_number,retry_reason,deployed_fix_review)
|
||||
SELECT d.tenant_id,d.id,d.related_id,d.scope_id,d.model_id,d.recovery_op_id,now(),
|
||||
(SELECT count(*) FROM wiki_pages p WHERE p.tenant_id=d.tenant_id AND p.knowledge_base_id=d.scope_id AND p.deleted_at IS NULL AND ${hasSource('p', 'd.related_id')}),
|
||||
(SELECT count(*) FROM chunks c WHERE c.tenant_id=d.tenant_id AND c.knowledge_id=d.related_id AND c.deleted_at IS NULL),
|
||||
(SELECT coalesce(sum(length(c.content)),0) FROM chunks c WHERE c.tenant_id=d.tenant_id AND c.knowledge_id=d.related_id AND c.deleted_at IS NULL),
|
||||
(SELECT coalesce(max(id),0) FROM task_dead_letters),${attempt},${literal(retryReason)},${review ? literal(JSON.stringify(review)) + '::jsonb' : 'NULL'}
|
||||
FROM source d JOIN inserted i ON i.id=d.recovery_op_id;
|
||||
COMMIT;`, false)
|
||||
assert.ok(status().some(row => row.dead_letter_id === fresh[0].id && attemptOf(row) === attempt), 'Recovery was not enqueued because state changed; inspect status before retrying')
|
||||
}
|
||||
triggered = trigger(status().filter(row => ids.includes(row.dead_letter_id) && attemptOf(row) === attempt && row.pending))
|
||||
} else if (triggerOnly) {
|
||||
triggered = trigger(latestAttempts(prior).filter(row => row.pending && (!ids.length || ids.includes(row.dead_letter_id))))
|
||||
}
|
||||
const history = status()
|
||||
const recovered = latestAttempts(history)
|
||||
const report = { checked_at: new Date().toISOString(), tenant_id: tenant, apply, triggered,
|
||||
original_dead_letters: before.length, attempted: recovered.length,
|
||||
remaining_not_attempted: before.filter(row => !recovered.some(recovery => recovery.dead_letter_id === row.id)).length,
|
||||
scope: 'Wiki recovery only. Original records and checkpoints are preserved. One document per batch. Further attempts require their exact reviewed published fix. Only original dead letter 10 has exact fifth/sixth/seventh-attempt exceptions. The seventh requires separately reviewed fresh source identity correction tests, matching frozen source and binary, a newer published fix, at most 1000 actual provider sends and a fixed 120-minute maximum. No eighth attempt or automatic attempt increase. Terminal budget states stop scheduling. Historical page recovery does not prove full-source reading.',
|
||||
candidates: before, recoveries: recovered, attempt_history: history }
|
||||
fs.mkdirSync(directory, { recursive: true, mode: 0o700 })
|
||||
const reportFile = path.join(directory, `status-${Date.now()}.json`)
|
||||
writePrivate(path.basename(reportFile), report)
|
||||
const { candidates, attempt_history, ...summary } = report
|
||||
console.log(JSON.stringify({ ...summary, report_directory: directory, report_file: reportFile }, null, 2))
|
||||
}
|
||||
|
||||
try { main() } catch (error) {
|
||||
// Docker/psql exceptions may contain full SQL or sensitive runtime output.
|
||||
console.error(error instanceof assert.AssertionError ? error.message : 'Wiki recovery stopped; inspect private records and container health. No automatic re-enqueue was attempted.')
|
||||
process.exitCode = 1
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
const { test } = require('node:test')
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const vm = require('node:vm')
|
||||
const policy = require('./wiki_recovery_policy.cjs')
|
||||
|
||||
function triggerFixture(overrides = {}) {
|
||||
const review = { tenant_id: 10059, dead_letter_id: 10, knowledge_id: 'doc', knowledge_base_id: 'kb', model_id: 'model',
|
||||
previous_attempt: 2, previous_error_code: 'PAGE_WRITE_INCOMPLETE', max_attempt: 3, fix_release: 'public-20260919-061500',
|
||||
budget_version: 1, api_image_id: `sha256:${'a'.repeat(64)}`, source_fingerprint: 'b'.repeat(64), model_fingerprint: 'c'.repeat(64),
|
||||
embedding_model_id: 'embedding', max_provider_requests: 200, deadline_at: new Date(Date.now() + 3600000).toISOString() }
|
||||
const manifest = { stage: 'published', release: review.fix_release, api_image_id: `sha256:${'a'.repeat(64)}`,
|
||||
published_at: '2026-09-18T22:00:00Z', ...overrides.manifest }
|
||||
const runtime = { running: true, release: review.fix_release, image_id: `sha256:${'a'.repeat(64)}`, ...overrides.runtime }
|
||||
const source = { id: 10, tenant_id: 10059, knowledge_id: 'doc', knowledge_base_id: 'kb', model_id: 'model',
|
||||
source_fingerprint: review.source_fingerprint, model_fingerprint: review.model_fingerprint, embedding_model_id: review.embedding_model_id,
|
||||
eligible: true, chunks: 1, pending: 1, ...overrides.source }
|
||||
const previous = { tenant_id: 10059, dead_letter_id: 10, knowledge_id: 'doc', knowledge_base_id: 'kb', model_id: 'model',
|
||||
attempt_number: 2, pending: false, new_failures: 1, span_status: 'failed', error_code: 'PAGE_WRITE_INCOMPLETE', queued_at: '2026-09-18T21:00:00Z' }
|
||||
const receipt = { ...previous, attempt_number: 3, pending_op_id: 99, pending: true, claimed_at: null, new_failures: 0,
|
||||
queued_at: '2026-09-18T23:00:00Z', deployed_fix_review: review,
|
||||
retry_reason: 'Deployed bounded complete-page repair with persistence regression tests', ...overrides.receipt }
|
||||
receipt.recovery_budget = { ...receipt, source_fingerprint: review.source_fingerprint, model_fingerprint: review.model_fingerprint,
|
||||
embedding_model_id: review.embedding_model_id, max_provider_requests: review.max_provider_requests, reserved_provider_requests: 0,
|
||||
deadline_at: review.deadline_at, reviewed_release: review.fix_release, reviewed_api_image: review.api_image_id, status: 'active' }
|
||||
let triggerCalls = 0
|
||||
let manifestReads = 0
|
||||
const errors = []
|
||||
const context = {
|
||||
__dirname,
|
||||
process: { argv: ['node', 'recover_wiki_deadletters.cjs', '--trigger', '--ids=10'] },
|
||||
console: { log() {}, error(message) { errors.push(message) } },
|
||||
require(name) {
|
||||
if (name === 'node:fs') return {
|
||||
readFileSync(file) {
|
||||
assert.equal(path.basename(file), 'public-release.json')
|
||||
manifestReads++
|
||||
return JSON.stringify(manifest)
|
||||
},
|
||||
mkdirSync() {}, writeFileSync() {},
|
||||
}
|
||||
if (name === 'node:child_process') return { execFileSync(command, args, options = {}) {
|
||||
assert.equal(command, 'docker')
|
||||
if (args.includes('psql')) {
|
||||
assert.ok(args.includes('PGOPTIONS=-c default_transaction_read_only=on'), 'Trigger must not enqueue another operation')
|
||||
if (options.input.includes('to_regclass')) return 't'
|
||||
if (options.input.includes('FROM (SELECT ledger.*')) return JSON.stringify([previous, receipt])
|
||||
return JSON.stringify([source])
|
||||
}
|
||||
if (args.includes('inspect')) return JSON.stringify(runtime)
|
||||
if (args.includes('cp')) return ''
|
||||
if (args.includes('exec') && args.includes('/tmp/wiki-backfill-trigger')) {
|
||||
triggerCalls++
|
||||
return JSON.stringify({ triggered_scopes: 1 })
|
||||
}
|
||||
throw new Error('Unexpected Docker operation')
|
||||
} }
|
||||
if (name === './wiki_recovery_policy.cjs') return policy
|
||||
return require(name)
|
||||
},
|
||||
}
|
||||
vm.runInNewContext(fs.readFileSync(path.join(__dirname, 'recover_wiki_deadletters.cjs'), 'utf8'), context)
|
||||
return { triggerCalls, manifestReads, errors, exitCode: context.process.exitCode || 0 }
|
||||
}
|
||||
|
||||
test('--trigger rechecks the persisted final review against the published image', () => {
|
||||
assert.deepEqual(triggerFixture(), { triggerCalls: 1, manifestReads: 1, errors: [], exitCode: 0 })
|
||||
for (const change of [
|
||||
{ manifest: { release: 'public-20260918-000000' } },
|
||||
{ manifest: { stage: 'origin-ready' } },
|
||||
{ runtime: { release: 'public-20260918-000000' } },
|
||||
{ runtime: { image_id: `sha256:${'b'.repeat(64)}` } },
|
||||
{ runtime: { running: false } },
|
||||
{ receipt: { deployed_fix_review: undefined } },
|
||||
]) {
|
||||
const result = triggerFixture(change)
|
||||
assert.equal(result.triggerCalls, 0)
|
||||
assert.equal(result.exitCode, 1)
|
||||
assert.equal(result.errors.length, 1)
|
||||
}
|
||||
})
|
||||
|
||||
test('--trigger rejects source or receipt changes and mismatched reviewed attempts', () => {
|
||||
for (const change of [
|
||||
{ source: { tenant_id: 10060 } },
|
||||
{ source: { knowledge_id: 'other' } },
|
||||
{ source: { knowledge_base_id: 'other' } },
|
||||
{ source: { model_id: 'other' } },
|
||||
{ receipt: { knowledge_base_id: 'other' } },
|
||||
{ receipt: { model_id: 'other' } },
|
||||
{ receipt: { attempt_number: 4 } },
|
||||
]) {
|
||||
const result = triggerFixture(change)
|
||||
assert.equal(result.triggerCalls, 0)
|
||||
assert.equal(result.exitCode, 1)
|
||||
assert.equal(result.errors.length, 1)
|
||||
}
|
||||
})
|
||||
|
||||
test('page recovery and window coverage never imply complete source reading without the new map proof', () => {
|
||||
assert.equal(policy.describeReadCoverage({ recovery_status: 'wiki_recovered' }), 'not_verified')
|
||||
assert.equal(policy.describeReadCoverage({ read_progress: { summary: { status: 'complete', source_windows_complete: true } } }), 'in_progress')
|
||||
assert.equal(policy.describeReadCoverage({ read_progress: { candidates: { status: 'paused_budget' } } }), 'paused_budget')
|
||||
assert.equal(policy.describeReadCoverage({ read_coverage_version: 'wiki-full-read-v1', source_read_complete: true }), 'source_processed')
|
||||
assert.equal(policy.describeReadCoverage({ read_coverage_version: 'old-prefix', source_read_complete: true }), 'not_verified')
|
||||
})
|
||||
|
||||
test('terminal reading failures cannot be reported as an active saved prefix', () => {
|
||||
const read_progress = { candidates: { status: 'reading', completed_windows: 1, total_windows: 7 } }
|
||||
assert.equal(policy.describeReadCoverage({ read_progress, span_status: 'failed' }), 'failed')
|
||||
assert.equal(policy.describeReadCoverage({ read_progress, recovery_budget: { status: 'stopped_budget', stop_reason: 'phase_failed' } }), 'failed')
|
||||
assert.equal(policy.describeReadCoverage({ read_progress, recovery_budget: { status: 'stopped_budget', stop_reason: 'deadline' } }), 'stopped_budget')
|
||||
assert.equal(policy.describeReadCoverage({ read_progress, recovery_budget: { status: 'stopped_invalid' } }), 'stopped_invalid')
|
||||
assert.equal(policy.describeReadCoverage({ read_progress, recovery_budget: { status: 'active' } }), 'in_progress')
|
||||
assert.equal(policy.describeReadCoverage({ read_progress, span_status: 'failed', read_coverage_version: 'wiki-full-read-v1', source_read_complete: true }), 'source_processed')
|
||||
assert.equal(read_progress.candidates.completed_windows, 1)
|
||||
})
|
||||
@@ -0,0 +1,23 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const assert = require('node:assert/strict')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
const root = path.resolve(__dirname, '../..')
|
||||
const release = process.argv[2]
|
||||
assert.match(release || '', /^public-\d{8}-\d{6}$/)
|
||||
const directory = path.join(root, 'releases', release)
|
||||
assert.ok(fs.statSync(path.join(directory, 'yawan_native_acceptance_20260915.dump')).size > 0, 'Backup required')
|
||||
const sql = statement => execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres', 'psql', '-X', '-q', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-At', '-v', 'ON_ERROR_STOP=1'], { input: statement, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim()
|
||||
const predicate = "tenant_id BETWEEN 10000 AND 10058 AND NOT is_system_admin AND deleted_at IS NULL AND email ~ '^(acceptance-.+|etl-[0-9]+|milvus-[0-9]+)@example[.]test$'"
|
||||
const accounts = JSON.parse(sql(`SELECT coalesce(json_agg(json_build_object('id',id,'tenant_id',tenant_id,'email',email,'is_active',is_active)), '[]'::json) FROM users WHERE ${predicate};`))
|
||||
assert.equal(accounts.length, 59, 'Only the known disposable migration fixtures may be retired')
|
||||
const beforePath = path.join(directory, 'retired-fixture-accounts.json')
|
||||
assert.ok(!fs.existsSync(beforePath), 'Do not overwrite previous account state')
|
||||
fs.writeFileSync(beforePath, JSON.stringify({ recorded_at: new Date().toISOString(), accounts }, null, 2), { mode: 0o600, flag: 'wx' })
|
||||
sql(`BEGIN;
|
||||
UPDATE auth_tokens SET is_revoked=true WHERE user_id IN (SELECT id FROM users WHERE ${predicate});
|
||||
UPDATE tenant_api_keys SET revoked_at=now(),updated_at=now() WHERE tenant_id IN (SELECT tenant_id FROM users WHERE ${predicate}) AND revoked_at IS NULL;
|
||||
UPDATE users SET is_active=false,updated_at=now() WHERE ${predicate};
|
||||
COMMIT;`)
|
||||
assert.equal(sql(`SELECT count(*) FROM users WHERE ${predicate} AND is_active;`), '0')
|
||||
console.log(JSON.stringify({ retired_fixture_accounts: accounts.length, business_tenant_unchanged: 10059 }))
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const root = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const storage = resolve(root, '..');
|
||||
const context = process.env.GRAPH_DOCKER_CONTEXT || 'colima-yawan';
|
||||
const container = process.env.GRAPH_APP_CONTAINER || 'yawan-native-acceptance';
|
||||
const inspected = spawnSync('docker', ['--context', context, 'inspect', container], { encoding: 'utf8' });
|
||||
if (inspected.status !== 0) throw new Error('Cannot inspect graph worker runtime');
|
||||
const config = JSON.parse(inspected.stdout)[0];
|
||||
const runtime = Object.fromEntries(config.Config.Env.map(value => {
|
||||
const index = value.indexOf('=');
|
||||
return [value.slice(0, index), value.slice(index + 1)];
|
||||
}));
|
||||
const selected = Object.fromEntries(Object.entries(runtime).filter(([key]) => /^(DB_|REDIS_)/.test(key) || key === 'LEGACY_GRAPH_ENABLE'));
|
||||
const args = ['--context', context, 'run', '--rm', '--network', `container:${container}`, '-v', `${root}:/workspace:ro`, '-v', `${storage}/go-cache:/go`, '-v', `${storage}/go-build:/build-cache`, '-e', 'GOMODCACHE=/go/pkg/mod', '-e', 'GOCACHE=/build-cache', '-e', 'GRAPH_NATIVE_WRITE_VERIFIED'];
|
||||
if (process.env.GRAPH_BACKFILL_REVISION) args.push('-e', 'GRAPH_BACKFILL_REVISION');
|
||||
if (process.env.GRAPH_LEDGER_RECONCILE_VERIFIED) args.push('-e', 'GRAPH_LEDGER_RECONCILE_VERIFIED');
|
||||
for (const key of Object.keys(selected)) args.push('-e', key);
|
||||
args.push('-w', '/workspace', 'golang:1.26-bookworm', 'go', 'run', 'scripts/native_graph_backfill.go', ...process.argv.slice(2));
|
||||
const result = spawnSync('docker', args, { env: { ...process.env, ...selected }, stdio: 'inherit' });
|
||||
process.exitCode = result.status ?? 1;
|
||||
@@ -0,0 +1,136 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const assert = require('node:assert/strict')
|
||||
const { execFileSync, spawn } = require('node:child_process')
|
||||
const { recoverySchedulingGate, remainingRecoveryCandidates } = require('./wiki_recovery_scheduler_policy.cjs')
|
||||
|
||||
const args = process.argv.slice(2)
|
||||
const options = { 'max-minutes': 180, 'max-documents': 32, 'batch-size': 1 }
|
||||
let reviewDirectory
|
||||
for (const arg of args) {
|
||||
if (arg === '--apply' || arg === '--detach') continue
|
||||
if (arg.startsWith('--review-directory=')) {
|
||||
reviewDirectory = path.resolve(arg.slice('--review-directory='.length))
|
||||
continue
|
||||
}
|
||||
const match = /^--(max-minutes|max-documents|batch-size)=(\d+)$/.exec(arg)
|
||||
assert.ok(match, 'Unknown recovery runner option')
|
||||
options[match[1]] = Number(match[2])
|
||||
}
|
||||
assert.ok(args.includes('--apply'), 'Explicit --apply required')
|
||||
assert.ok(options['max-minutes'] >= 1 && options['max-minutes'] <= 180, 'Time limit must be 1..180 minutes')
|
||||
assert.ok(options['max-documents'] >= 1 && options['max-documents'] <= 32, 'Document limit must be 1..32')
|
||||
assert.equal(options['batch-size'], 1, 'Recovery is limited to one document per batch')
|
||||
|
||||
const directory = path.resolve(__dirname, '../../acceptance/wiki-deadletter-recovery-20260919')
|
||||
const recoveryScript = path.join(__dirname, 'recover_wiki_deadletters.cjs')
|
||||
const stateFile = path.join(directory, 'runner-state.json')
|
||||
const lockFile = path.join(directory, 'runner.lock')
|
||||
fs.mkdirSync(directory, { recursive: true, mode: 0o700 })
|
||||
|
||||
if (args.includes('--detach')) {
|
||||
const log = fs.openSync(path.join(directory, 'runner-console.log'), 'a', 0o600)
|
||||
const child = spawn(process.execPath, [__filename, ...args.filter(arg => arg !== '--detach')], {
|
||||
detached: true, stdio: ['ignore', log, log], cwd: path.dirname(recoveryScript),
|
||||
})
|
||||
child.unref()
|
||||
fs.closeSync(log)
|
||||
console.log(JSON.stringify({ launched_pid: child.pid, state_file: stateFile }))
|
||||
} else {
|
||||
main().catch(() => {
|
||||
console.error('Recovery runner stopped. Inspect runner-state.json and private status reports.')
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
|
||||
function summarize(report) {
|
||||
const counts = {}
|
||||
const readCoverage = {}
|
||||
for (const row of report.recoveries) counts[row.recovery_status] = (counts[row.recovery_status] || 0) + 1
|
||||
for (const row of report.recoveries) readCoverage[row.full_read_status || 'not_verified'] = (readCoverage[row.full_read_status || 'not_verified'] || 0) + 1
|
||||
const { historical_full_read_unverified_ids, source_reading_complete } = recoverySchedulingGate(report)
|
||||
return { attempted: report.attempted, remaining_not_attempted: report.remaining_not_attempted, counts,
|
||||
full_read_counts: readCoverage,
|
||||
historical_full_read_unverified_ids, source_reading_complete,
|
||||
current_reading: report.recoveries.filter(row => row.pending).map(row => ({ dead_letter_id: row.dead_letter_id, read_progress: row.read_progress || {}, page_progress: row.page_progress || {} })),
|
||||
page_associations: report.recoveries.reduce((sum, row) => sum + row.current_pages, 0),
|
||||
new_failure_records: report.recoveries.reduce((sum, row) => sum + row.new_failures, 0),
|
||||
report_file: report.report_file }
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// An existing lock requires operator inspection, even if its process died.
|
||||
// Never start a second scheduler on the basis of a possibly reused PID.
|
||||
const lock = fs.openSync(lockFile, 'wx', 0o600)
|
||||
fs.writeFileSync(lock, JSON.stringify({ pid: process.pid, started_at: new Date().toISOString() }))
|
||||
fs.closeSync(lock)
|
||||
const started = Date.now()
|
||||
let queued = 0
|
||||
let lastReport
|
||||
let stopping = false
|
||||
process.on('SIGTERM', () => { stopping = true })
|
||||
process.on('SIGINT', () => { stopping = true })
|
||||
const save = (state, extra = {}) => {
|
||||
const value = { state, pid: process.pid, started_at: new Date(started).toISOString(),
|
||||
updated_at: new Date().toISOString(), options, newly_enqueued: queued,
|
||||
...(lastReport ? summarize(lastReport) : {}), ...extra }
|
||||
const temporary = `${stateFile}.${process.pid}.tmp`
|
||||
fs.writeFileSync(temporary, JSON.stringify(value, null, 2) + '\n', { mode: 0o600 })
|
||||
fs.renameSync(temporary, stateFile)
|
||||
fs.appendFileSync(path.join(directory, 'runner-events.jsonl'), JSON.stringify(value) + '\n', { mode: 0o600 })
|
||||
console.log(JSON.stringify(value))
|
||||
}
|
||||
const inspect = (commandArgs = []) => {
|
||||
const consoleReport = JSON.parse(execFileSync(process.execPath, [recoveryScript, ...commandArgs], {
|
||||
encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 120000, maxBuffer: 4 * 1024 * 1024,
|
||||
}))
|
||||
assert.equal(path.dirname(consoleReport.report_file), directory, 'Unexpected report directory')
|
||||
lastReport = { ...JSON.parse(fs.readFileSync(consoleReport.report_file, 'utf8')), report_file: consoleReport.report_file }
|
||||
return lastReport
|
||||
}
|
||||
try {
|
||||
save('starting')
|
||||
while (true) {
|
||||
const report = inspect()
|
||||
const { state, ...coverage } = recoverySchedulingGate(report)
|
||||
if (state === 'stopped_for_review') {
|
||||
save(state, coverage)
|
||||
break
|
||||
}
|
||||
if (stopping || Date.now() - started >= options['max-minutes'] * 60000) {
|
||||
save(stopping ? 'stopped_by_signal' : 'time_limit_reached')
|
||||
break
|
||||
}
|
||||
if (state === 'waiting_for_current_batch') {
|
||||
save(state, coverage)
|
||||
await new Promise(resolve => setTimeout(resolve, 45000))
|
||||
continue
|
||||
}
|
||||
if (report.remaining_not_attempted === 0) {
|
||||
save(state, coverage)
|
||||
break
|
||||
}
|
||||
if (queued >= options['max-documents']) {
|
||||
save('document_limit_reached')
|
||||
break
|
||||
}
|
||||
const candidates = remainingRecoveryCandidates(report)
|
||||
const ids = candidates.slice(0, Math.min(options['batch-size'], options['max-documents'] - queued)).map(row => row.id)
|
||||
assert.ok(ids.length > 0, 'No eligible remaining originals')
|
||||
const reviewFile = reviewDirectory && path.join(reviewDirectory, `dead-letter-${ids[0]}-attempt-1.json`)
|
||||
if (!reviewFile || !fs.existsSync(reviewFile)) {
|
||||
save('stopped_for_review', { batch_ids: ids, reason: 'A per-document reviewed request limit and fixed deadline are required before enqueueing.' })
|
||||
break
|
||||
}
|
||||
save('enqueueing_batch', { batch_ids: ids })
|
||||
inspect(['--apply', `--ids=${ids.join(',')}`, `--review-file=${reviewFile}`])
|
||||
queued += ids.length
|
||||
save('batch_enqueued', { batch_ids: ids })
|
||||
}
|
||||
} catch (error) {
|
||||
save('stopped_on_error', { reason: error instanceof assert.AssertionError ? error.message : 'Recovery command failed; no automatic retry was scheduled.' })
|
||||
throw error
|
||||
} finally {
|
||||
fs.unlinkSync(lockFile)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
'use strict'
|
||||
|
||||
// Offline CLI. It consumes reviewed fixture files and never calls a provider.
|
||||
// A real deployment adapter must reserve/attest its diagnostic lane before
|
||||
// using the pure continuation module; this command is only for acceptance.
|
||||
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const { runDiagnosticContinuation } = require('./wiki_leaf_diagnostic_continuation.cjs')
|
||||
|
||||
function argument(name) {
|
||||
const prefix = `--${name}=`
|
||||
const value = process.argv.slice(2).find(arg => arg.startsWith(prefix))
|
||||
return value?.slice(prefix.length)
|
||||
}
|
||||
|
||||
const allowed = new Set(['--manifest=', '--source=', '--outcome='])
|
||||
const unknown = process.argv.slice(2).filter(arg => ![...allowed].some(prefix => arg.startsWith(prefix)))
|
||||
if (unknown.length || !argument('manifest') || !argument('source') || !argument('outcome')) {
|
||||
console.error('usage: node scripts/run_wiki_leaf_diagnostic_continuation.cjs --manifest=FILE --source=FILE --outcome=FILE')
|
||||
process.exitCode = 2
|
||||
} else {
|
||||
try {
|
||||
const read = file => fs.readFileSync(path.resolve(file), 'utf8')
|
||||
const manifest = JSON.parse(read(argument('manifest')))
|
||||
const source = read(argument('source'))
|
||||
const outcome = JSON.parse(read(argument('outcome')))
|
||||
const capture = []
|
||||
const result = runDiagnosticContinuation(manifest, source, () => outcome, artifact => {
|
||||
// Keep the CLI output replay-safe. The real private capture sink owns
|
||||
// the raw response; this fixture command emits hashes and byte counts.
|
||||
capture.push({ status: artifact.status, idempotency_key: artifact.idempotency_key,
|
||||
response_sha256: artifact.response_sha256, response_bytes: artifact.response_bytes })
|
||||
})
|
||||
const windows = result.windows.map(window => ({ key: window.key, status: window.status,
|
||||
start_rune: window.start_rune, end_rune: window.end_rune, text_sha256: window.text_sha256 }))
|
||||
process.stdout.write(JSON.stringify({ receipt: result.receipt, windows, capture }, null, 2) + '\n')
|
||||
} catch (error) {
|
||||
process.stderr.write(JSON.stringify({ passed: false, code: error.code || 'continuation_failed', error: error.message }) + '\n')
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
|
||||
function copyEntry(source, destination) {
|
||||
const stat = fs.lstatSync(source)
|
||||
if (stat.isFile()) {
|
||||
fs.copyFileSync(source, destination, fs.constants.COPYFILE_EXCL)
|
||||
} else if (stat.isDirectory()) {
|
||||
fs.mkdirSync(destination, { mode: stat.mode & 0o777 })
|
||||
for (const name of fs.readdirSync(source)) copyEntry(path.join(source, name), path.join(destination, name))
|
||||
} else {
|
||||
throw new Error('API build context requires regular files and directories; symlinks are not supported')
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the reviewed inputs outside the source tree; never reuse a partial context.
|
||||
function stagePublicAPIContext(root, release, revision) {
|
||||
if (!/^public-\d{8}-\d{6}$/.test(release || '')) throw new Error('Invalid API context release')
|
||||
if (revision !== undefined && !/^\d{17}$/.test(revision)) throw new Error('Invalid API context revision')
|
||||
const source = path.join(root, 'src')
|
||||
const inputs = [
|
||||
['scripts/Dockerfile.public-api', 'Dockerfile', 'file'],
|
||||
['WeKnora-native-acceptance-next', 'WeKnora-native-acceptance-next', 'file'],
|
||||
['migrations', 'migrations', 'directory'],
|
||||
]
|
||||
for (const [name, , type] of inputs) {
|
||||
const stat = fs.lstatSync(path.join(source, name))
|
||||
if (!(type === 'file' ? stat.isFile() : stat.isDirectory())) throw new Error(`Invalid API context input: ${name}`)
|
||||
}
|
||||
const directory = path.join(root, 'releases', release, revision === undefined ? 'api-context' : `api-context-${revision}`)
|
||||
fs.mkdirSync(directory, { mode: 0o700 })
|
||||
for (const [name, target] of inputs) copyEntry(path.join(source, name), path.join(directory, target))
|
||||
return directory
|
||||
}
|
||||
|
||||
module.exports = { stagePublicAPIContext }
|
||||
@@ -0,0 +1,108 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const test = require('node:test')
|
||||
const fs = require('node:fs')
|
||||
const os = require('node:os')
|
||||
const path = require('node:path')
|
||||
const { stagePublicAPIContext } = require('./stage_public_api_context.cjs')
|
||||
|
||||
const release = 'public-20260921-040700'
|
||||
const revision = '20260921040700123'
|
||||
|
||||
function fixture(t) {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'public-api-context-'))
|
||||
t.after(() => fs.rmSync(root, { recursive: true, force: true }))
|
||||
const source = path.join(root, 'src')
|
||||
fs.mkdirSync(path.join(source, 'scripts'), { recursive: true })
|
||||
fs.mkdirSync(path.join(source, 'migrations/versioned'), { recursive: true })
|
||||
fs.mkdirSync(path.join(source, 'migrations/empty'))
|
||||
fs.mkdirSync(path.join(root, 'releases', release), { recursive: true })
|
||||
const dockerfile = fs.readFileSync(path.join(__dirname, 'Dockerfile.public-api'))
|
||||
const binary = Buffer.from([0x7f, 0x45, 0x4c, 0x46, 0x00, 0xff, 0x0a])
|
||||
const migration = Buffer.from('BEGIN;\nSELECT 1;\nCOMMIT;\n')
|
||||
fs.writeFileSync(path.join(source, 'scripts/Dockerfile.public-api'), dockerfile)
|
||||
fs.writeFileSync(path.join(source, 'WeKnora-native-acceptance-next'), binary, { mode: 0o755 })
|
||||
fs.writeFileSync(path.join(source, 'migrations/versioned/001.sql'), migration)
|
||||
return { root, source, dockerfile, binary, migration }
|
||||
}
|
||||
|
||||
function entries(directory) {
|
||||
return fs.readdirSync(directory).sort().flatMap(name => {
|
||||
const file = path.join(directory, name)
|
||||
return fs.statSync(file).isDirectory() ? [name + '/', ...entries(file).map(child => name + '/' + child)] : [name]
|
||||
})
|
||||
}
|
||||
|
||||
test('stages only Dockerfile, executable bytes and the complete migration tree', t => {
|
||||
const value = fixture(t)
|
||||
fs.writeFileSync(path.join(value.source, '.env'), 'PRIVATE_TEST_SECRET=do-not-copy\n')
|
||||
fs.writeFileSync(path.join(value.root, 'releases', release, 'runtime.env'), 'PRIVATE_RELEASE_SECRET=do-not-copy\n')
|
||||
fs.mkdirSync(path.join(value.source, 'unrelated-cache'))
|
||||
const unrelated = fs.openSync(path.join(value.source, 'unrelated-cache/large.bin'), 'wx')
|
||||
try { fs.ftruncateSync(unrelated, 512 * 1024 * 1024) } finally { fs.closeSync(unrelated) }
|
||||
const context = stagePublicAPIContext(value.root, release)
|
||||
assert.equal(context, path.join(value.root, 'releases', release, 'api-context'))
|
||||
assert.deepEqual(entries(context), ['Dockerfile', 'WeKnora-native-acceptance-next', 'migrations/',
|
||||
'migrations/empty/', 'migrations/versioned/', 'migrations/versioned/001.sql'])
|
||||
assert.deepEqual(fs.readFileSync(path.join(context, 'Dockerfile')), value.dockerfile)
|
||||
assert.deepEqual(fs.readFileSync(path.join(context, 'WeKnora-native-acceptance-next')), value.binary)
|
||||
assert.deepEqual(fs.readFileSync(path.join(context, 'migrations/versioned/001.sql')), value.migration)
|
||||
assert.equal(fs.statSync(path.join(context, 'WeKnora-native-acceptance-next')).mode & 0o777, 0o755)
|
||||
assert.equal(fs.statSync(context).mode & 0o777, 0o700)
|
||||
})
|
||||
|
||||
test('contexts are immutable snapshots and refresh revisions use independent directories', t => {
|
||||
const value = fixture(t)
|
||||
const context = stagePublicAPIContext(value.root, release)
|
||||
const changed = Buffer.from('new reviewed binary\n')
|
||||
fs.writeFileSync(path.join(value.source, 'WeKnora-native-acceptance-next'), changed)
|
||||
assert.throws(() => stagePublicAPIContext(value.root, release), { code: 'EEXIST' })
|
||||
assert.deepEqual(fs.readFileSync(path.join(context, 'WeKnora-native-acceptance-next')), value.binary)
|
||||
const refreshed = stagePublicAPIContext(value.root, release, revision)
|
||||
assert.equal(refreshed, path.join(value.root, 'releases', release, `api-context-${revision}`))
|
||||
assert.deepEqual(fs.readFileSync(path.join(refreshed, 'WeKnora-native-acceptance-next')), changed)
|
||||
assert.throws(() => stagePublicAPIContext(value.root, release, revision), { code: 'EEXIST' })
|
||||
})
|
||||
|
||||
test('never overwrites or cleans a previous partial context', t => {
|
||||
const value = fixture(t)
|
||||
const context = path.join(value.root, 'releases', release, 'api-context')
|
||||
fs.mkdirSync(context)
|
||||
fs.writeFileSync(path.join(context, 'partial-evidence'), 'retained')
|
||||
assert.throws(() => stagePublicAPIContext(value.root, release), { code: 'EEXIST' })
|
||||
assert.deepEqual(entries(context), ['partial-evidence'])
|
||||
})
|
||||
|
||||
test('rejects invalid identifiers before creating a context', t => {
|
||||
const value = fixture(t)
|
||||
for (const candidate of ['', '../outside', 'public-20260921-040700/../../outside']) {
|
||||
assert.throws(() => stagePublicAPIContext(value.root, candidate), /Invalid API context release/)
|
||||
}
|
||||
for (const candidate of ['', '../outside', '20260921', '20260921040700123/../../outside']) {
|
||||
assert.throws(() => stagePublicAPIContext(value.root, release, candidate), /Invalid API context revision/)
|
||||
}
|
||||
assert.deepEqual(fs.readdirSync(path.join(value.root, 'releases', release)), [])
|
||||
})
|
||||
|
||||
test('rejects symlinked binary and migration roots', t => {
|
||||
for (const name of ['WeKnora-native-acceptance-next', 'migrations']) {
|
||||
const value = fixture(t)
|
||||
const input = path.join(value.source, name)
|
||||
const target = path.join(value.root, 'outside')
|
||||
fs.renameSync(input, target)
|
||||
fs.symlinkSync(target, input)
|
||||
assert.throws(() => stagePublicAPIContext(value.root, release), /Invalid API context input/)
|
||||
assert.deepEqual(fs.readdirSync(path.join(value.root, 'releases', release)), [])
|
||||
}
|
||||
})
|
||||
|
||||
test('rejects nested migration symlinks without copying their external contents', t => {
|
||||
const value = fixture(t)
|
||||
const secret = path.join(value.root, 'private.env')
|
||||
fs.writeFileSync(secret, 'PRIVATE_TEST_SECRET=do-not-copy\n')
|
||||
fs.symlinkSync(secret, path.join(value.source, 'migrations/private.env'))
|
||||
assert.throws(() => stagePublicAPIContext(value.root, release), /symlinks are not supported/)
|
||||
const context = path.join(value.root, 'releases', release, 'api-context')
|
||||
assert.equal(fs.existsSync(path.join(context, 'migrations/private.env')), false)
|
||||
assert.throws(() => stagePublicAPIContext(value.root, release), { code: 'EEXIST' })
|
||||
assert.equal(fs.readFileSync(secret, 'utf8'), 'PRIVATE_TEST_SECRET=do-not-copy\n')
|
||||
})
|
||||
Executable
+771
@@ -0,0 +1,771 @@
|
||||
#!/bin/bash
|
||||
# 该脚本用于按需启动/停止Ollama和docker-compose服务
|
||||
|
||||
# 设置颜色
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # 无颜色
|
||||
|
||||
# 获取项目根目录(脚本所在目录的上一级)
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
|
||||
|
||||
# 版本信息
|
||||
VERSION="1.0.1" # 版本更新
|
||||
SCRIPT_NAME=$(basename "$0")
|
||||
|
||||
# 显示帮助信息
|
||||
show_help() {
|
||||
printf "%b\n" "${GREEN}WeKnora 启动脚本 v${VERSION}${NC}"
|
||||
printf "%b\n" "${GREEN}用法:${NC} $0 [选项]"
|
||||
echo "选项:"
|
||||
echo " -h, --help 显示帮助信息"
|
||||
echo " -o, --ollama 启动Ollama服务"
|
||||
echo " -d, --docker 启动Docker容器服务"
|
||||
echo " -a, --all 启动所有服务(默认)"
|
||||
echo " -s, --stop 停止所有服务"
|
||||
echo " -c, --check 检查环境并诊断问题"
|
||||
echo " -r, --restart 重新构建并重启指定容器"
|
||||
echo " -l, --list 列出所有正在运行的容器"
|
||||
echo " -p, --pull 拉取最新的Docker镜像"
|
||||
echo " --no-pull 启动时不拉取镜像(默认会拉取)"
|
||||
echo " -v, --version 显示版本信息"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# 显示版本信息
|
||||
show_version() {
|
||||
printf "%b\n" "${GREEN}WeKnora 启动脚本 v${VERSION}${NC}"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# 日志函数
|
||||
log_info() {
|
||||
printf "%b\n" "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
printf "%b\n" "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
printf "%b\n" "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
printf "%b\n" "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
# 选择可用的 Docker Compose 命令(优先 docker compose,其次 docker-compose)
|
||||
DOCKER_COMPOSE_BIN=""
|
||||
DOCKER_COMPOSE_SUBCMD=""
|
||||
|
||||
detect_compose_cmd() {
|
||||
# 优先使用 Docker Compose 插件
|
||||
if docker compose version &> /dev/null; then
|
||||
DOCKER_COMPOSE_BIN="docker"
|
||||
DOCKER_COMPOSE_SUBCMD="compose"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 回退到 docker-compose (v1)
|
||||
if command -v docker-compose &> /dev/null; then
|
||||
if docker-compose version &> /dev/null; then
|
||||
DOCKER_COMPOSE_BIN="docker-compose"
|
||||
DOCKER_COMPOSE_SUBCMD=""
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# 都不可用
|
||||
return 1
|
||||
}
|
||||
|
||||
# 检查并创建.env文件
|
||||
check_env_file() {
|
||||
log_info "检查环境变量配置..."
|
||||
if [ ! -f "$PROJECT_ROOT/.env" ]; then
|
||||
log_warning ".env 文件不存在,将从模板创建"
|
||||
if [ -f "$PROJECT_ROOT/.env.example" ]; then
|
||||
cp "$PROJECT_ROOT/.env.example" "$PROJECT_ROOT/.env"
|
||||
log_success "已从 .env.example 创建 .env 文件"
|
||||
else
|
||||
log_error "未找到 .env.example 模板文件,无法创建 .env 文件"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log_info ".env 文件已存在"
|
||||
fi
|
||||
|
||||
# 检查必要的环境变量是否已设置
|
||||
source "$PROJECT_ROOT/.env"
|
||||
local missing_vars=()
|
||||
|
||||
# 检查基础变量
|
||||
if [ -z "$DB_DRIVER" ]; then missing_vars+=("DB_DRIVER"); fi
|
||||
if [ -z "$STORAGE_TYPE" ]; then missing_vars+=("STORAGE_TYPE"); fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# 安装Ollama(根据平台不同采用不同方法)
|
||||
install_ollama() {
|
||||
# 检查是否为远程服务
|
||||
get_ollama_base_url
|
||||
|
||||
if [ $IS_REMOTE -eq 1 ]; then
|
||||
log_info "检测到远程Ollama服务配置,无需在本地安装Ollama"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "本地Ollama未安装,正在安装..."
|
||||
|
||||
OS=$(uname)
|
||||
if [ "$OS" = "Darwin" ]; then
|
||||
# Mac安装方式
|
||||
log_info "检测到Mac系统,使用brew安装Ollama..."
|
||||
if ! command -v brew &> /dev/null; then
|
||||
# 通过安装包安装
|
||||
log_info "Homebrew未安装,使用直接下载方式..."
|
||||
curl -fsSL https://ollama.com/download/Ollama-darwin.zip -o ollama.zip
|
||||
unzip ollama.zip
|
||||
mv ollama /usr/local/bin
|
||||
rm ollama.zip
|
||||
else
|
||||
brew install ollama
|
||||
fi
|
||||
else
|
||||
# Linux安装方式
|
||||
log_info "检测到Linux系统,使用安装脚本..."
|
||||
curl -fsSL https://ollama.com/install.sh | sh
|
||||
fi
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_success "本地Ollama安装完成"
|
||||
return 0
|
||||
else
|
||||
log_error "本地Ollama安装失败"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 获取Ollama基础URL,检查是否为远程服务
|
||||
get_ollama_base_url() {
|
||||
|
||||
check_env_file
|
||||
|
||||
# 从环境变量获取Ollama基础URL
|
||||
OLLAMA_URL=${OLLAMA_BASE_URL:-"http://host.docker.internal:11434"}
|
||||
# 提取主机部分
|
||||
OLLAMA_HOST=$(echo "$OLLAMA_URL" | sed -E 's|^https?://||' | sed -E 's|:[0-9]+$||' | sed -E 's|/.*$||')
|
||||
# 提取端口部分
|
||||
OLLAMA_PORT=$(echo "$OLLAMA_URL" | grep -oE ':[0-9]+' | grep -oE '[0-9]+' || echo "11434")
|
||||
# 检查是否为localhost或127.0.0.1
|
||||
IS_REMOTE=0
|
||||
if [ "$OLLAMA_HOST" = "localhost" ] || [ "$OLLAMA_HOST" = "127.0.0.1" ] || [ "$OLLAMA_HOST" = "host.docker.internal" ]; then
|
||||
IS_REMOTE=0 # 本地服务
|
||||
else
|
||||
IS_REMOTE=1 # 远程服务
|
||||
fi
|
||||
}
|
||||
|
||||
# 启动Ollama服务
|
||||
start_ollama() {
|
||||
log_info "正在检查Ollama服务..."
|
||||
# 提取主机和端口
|
||||
get_ollama_base_url
|
||||
log_info "Ollama服务地址: $OLLAMA_URL"
|
||||
|
||||
if [ $IS_REMOTE -eq 1 ]; then
|
||||
log_info "检测到远程Ollama服务,将直接使用远程服务,不进行本地安装和启动"
|
||||
# 检查远程服务是否可用
|
||||
if curl -s "$OLLAMA_URL/api/tags" &> /dev/null; then
|
||||
log_success "远程Ollama服务可访问"
|
||||
return 0
|
||||
else
|
||||
log_warning "远程Ollama服务不可访问,请确认服务地址正确且已启动"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# 以下为本地服务的处理
|
||||
# 检查Ollama是否已安装
|
||||
if ! command -v ollama &> /dev/null; then
|
||||
install_ollama
|
||||
if [ $? -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# 检查Ollama服务是否已运行
|
||||
if curl -s "http://localhost:$OLLAMA_PORT/api/tags" &> /dev/null; then
|
||||
log_success "本地Ollama服务已经在运行,端口:$OLLAMA_PORT"
|
||||
else
|
||||
log_info "启动本地Ollama服务..."
|
||||
# 注意:官方推荐使用 systemctl 或 launchctl 管理服务,直接后台运行仅用于临时场景
|
||||
systemctl restart ollama || (ollama serve > /dev/null 2>&1 < /dev/null &)
|
||||
|
||||
# 等待服务启动
|
||||
MAX_RETRIES=30
|
||||
COUNT=0
|
||||
while [ $COUNT -lt $MAX_RETRIES ]; do
|
||||
if curl -s "http://localhost:$OLLAMA_PORT/api/tags" &> /dev/null; then
|
||||
log_success "本地Ollama服务已成功启动,端口:$OLLAMA_PORT"
|
||||
break
|
||||
fi
|
||||
echo -ne "等待Ollama服务启动... ($COUNT/$MAX_RETRIES)\r"
|
||||
sleep 1
|
||||
COUNT=$((COUNT + 1))
|
||||
done
|
||||
echo "" # 换行
|
||||
|
||||
if [ $COUNT -eq $MAX_RETRIES ]; then
|
||||
log_error "本地Ollama服务启动失败"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
log_success "本地Ollama服务地址: http://localhost:$OLLAMA_PORT"
|
||||
return 0
|
||||
}
|
||||
|
||||
# 停止Ollama服务
|
||||
stop_ollama() {
|
||||
log_info "正在停止Ollama服务..."
|
||||
|
||||
# 检查是否为远程服务
|
||||
get_ollama_base_url
|
||||
|
||||
if [ $IS_REMOTE -eq 1 ]; then
|
||||
log_info "检测到远程Ollama服务,无需在本地停止"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 检查Ollama是否已安装
|
||||
if ! command -v ollama &> /dev/null; then
|
||||
log_info "本地Ollama未安装,无需停止"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 查找并终止Ollama进程
|
||||
if pgrep -x "ollama" > /dev/null; then
|
||||
# 优先使用systemctl
|
||||
if command -v systemctl &> /dev/null; then
|
||||
sudo systemctl stop ollama
|
||||
else
|
||||
pkill -f "ollama serve"
|
||||
fi
|
||||
log_success "本地Ollama服务已停止"
|
||||
else
|
||||
log_info "本地Ollama服务未运行"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# 检查Docker是否已安装
|
||||
check_docker() {
|
||||
log_info "检查Docker环境..."
|
||||
|
||||
if ! command -v docker &> /dev/null; then
|
||||
log_error "未安装Docker,请先安装Docker"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 检查并选择可用的 Docker Compose 命令
|
||||
if detect_compose_cmd; then
|
||||
if [ "$DOCKER_COMPOSE_BIN" = "docker" ]; then
|
||||
log_info "已检测到 Docker Compose 插件 (docker compose)"
|
||||
else
|
||||
log_info "已检测到 docker-compose (v1)"
|
||||
fi
|
||||
else
|
||||
log_error "未检测到 Docker Compose(既没有 docker compose 也没有 docker-compose)。请安装其中之一。"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 检查Docker服务运行状态
|
||||
if ! docker info &> /dev/null; then
|
||||
log_error "Docker服务未运行,请启动Docker服务"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_success "Docker环境检查通过"
|
||||
return 0
|
||||
}
|
||||
|
||||
check_platform() {
|
||||
# 检测当前系统平台
|
||||
log_info "检测系统平台信息..."
|
||||
if [ "$(uname -m)" = "x86_64" ]; then
|
||||
export PLATFORM="linux/amd64"
|
||||
elif [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then
|
||||
export PLATFORM="linux/arm64"
|
||||
else
|
||||
log_warning "未识别的平台类型:$(uname -m),将使用默认平台 linux/amd64"
|
||||
export PLATFORM="linux/amd64"
|
||||
fi
|
||||
log_info "当前平台:$PLATFORM"
|
||||
}
|
||||
|
||||
# 预拉取沙箱镜像(Agent Skills 执行所需,仅拉取不启动)
|
||||
ensure_sandbox_image() {
|
||||
local sandbox_image="wechatopenai/weknora-sandbox:${WEKNORA_VERSION:-latest}"
|
||||
|
||||
# 检查本地是否已存在沙箱镜像
|
||||
if docker image inspect "$sandbox_image" &> /dev/null; then
|
||||
log_success "沙箱镜像已就绪: $sandbox_image"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "沙箱镜像 ($sandbox_image) 未检测到,正在后台拉取..."
|
||||
log_info "Agent Skills 功能依赖此镜像,首次执行前需要拉取完成"
|
||||
|
||||
# 后台拉取,不阻塞主流程
|
||||
(
|
||||
if PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD --profile sandbox pull sandbox 2>/dev/null; then
|
||||
log_success "沙箱镜像拉取完成: $sandbox_image"
|
||||
else
|
||||
log_warning "沙箱镜像拉取失败,Agent Skills 功能可能不可用"
|
||||
log_warning "可稍后手动拉取: $DOCKER_COMPOSE_BIN $DOCKER_COMPOSE_SUBCMD --profile sandbox pull sandbox"
|
||||
fi
|
||||
) &
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# 启动Docker容器
|
||||
start_docker() {
|
||||
log_info "正在启动Docker容器..."
|
||||
|
||||
# 检查Docker环境
|
||||
check_docker
|
||||
if [ $? -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 检查.env文件
|
||||
check_env_file
|
||||
|
||||
# 读取.env文件
|
||||
source "$PROJECT_ROOT/.env"
|
||||
storage_type=${STORAGE_TYPE:-local}
|
||||
|
||||
check_platform
|
||||
|
||||
# 进入项目根目录再执行docker-compose命令
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
# 启动基本服务
|
||||
log_info "启动核心服务容器..."
|
||||
# 统一通过已检测到的 Compose 命令启动
|
||||
if [ "$NO_PULL" = true ]; then
|
||||
# 不拉取镜像,使用本地镜像
|
||||
log_info "跳过镜像拉取,使用本地镜像..."
|
||||
PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD up --build -d
|
||||
else
|
||||
# 拉取最新镜像
|
||||
log_info "拉取最新镜像..."
|
||||
PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD up --pull always -d
|
||||
fi
|
||||
if [ $? -ne 0 ]; then
|
||||
log_error "Docker容器启动失败"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_success "所有Docker容器已成功启动"
|
||||
|
||||
# 显示容器状态
|
||||
log_info "当前容器状态:"
|
||||
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD ps
|
||||
|
||||
# 预拉取Sandbox镜像(Agent Skills 执行所需,仅拉取不启动)
|
||||
ensure_sandbox_image
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# 停止Docker容器
|
||||
stop_docker() {
|
||||
log_info "正在停止Docker容器..."
|
||||
|
||||
# 检查Docker环境
|
||||
check_docker
|
||||
if [ $? -ne 0 ]; then
|
||||
# 即使检查失败也尝试停止,以防万一
|
||||
log_warning "Docker环境检查失败,仍将尝试停止容器..."
|
||||
fi
|
||||
|
||||
# 进入项目根目录再执行docker-compose命令
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
# 停止所有容器
|
||||
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD down --remove-orphans
|
||||
if [ $? -ne 0 ]; then
|
||||
log_error "Docker容器停止失败"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_success "所有Docker容器已停止"
|
||||
return 0
|
||||
}
|
||||
|
||||
# 列出所有正在运行的容器
|
||||
list_containers() {
|
||||
log_info "列出所有正在运行的容器..."
|
||||
|
||||
# 检查Docker环境
|
||||
check_docker
|
||||
if [ $? -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 进入项目根目录再执行docker-compose命令
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
# 列出所有容器
|
||||
printf "%b\n" "${BLUE}当前正在运行的容器:${NC}"
|
||||
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD ps --services | sort
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# 拉取最新的Docker镜像
|
||||
pull_images() {
|
||||
log_info "正在拉取最新的Docker镜像..."
|
||||
|
||||
# 检查Docker环境
|
||||
check_docker
|
||||
if [ $? -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 检查.env文件
|
||||
check_env_file
|
||||
|
||||
# 读取.env文件
|
||||
source "$PROJECT_ROOT/.env"
|
||||
storage_type=${STORAGE_TYPE:-local}
|
||||
|
||||
check_platform
|
||||
|
||||
# 进入项目根目录再执行docker-compose命令
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
# 拉取所有镜像
|
||||
log_info "拉取所有服务的最新镜像..."
|
||||
PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD pull
|
||||
if [ $? -ne 0 ]; then
|
||||
log_error "镜像拉取失败"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 拉取 sandbox 镜像(sandbox 在 profile 中,需要单独拉取)
|
||||
log_info "拉取沙箱镜像..."
|
||||
PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD --profile sandbox pull sandbox 2>/dev/null || \
|
||||
log_warning "沙箱镜像拉取失败(非必需,跳过)"
|
||||
|
||||
log_success "所有镜像已成功拉取到最新版本"
|
||||
|
||||
# 显示拉取的镜像信息
|
||||
log_info "已拉取的镜像:"
|
||||
docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.CreatedAt}}\t{{.Size}}" | head -10
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# 重启指定容器
|
||||
restart_container() {
|
||||
local container_name="$1"
|
||||
|
||||
if [ -z "$container_name" ]; then
|
||||
log_error "未指定容器名称"
|
||||
echo "可用的容器有:"
|
||||
list_containers
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "正在重新构建并重启容器: $container_name"
|
||||
|
||||
# 检查Docker环境
|
||||
check_docker
|
||||
if [ $? -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
check_platform
|
||||
|
||||
# 进入项目根目录再执行docker-compose命令
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
# 检查容器是否存在
|
||||
if ! "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD ps --services | grep -q "^$container_name$"; then
|
||||
log_error "容器 '$container_name' 不存在或未运行"
|
||||
echo "可用的容器有:"
|
||||
list_containers
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 构建并重启容器
|
||||
log_info "正在重新构建容器 '$container_name'..."
|
||||
PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD build "$container_name"
|
||||
if [ $? -ne 0 ]; then
|
||||
log_error "容器 '$container_name' 构建失败"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "正在重启容器 '$container_name'..."
|
||||
PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD up -d --no-deps "$container_name"
|
||||
if [ $? -ne 0 ]; then
|
||||
log_error "容器 '$container_name' 重启失败"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_success "容器 '$container_name' 已成功重新构建并重启"
|
||||
return 0
|
||||
}
|
||||
|
||||
# 检查系统环境
|
||||
check_environment() {
|
||||
log_info "开始环境检查..."
|
||||
|
||||
# 检查操作系统
|
||||
OS=$(uname)
|
||||
log_info "操作系统: $OS"
|
||||
|
||||
# 检查Docker
|
||||
check_docker
|
||||
|
||||
# 检查.env文件
|
||||
check_env_file
|
||||
|
||||
get_ollama_base_url
|
||||
|
||||
if [ $IS_REMOTE -eq 1 ]; then
|
||||
log_info "检测到远程Ollama服务配置"
|
||||
if curl -s "$OLLAMA_URL/api/tags" &> /dev/null; then
|
||||
version=$(curl -s "$OLLAMA_URL/api/tags" | grep -o '"version":"[^"]*"' | cut -d'"' -f4)
|
||||
log_success "远程Ollama服务可访问,版本: $version"
|
||||
else
|
||||
log_warning "远程Ollama服务不可访问,请确认服务地址正确且已启动"
|
||||
fi
|
||||
else
|
||||
if command -v ollama &> /dev/null; then
|
||||
log_success "本地Ollama已安装"
|
||||
if curl -s "http://localhost:$OLLAMA_PORT/api/tags" &> /dev/null; then
|
||||
version=$(curl -s "http://localhost:$OLLAMA_PORT/api/tags" | grep -o '"version":"[^"]*"' | cut -d'"' -f4)
|
||||
log_success "本地Ollama服务正在运行,版本: $version"
|
||||
else
|
||||
log_warning "本地Ollama已安装但服务未运行"
|
||||
fi
|
||||
else
|
||||
log_warning "本地Ollama未安装"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 检查沙箱镜像
|
||||
log_info "检查沙箱镜像..."
|
||||
local sandbox_image="wechatopenai/weknora-sandbox:${WEKNORA_VERSION:-latest}"
|
||||
if docker image inspect "$sandbox_image" &> /dev/null; then
|
||||
log_success "沙箱镜像已就绪: $sandbox_image"
|
||||
else
|
||||
log_warning "沙箱镜像未找到: $sandbox_image (Agent Skills 功能需要此镜像)"
|
||||
log_info "可通过以下命令拉取: $0 -p 或 docker pull $sandbox_image"
|
||||
fi
|
||||
|
||||
# 检查磁盘空间
|
||||
log_info "检查磁盘空间..."
|
||||
df -h | grep -E "(Filesystem|/$)"
|
||||
|
||||
# 检查内存
|
||||
log_info "检查内存使用情况..."
|
||||
if [ "$OS" = "Darwin" ]; then
|
||||
vm_stat | perl -ne '/page size of (\d+)/ and $size=$1; /Pages free:\s*(\d+)/ and print "Free Memory: ", $1 * $size / 1048576, " MB\n"'
|
||||
else
|
||||
free -h | grep -E "(total|Mem:)"
|
||||
fi
|
||||
|
||||
# 检查CPU
|
||||
log_info "CPU信息:"
|
||||
if [ "$OS" = "Darwin" ]; then
|
||||
sysctl -n machdep.cpu.brand_string
|
||||
echo "CPU核心数: $(sysctl -n hw.ncpu)"
|
||||
else
|
||||
grep "model name" /proc/cpuinfo | head -1
|
||||
echo "CPU核心数: $(nproc)"
|
||||
fi
|
||||
|
||||
# 检查容器状态
|
||||
log_info "检查容器状态..."
|
||||
if docker info &> /dev/null; then
|
||||
docker ps -a
|
||||
else
|
||||
log_warning "无法获取容器状态,Docker可能未运行"
|
||||
fi
|
||||
|
||||
log_success "环境检查完成"
|
||||
return 0
|
||||
}
|
||||
|
||||
# 解析命令行参数
|
||||
START_OLLAMA=false
|
||||
START_DOCKER=false
|
||||
STOP_SERVICES=false
|
||||
CHECK_ENVIRONMENT=false
|
||||
LIST_CONTAINERS=false
|
||||
RESTART_CONTAINER=false
|
||||
PULL_IMAGES=false
|
||||
NO_PULL=false
|
||||
CONTAINER_NAME=""
|
||||
|
||||
# 没有参数时默认启动所有服务
|
||||
if [ $# -eq 0 ]; then
|
||||
START_OLLAMA=true
|
||||
START_DOCKER=true
|
||||
fi
|
||||
|
||||
while [ "$1" != "" ]; do
|
||||
case $1 in
|
||||
-h | --help ) show_help
|
||||
;;
|
||||
-o | --ollama ) START_OLLAMA=true
|
||||
;;
|
||||
-d | --docker ) START_DOCKER=true
|
||||
;;
|
||||
-a | --all ) START_OLLAMA=true
|
||||
START_DOCKER=true
|
||||
;;
|
||||
-s | --stop ) STOP_SERVICES=true
|
||||
;;
|
||||
-c | --check ) CHECK_ENVIRONMENT=true
|
||||
;;
|
||||
-l | --list ) LIST_CONTAINERS=true
|
||||
;;
|
||||
-p | --pull ) PULL_IMAGES=true
|
||||
;;
|
||||
--no-pull ) NO_PULL=true
|
||||
START_OLLAMA=true
|
||||
START_DOCKER=true
|
||||
;;
|
||||
-r | --restart ) RESTART_CONTAINER=true
|
||||
CONTAINER_NAME="$2"
|
||||
shift
|
||||
;;
|
||||
-v | --version ) show_version
|
||||
;;
|
||||
* ) log_error "未知选项: $1"
|
||||
show_help
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# 执行环境检查
|
||||
if [ "$CHECK_ENVIRONMENT" = true ]; then
|
||||
check_environment
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# 列出所有容器
|
||||
if [ "$LIST_CONTAINERS" = true ]; then
|
||||
list_containers
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# 拉取最新镜像
|
||||
if [ "$PULL_IMAGES" = true ]; then
|
||||
pull_images
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# 重启指定容器
|
||||
if [ "$RESTART_CONTAINER" = true ]; then
|
||||
restart_container "$CONTAINER_NAME"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# 执行服务操作
|
||||
if [ "$STOP_SERVICES" = true ]; then
|
||||
# 停止服务
|
||||
stop_ollama
|
||||
OLLAMA_RESULT=$?
|
||||
|
||||
stop_docker
|
||||
DOCKER_RESULT=$?
|
||||
|
||||
# 显示总结
|
||||
echo ""
|
||||
log_info "=== 停止结果 ==="
|
||||
if [ $OLLAMA_RESULT -eq 0 ]; then
|
||||
log_success "✓ Ollama服务已停止"
|
||||
else
|
||||
log_error "✗ Ollama服务停止失败"
|
||||
fi
|
||||
|
||||
if [ $DOCKER_RESULT -eq 0 ]; then
|
||||
log_success "✓ Docker容器已停止"
|
||||
else
|
||||
log_error "✗ Docker容器停止失败"
|
||||
fi
|
||||
|
||||
log_success "服务停止完成。"
|
||||
else
|
||||
# 启动服务
|
||||
OLLAMA_RESULT=1
|
||||
DOCKER_RESULT=1
|
||||
if [ "$START_OLLAMA" = true ]; then
|
||||
start_ollama
|
||||
OLLAMA_RESULT=$?
|
||||
fi
|
||||
|
||||
if [ "$START_DOCKER" = true ]; then
|
||||
start_docker
|
||||
DOCKER_RESULT=$?
|
||||
fi
|
||||
|
||||
# 显示总结
|
||||
echo ""
|
||||
log_info "=== 启动结果 ==="
|
||||
if [ "$START_OLLAMA" = true ]; then
|
||||
if [ $OLLAMA_RESULT -eq 0 ]; then
|
||||
log_success "✓ Ollama服务已启动"
|
||||
else
|
||||
log_error "✗ Ollama服务启动失败"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$START_DOCKER" = true ]; then
|
||||
if [ $DOCKER_RESULT -eq 0 ]; then
|
||||
log_success "✓ Docker容器已启动"
|
||||
else
|
||||
log_error "✗ Docker容器启动失败"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$START_OLLAMA" = true ] && [ "$START_DOCKER" = true ]; then
|
||||
if [ $OLLAMA_RESULT -eq 0 ] && [ $DOCKER_RESULT -eq 0 ]; then
|
||||
log_success "所有服务启动完成,可通过以下地址访问:"
|
||||
printf "%b\n" "${GREEN} - 前端界面: http://localhost:${FRONTEND_PORT:-80}${NC}"
|
||||
printf "%b\n" "${GREEN} - API接口: http://localhost:${APP_PORT:-8080}${NC}"
|
||||
echo ""
|
||||
log_info "正在持续输出容器日志(按 Ctrl+C 退出日志,容器不会停止)..."
|
||||
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD logs app docreader postgres --since=10s -f
|
||||
else
|
||||
log_error "部分服务启动失败,请检查日志并修复问题"
|
||||
fi
|
||||
elif [ "$START_OLLAMA" = true ] && [ $OLLAMA_RESULT -eq 0 ]; then
|
||||
log_success "Ollama服务启动完成,可通过以下地址访问:"
|
||||
printf "%b\n" "${GREEN} - Ollama API: http://localhost:$OLLAMA_PORT${NC}"
|
||||
elif [ "$START_DOCKER" = true ] && [ $DOCKER_RESULT -eq 0 ]; then
|
||||
log_success "Docker容器启动完成,可通过以下地址访问:"
|
||||
printf "%b\n" "${GREEN} - 前端界面: http://localhost:${FRONTEND_PORT:-80}${NC}"
|
||||
printf "%b\n" "${GREEN} - API接口: http://localhost:${APP_PORT:-8080}${NC}"
|
||||
echo ""
|
||||
log_info "正在持续输出容器日志(按 Ctrl+C 退出日志,容器不会停止)..."
|
||||
"$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD logs app docreader postgres --since=10s -f
|
||||
fi
|
||||
fi
|
||||
|
||||
exit 0
|
||||
Executable
+178
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
#
|
||||
# 本地测试 Homebrew Formula
|
||||
#
|
||||
# 流程:打包 → 创建本地 tap → 写入 Formula → brew install → 验证
|
||||
#
|
||||
# 用法:
|
||||
# ./scripts/test-homebrew.sh # 完整测试(含前端构建)
|
||||
# SKIP_FRONTEND=1 ./scripts/test-homebrew.sh # 跳过前端构建
|
||||
# SKIP_BUILD=1 ./scripts/test-homebrew.sh # 跳过构建(使用已有 tarball)
|
||||
#
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
cd "${ROOT_DIR}"
|
||||
|
||||
TAP_NAME="weknora/test"
|
||||
FORMULA_NAME="weknora-lite-test"
|
||||
VERSION="test"
|
||||
GOOS=$(go env GOOS)
|
||||
GOARCH=$(go env GOARCH)
|
||||
ARCHIVE="WeKnora-lite_${VERSION}_${GOOS}_${GOARCH}"
|
||||
TARBALL="${ROOT_DIR}/dist/${ARCHIVE}.tar.gz"
|
||||
|
||||
# ── Step 1: Build package ──
|
||||
if [ "${SKIP_BUILD:-}" != "1" ]; then
|
||||
echo "=== Step 1: Build local package ==="
|
||||
SKIP_FRONTEND="${SKIP_FRONTEND:-0}" ./scripts/package-lite.sh "${VERSION}"
|
||||
else
|
||||
echo "=== Step 1: Skipped (SKIP_BUILD=1) ==="
|
||||
fi
|
||||
|
||||
if [ ! -f "${TARBALL}" ]; then
|
||||
echo "Error: tarball not found at ${TARBALL}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SHA=$(shasum -a 256 "${TARBALL}" | awk '{print $1}')
|
||||
echo ""
|
||||
echo " Tarball: ${TARBALL}"
|
||||
echo " SHA256: ${SHA}"
|
||||
|
||||
# ── Step 2: Create local tap ──
|
||||
echo ""
|
||||
echo "=== Step 2: Set up local tap ==="
|
||||
|
||||
# Create tap if it doesn't exist
|
||||
TAP_DIR="$(brew --repository)/Library/Taps/weknora/homebrew-test"
|
||||
if [ ! -d "${TAP_DIR}" ]; then
|
||||
mkdir -p "${TAP_DIR}/Formula"
|
||||
(cd "${TAP_DIR}" && git init -q && git commit --allow-empty -m "init" -q)
|
||||
echo " Created tap: ${TAP_NAME}"
|
||||
else
|
||||
echo " Tap already exists: ${TAP_NAME}"
|
||||
fi
|
||||
|
||||
# ── Step 3: Write Formula into tap ──
|
||||
echo ""
|
||||
echo "=== Step 3: Generate Formula ==="
|
||||
|
||||
cat > "${TAP_DIR}/Formula/${FORMULA_NAME}.rb" << RUBY
|
||||
class WeknoraLiteTest < Formula
|
||||
desc "WeKnora Lite (local test)"
|
||||
homepage "https://github.com/Tencent/WeKnora"
|
||||
version "${VERSION}"
|
||||
license "Apache-2.0"
|
||||
|
||||
url "file://${TARBALL}"
|
||||
sha256 "${SHA}"
|
||||
|
||||
def install
|
||||
libexec.install "WeKnora-lite"
|
||||
pkgshare.install "web" if File.directory?("web")
|
||||
pkgshare.install "config" if File.directory?("config")
|
||||
pkgshare.install ".env.lite.example"
|
||||
if File.directory?("migrations")
|
||||
pkgshare.install "migrations"
|
||||
end
|
||||
|
||||
(bin/"weknora-lite-test").write <<~SH
|
||||
#!/bin/bash
|
||||
CONFIG_DIR="\\\${WEKNORA_CONFIG_DIR:-\\\${XDG_CONFIG_HOME:-\\\$HOME/.config}/weknora-test}"
|
||||
DATA_DIR="\\\${WEKNORA_DATA_DIR:-\\\${XDG_DATA_HOME:-\\\$HOME/.local/share}/weknora-test}"
|
||||
|
||||
mkdir -p "\\\$DATA_DIR/files" "\\\$CONFIG_DIR/config" 2>/dev/null
|
||||
|
||||
if [ ! -f "\\\$CONFIG_DIR/config/config.yaml" ]; then
|
||||
cp -r "#{pkgshare}/config/" "\\\$CONFIG_DIR/config/"
|
||||
fi
|
||||
|
||||
if [ ! -d "\\\$CONFIG_DIR/migrations" ] && [ -d "#{pkgshare}/migrations" ]; then
|
||||
ln -sf "#{pkgshare}/migrations" "\\\$CONFIG_DIR/migrations"
|
||||
fi
|
||||
|
||||
if [ ! -f "\\\$CONFIG_DIR/.env.lite" ]; then
|
||||
cp "#{pkgshare}/.env.lite.example" "\\\$CONFIG_DIR/.env.lite"
|
||||
sed -i'' -e "s|DB_PATH=.*|DB_PATH=\\\$DATA_DIR/weknora.db|" "\\\$CONFIG_DIR/.env.lite"
|
||||
sed -i'' -e "s|LOCAL_STORAGE_BASE_DIR=.*|LOCAL_STORAGE_BASE_DIR=\\\$DATA_DIR/files|" "\\\$CONFIG_DIR/.env.lite"
|
||||
echo ""
|
||||
echo "已创建配置文件: \\\$CONFIG_DIR/.env.lite"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
set -a
|
||||
source "\\\$CONFIG_DIR/.env.lite"
|
||||
set +a
|
||||
|
||||
export DB_PATH="\\\${DB_PATH:-\\\$DATA_DIR/weknora.db}"
|
||||
export LOCAL_STORAGE_BASE_DIR="\\\${LOCAL_STORAGE_BASE_DIR:-\\\$DATA_DIR/files}"
|
||||
export WEKNORA_WEB_DIR="\\\${WEKNORA_WEB_DIR:-#{pkgshare}/web}"
|
||||
|
||||
cd "\\\$CONFIG_DIR"
|
||||
exec "#{libexec}/WeKnora-lite" "\\\$@"
|
||||
SH
|
||||
end
|
||||
|
||||
def post_install
|
||||
(var/"weknora-test").mkpath
|
||||
(var/"log").mkpath
|
||||
end
|
||||
|
||||
service do
|
||||
run [bin/"weknora-lite-test"]
|
||||
keep_alive true
|
||||
working_dir var/"weknora-test"
|
||||
log_path var/"log/weknora-lite-test.log"
|
||||
error_log_path var/"log/weknora-lite-test.log"
|
||||
end
|
||||
|
||||
test do
|
||||
assert_predicate bin/"weknora-lite-test", :executable?
|
||||
end
|
||||
end
|
||||
RUBY
|
||||
|
||||
(cd "${TAP_DIR}" && git add -A && git commit -m "update formula" -q --allow-empty)
|
||||
echo " Formula written to: ${TAP_DIR}/Formula/${FORMULA_NAME}.rb"
|
||||
|
||||
# ── Step 4: Install ──
|
||||
echo ""
|
||||
echo "=== Step 4: Install ==="
|
||||
brew reinstall "${TAP_NAME}/${FORMULA_NAME}" 2>&1 || brew install "${TAP_NAME}/${FORMULA_NAME}"
|
||||
|
||||
# ── Step 5: Verify ──
|
||||
echo ""
|
||||
echo "=== Step 5: Verify ==="
|
||||
echo ""
|
||||
echo " which:"
|
||||
which weknora-lite-test || true
|
||||
echo ""
|
||||
echo " Installed files:"
|
||||
brew list "${TAP_NAME}/${FORMULA_NAME}"
|
||||
echo ""
|
||||
echo " Test paths (isolated from production):"
|
||||
echo " Config: ~/.config/weknora-test/.env.lite"
|
||||
echo " Data: ~/.local/share/weknora-test/"
|
||||
|
||||
echo ""
|
||||
echo "=== Done ==="
|
||||
echo ""
|
||||
echo "前台运行:"
|
||||
echo " weknora-lite-test"
|
||||
echo ""
|
||||
echo "后台服务:"
|
||||
echo " brew services start ${TAP_NAME}/${FORMULA_NAME}"
|
||||
echo " brew services info ${TAP_NAME}/${FORMULA_NAME}"
|
||||
echo " brew services stop ${TAP_NAME}/${FORMULA_NAME}"
|
||||
echo ""
|
||||
echo "日志:"
|
||||
echo " $(brew --prefix)/var/log/weknora-lite-test.log"
|
||||
echo ""
|
||||
echo "卸载测试:"
|
||||
echo " brew services stop ${FORMULA_NAME} 2>/dev/null"
|
||||
echo " brew uninstall ${FORMULA_NAME}"
|
||||
echo " brew untap ${TAP_NAME}"
|
||||
echo " rm -rf ~/.config/weknora-test ~/.local/share/weknora-test"
|
||||
@@ -0,0 +1,129 @@
|
||||
import copy
|
||||
import sqlite3
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from weknora_identity_etl import ADMIN_TARGET_ID
|
||||
from weknora_operations_etl import (KINDS, notification_owner, operations_plan, operations_sql,
|
||||
reconcile, redact, validate_plan)
|
||||
|
||||
USER = "63ce184e-5ab6-5240-8269-14026d501a68"
|
||||
AGENT = "da4d5f8f-e1c4-574e-b34a-8a324615b603"
|
||||
KB = "237fa6a0-1e49-58c8-8185-4191d14a9094"
|
||||
NOW = "2026-09-15T12:00:00"
|
||||
|
||||
|
||||
def fixture():
|
||||
rows = {kind: [] for kind in (*KINDS, "campaign", "segment")}
|
||||
rows["pointaccount"] = [{"id": 1, "user_id": 1, "balance_points": 110, "updated_at": NOW}]
|
||||
rows["pointtransaction"] = [{"id": 1, "user_id": 1, "delta_points": 100, "kind": "credit", "reason": "grant",
|
||||
"operator_user_id": 1, "metadata": {}, "created_at": NOW},
|
||||
{"id": 2, "user_id": 1, "delta_points": -10, "kind": "debit", "reason": "generation",
|
||||
"operator_user_id": 1, "metadata": {"usage_event_id": 1}, "created_at": NOW}]
|
||||
rows["usageevent"] = [{"id": 1, "user_id": 1, "agent_id": 2, "provider_id": 1, "model": "test",
|
||||
"unit": "token", "quantity": 1, "currency": "USD", "amount_minor": 5000,
|
||||
"llm_call_id": 1, "campaign_id": 1, "run_id": 1, "segment_id": 1,
|
||||
"input_quantity": 1, "output_quantity": 0, "service_type": "llm",
|
||||
"status": "measured", "metadata": {}, "created_at": NOW}]
|
||||
rows["campaign"] = [{"id": 1, "owner_user_id": 1}, {"id": 2, "owner_user_id": 8}]
|
||||
rows["segment"] = [{"id": 1, "campaign_id": 1}, {"id": 2, "campaign_id": 1}, {"id": 42, "campaign_id": 2}]
|
||||
return rows
|
||||
|
||||
|
||||
def plan(rows=None):
|
||||
rows = rows or fixture()
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("PRAGMA query_only=ON")
|
||||
try:
|
||||
with patch("weknora_operations_etl._read", side_effect=lambda _db, kind: copy.deepcopy(rows[kind])):
|
||||
result = operations_plan(db, {"1": ADMIN_TARGET_ID, "8": USER}, {"2": AGENT}, {"3": KB}, {}, {})
|
||||
finally:
|
||||
db.close()
|
||||
result["source_snapshot_sha256"] = "a" * 64
|
||||
return result
|
||||
|
||||
|
||||
class OperationsTests(unittest.TestCase):
|
||||
def test_mismatch_preserves_authoritative_balance_without_credit(self):
|
||||
result = plan()
|
||||
self.assertEqual(result["reconciliation"][0]["difference"], 20)
|
||||
native = [i for i in result["items"] if i["target_table"] == "workbench_balances"]
|
||||
self.assertEqual(native[0]["native_snapshot"]["balance"], 110)
|
||||
self.assertEqual(native[0]["history_payload"]["balance_reconciliation"]["difference"], 20)
|
||||
self.assertEqual(len([i for i in result["items"] if i["target_table"] == "workbench_transactions"]), 2)
|
||||
validate_plan(result)
|
||||
|
||||
def test_missing_account_does_not_create_credit_or_balance(self):
|
||||
rows = fixture()
|
||||
rows["pointaccount"] = []
|
||||
result = plan(rows)
|
||||
self.assertEqual(result["reconciliation"][0]["reason"], "missing_source_account")
|
||||
self.assertFalse(any(i["target_table"] == "workbench_balances" for i in result["items"]))
|
||||
self.assertEqual(sum(i["target_table"] == "workbench_transactions" for i in result["items"]), 2)
|
||||
|
||||
def test_points_use_actual_debits_not_currency_or_token_estimates(self):
|
||||
usage = next(i for i in plan()["items"] if i["source_kind"] == "usageevent")
|
||||
self.assertEqual(usage["native_snapshot"]["points"], 10)
|
||||
self.assertEqual(usage["native_snapshot"]["metadata"]["legacy_amount_minor"], 5000)
|
||||
|
||||
def test_storage_quantity_is_not_tokens(self):
|
||||
rows = fixture()
|
||||
rows["usageevent"][0].update(unit="byte", input_quantity=999, output_quantity=999)
|
||||
usage = next(i["native_snapshot"] for i in plan(rows)["items"] if i["source_kind"] == "usageevent")
|
||||
self.assertEqual((usage["input_tokens"], usage["output_tokens"]), (0, 0))
|
||||
|
||||
def test_usage_ownerless_is_admin_archive_only(self):
|
||||
rows = fixture()
|
||||
rows["usageevent"][0]["user_id"] = 0
|
||||
usage = next(i for i in plan(rows)["items"] if i["source_kind"] == "usageevent")
|
||||
self.assertEqual(usage["target_table"], "")
|
||||
self.assertEqual(usage["owner_id"], "")
|
||||
|
||||
def test_ambiguous_notifications_not_assigned_by_numeric_reference(self):
|
||||
ids = {"1": ADMIN_TARGET_ID, "8": USER}
|
||||
owner, reason = notification_owner({"type": "failure", "ref_id": 2}, {1: 1, 2: 8}, {2: 1}, ids)
|
||||
self.assertEqual(owner, "")
|
||||
self.assertTrue(reason)
|
||||
owner, reason = notification_owner({"type": "failure", "ref_id": 42}, {1: 1, 2: 8}, {42: 2}, ids)
|
||||
self.assertEqual(owner, USER)
|
||||
|
||||
def test_alerts_are_never_broadcast(self):
|
||||
owner, _ = notification_owner({"type": "alert", "ref_id": 0}, {}, {}, {"1": ADMIN_TARGET_ID})
|
||||
self.assertEqual(owner, "")
|
||||
|
||||
def test_redacted_native_log_preserves_original_in_private_plan(self):
|
||||
rows = fixture()
|
||||
secret = "sk-123456789abcdefghijklmnop"
|
||||
rows["systemlog"] = [{"id": 1, "level": "WARNING", "source": "provider", "message": "api_key=" + secret,
|
||||
"trace_id": "", "status": 500, "created_at": NOW}]
|
||||
item = next(i for i in plan(rows)["items"] if i["source_kind"] == "systemlog")
|
||||
self.assertIn(secret, item["source_snapshot"]["message"])
|
||||
self.assertNotIn(secret, item["native_snapshot"]["message"])
|
||||
self.assertEqual(item["native_snapshot"]["level"], "warning")
|
||||
self.assertEqual(redact({"api_key": secret})["api_key"], "[REDACTED]")
|
||||
|
||||
def test_native_balance_tampering_rejected(self):
|
||||
result = plan()
|
||||
result["items"][0]["native_snapshot"]["balance"] += 20
|
||||
with self.assertRaisesRegex(ValueError, "balance"):
|
||||
validate_plan(result)
|
||||
|
||||
def test_native_owner_tampering_rejected(self):
|
||||
result = plan()
|
||||
result["items"][0]["native_snapshot"]["user_id"] = USER
|
||||
with self.assertRaisesRegex(ValueError, "owner"):
|
||||
validate_plan(result)
|
||||
|
||||
def test_duplicate_accounts_rejected(self):
|
||||
with self.assertRaisesRegex(ValueError, "Duplicate"):
|
||||
reconcile([{"user_id": 1, "balance_points": 0}] * 2, [])
|
||||
|
||||
def test_sql_requires_snapshot_provenance(self):
|
||||
result = plan()
|
||||
result.pop("source_snapshot_sha256")
|
||||
with self.assertRaisesRegex(ValueError, "fingerprint"):
|
||||
operations_sql(result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,150 @@
|
||||
import unittest
|
||||
|
||||
from weknora_access_etl import (ACCESS_TABLES, ADMIN_TARGET_ID, digest, make_plan, migration_sql,
|
||||
role_permissions, stable, TARGET_DATABASE, TARGET_TENANT)
|
||||
from apply_weknora_access import rollback_sql
|
||||
|
||||
|
||||
class AccessMigrationTests(unittest.TestCase):
|
||||
def test_editor_covers_grouped_operations_only(self):
|
||||
permissions, _ = role_permissions({"permissions": [
|
||||
"read", "feature:tasks", "tasks:read", "tasks:write", "tasks:review", "tasks:deliver",
|
||||
"feature:agents", "agents:read", "agents:write", "memories:read", "memories:write", "memories:review",
|
||||
"feature:flows", "flows:read", "feature:knowledge", "kb:read", "kb:write",
|
||||
"feature:digital-humans", "digital_humans:read", "logs:read", "usage:read"]})
|
||||
expected = [module + "." + action for module in
|
||||
("content", "flows", "agents", "knowledge", "digital-humans") for action in ("read", "write")]
|
||||
self.assertEqual(permissions, sorted(expected + ["content.review", "content.deliver"]))
|
||||
|
||||
def test_generic_read_does_not_grant_any_module(self):
|
||||
self.assertEqual(role_permissions({"permissions": ["read"]})[0], [])
|
||||
|
||||
def test_legacy_reviewer_cannot_gain_export_or_agent_management(self):
|
||||
permissions, notes = role_permissions({"permissions": ["audit:read", "read", "review"]})
|
||||
self.assertEqual(permissions, ["content.read", "content.review"])
|
||||
self.assertNotIn("content_read_withheld_because_native_read_also_exposes_export", notes)
|
||||
|
||||
def test_content_delivery_does_not_grant_generation(self):
|
||||
self.assertEqual(role_permissions({"permissions": ["tasks:review", "tasks:deliver"]})[0],
|
||||
["content.deliver", "content.read", "content.review"])
|
||||
self.assertEqual(role_permissions({"permissions": ["tasks:review", "tasks:write"]})[0],
|
||||
["content.read", "content.review", "content.write"])
|
||||
self.assertEqual(role_permissions({"permissions": ["tasks:read", "tasks:deliver"]})[0], [])
|
||||
|
||||
def test_agent_write_does_not_infer_memory_review(self):
|
||||
permissions, _ = role_permissions({"permissions": ["agents:read", "agents:write", "memories:write"]})
|
||||
self.assertIn("agents.read", permissions)
|
||||
self.assertNotIn("agents.write", permissions)
|
||||
|
||||
def test_memory_review_does_not_infer_agent_write(self):
|
||||
self.assertEqual(role_permissions({"permissions": ["feature:memories", "memories:review"]})[0], [])
|
||||
|
||||
def test_old_config_management_does_not_grant_all_settings_or_access(self):
|
||||
permissions, _ = role_permissions({"permissions": ["config:write", "admin:manage", "users:manage"]})
|
||||
self.assertNotIn("settings.write", permissions)
|
||||
self.assertNotIn("access.write", permissions)
|
||||
|
||||
def test_source_admin_does_not_create_global_admin(self):
|
||||
self.assertEqual(role_permissions({"permissions": ["*"]})[0], [])
|
||||
|
||||
def test_stable_role_identity(self):
|
||||
self.assertEqual(stable("editor"), stable("editor"))
|
||||
self.assertNotEqual(stable("editor"), stable("viewer"))
|
||||
|
||||
def plan(self):
|
||||
value = {"target_database": TARGET_DATABASE, "tenant_id": TARGET_TENANT,
|
||||
"grants": [], "roles": [], "assignments": [], "publications": [], "source_role_definitions": []}
|
||||
value["plan_sha256"] = digest(value)
|
||||
return value
|
||||
|
||||
def test_sql_defaults_to_rollback_and_does_not_update_or_delete_native_rows(self):
|
||||
sql = migration_sql(self.plan())
|
||||
self.assertTrue(sql.rstrip().endswith("ROLLBACK;"))
|
||||
self.assertNotIn("UPDATE users", sql)
|
||||
self.assertNotIn("UPDATE tenant_members", sql)
|
||||
self.assertNotIn("DELETE FROM", sql)
|
||||
self.assertIn("m.status='suspended'", sql)
|
||||
self.assertIn("Native resource authorization already exists", sql)
|
||||
self.assertIn("change->'legacy_source_spec'=saved", sql)
|
||||
self.assertIn("change->'mapped_permissions'=spec->'permissions'", sql)
|
||||
|
||||
def test_tampered_or_wrong_target_plan_is_rejected(self):
|
||||
plan = self.plan()
|
||||
plan["tenant_id"] = 2
|
||||
with self.assertRaises(ValueError):
|
||||
migration_sql(plan)
|
||||
plan = self.plan()
|
||||
plan["roles"] = [{"permissions": ["access.write"]}]
|
||||
with self.assertRaises(ValueError):
|
||||
migration_sql(plan)
|
||||
|
||||
def test_explicit_grants_are_not_inferred(self):
|
||||
plan = self.plan()
|
||||
plan["grants"] = [{"kind": "agents"}]
|
||||
plan["plan_sha256"] = digest({k: v for k, v in plan.items() if k != "plan_sha256"})
|
||||
with self.assertRaises(ValueError):
|
||||
migration_sql(plan)
|
||||
|
||||
def source_and_target(self):
|
||||
role = {"role": "viewer", "organization_id": 1, "label": "Viewer", "permissions": ["read"]}
|
||||
source = {"roles": [role], "users": [{"id": 1, "enabled": 1},
|
||||
{"id": 3, "enabled": 1, "organization_id": 1, "role": "viewer"}],
|
||||
"grants": {}, "memory_count": 0,
|
||||
"resources": {key: [] for key in ("agent", "agentskill", "canvas", "knowledgebase", "digitalhuman")}}
|
||||
identity = {"source_user_id": 3, "target_user_id": "user3", "user_tenant": TARGET_TENANT,
|
||||
"is_active": False, "is_system_admin": False, "can_access_all_tenants": False, "user_deleted": None,
|
||||
"source_snapshot": {"role": "viewer"}, "memberships": [
|
||||
{"tenant_id": TARGET_TENANT, "role": "contributor", "status": "suspended", "deleted_at": None}]}
|
||||
target = {"database": TARGET_DATABASE, "organization": {"source_role_definitions": [role]},
|
||||
"identities": [{"source_user_id": 1, "target_user_id": ADMIN_TARGET_ID}, identity],
|
||||
"resources": [], "asset_mappings": [], "flow_mappings": [], "access_tables": list(ACCESS_TABLES)}
|
||||
return source, target
|
||||
|
||||
def build(self, source, target):
|
||||
return make_plan(source, target, {"tenant_id": TARGET_TENANT}, "snapshot")
|
||||
|
||||
def test_explicit_deny_all_assignment_is_created_for_viewer(self):
|
||||
plan = self.build(*self.source_and_target())
|
||||
self.assertEqual(len(plan["assignments"]), 1)
|
||||
self.assertEqual(plan["roles"][0]["permissions"], [])
|
||||
self.assertEqual(plan["assignments"][0]["role_id"], stable("viewer"))
|
||||
|
||||
def test_active_migrated_account_is_rejected(self):
|
||||
source, target = self.source_and_target()
|
||||
target["identities"][1]["is_active"] = True
|
||||
with self.assertRaisesRegex(ValueError, "inactive and suspended"):
|
||||
self.build(source, target)
|
||||
|
||||
def test_cross_tenant_membership_is_rejected(self):
|
||||
source, target = self.source_and_target()
|
||||
target["identities"][1]["memberships"].append(
|
||||
{"tenant_id": 9, "role": "viewer", "status": "active", "deleted_at": None})
|
||||
with self.assertRaisesRegex(ValueError, "inactive and suspended"):
|
||||
self.build(source, target)
|
||||
|
||||
def test_missing_enabled_identity_is_rejected(self):
|
||||
source, target = self.source_and_target()
|
||||
target["identities"].pop()
|
||||
with self.assertRaisesRegex(ValueError, "lacks a durable identity"):
|
||||
self.build(source, target)
|
||||
|
||||
def test_source_permission_drift_is_rejected(self):
|
||||
source, target = self.source_and_target()
|
||||
target["organization"]["source_role_definitions"] = []
|
||||
with self.assertRaisesRegex(ValueError, "role definitions differ"):
|
||||
self.build(source, target)
|
||||
|
||||
def test_rollback_is_scoped_guarded_and_rehearsal_by_default(self):
|
||||
from weknora_access_etl import LEDGER
|
||||
state = {table: [] for table in (*ACCESS_TABLES, LEDGER)}
|
||||
sql = rollback_sql(self.plan(), state, state)
|
||||
self.assertTrue(sql.rstrip().endswith("ROLLBACK;"))
|
||||
self.assertIn("Native edits or missing rows", sql)
|
||||
self.assertIn("New references to migrated access rows", sql)
|
||||
self.assertIn("Account activation changed", sql)
|
||||
self.assertNotIn("DELETE FROM users", sql)
|
||||
self.assertNotIn("DELETE FROM tenant_members", sql)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,318 @@
|
||||
"""Read-only asset plans, lossless text and native SQL migration contracts."""
|
||||
import copy
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from weknora_asset_etl import (AGENT_COLUMNS, SKILL_COLUMNS, TEMPLATE_COLUMNS, ADMIN_TARGET_ID,
|
||||
LEDGER, BINDING_LEDGER, asset_id, asset_plan, asset_sql, safe_asset_summary)
|
||||
from weknora_model_etl import model_plan, model_id
|
||||
|
||||
USER_ID = "ab043a7d-d10e-58fe-8417-3fd2cbab5f77"
|
||||
KB_ID = "7e6ac485-f766-4d58-ae38-2f0456a022ef"
|
||||
|
||||
|
||||
class AssetTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.db = sqlite3.connect(":memory:")
|
||||
self.addCleanup(self.db.close)
|
||||
integer_columns = {"id", "model_provider_id", "model_catalog_id", "prompt_template_id", "rules_version",
|
||||
"builtin", "organization_id", "owner_user_id", "digital_human_id", "enabled", "version"}
|
||||
for table, columns in (("agent", AGENT_COLUMNS), ("agentskill", SKILL_COLUMNS), ("agentprompttemplate", TEMPLATE_COLUMNS)):
|
||||
self.db.execute("CREATE TABLE " + table + " (" + ",".join(
|
||||
column + (" INTEGER" if column in integer_columns else " TEXT") for column in columns) + ")")
|
||||
self.db.executescript("""
|
||||
CREATE TABLE agentskillbinding (id INTEGER,agent_id INTEGER,skill_id INTEGER);
|
||||
CREATE TABLE agentgrant (id INTEGER,agent_id INTEGER,user_id INTEGER,permission TEXT,enabled INTEGER,created_at TEXT);
|
||||
CREATE TABLE agentskillgrant (id INTEGER,skill_id INTEGER,user_id INTEGER,permission TEXT,enabled INTEGER,created_at TEXT);
|
||||
CREATE TABLE modelprovider (id INTEGER,name TEXT,base_url TEXT,api_key TEXT,models TEXT,
|
||||
default_model TEXT,builtin INTEGER,enabled INTEGER,organization_id INTEGER);
|
||||
CREATE TABLE modelcatalog (id INTEGER,provider_id INTEGER,model_name TEXT,display_name TEXT,
|
||||
capabilities TEXT,capability_status TEXT,enabled INTEGER,builtin INTEGER,rate_version TEXT,
|
||||
input_rate REAL,output_rate REAL);
|
||||
CREATE TABLE appsetting (key TEXT,value TEXT);
|
||||
INSERT INTO modelprovider VALUES (3,'deepseek','https://api.deepseek.com/v1','DO-NOT-READ-SECRET',
|
||||
'["deepseek-flash","deepseek-v4-pro"]','deepseek-flash',0,1,1);
|
||||
INSERT INTO modelcatalog VALUES (6,3,'deepseek-v4-pro','Pro','[]','untested',1,0,'r1',1,2);
|
||||
INSERT INTO modelcatalog VALUES (7,3,'deepseek-flash','Flash','[]','untested',1,0,'r1',1,2);
|
||||
INSERT INTO appsetting VALUES ('default_model_provider_id','3');
|
||||
""")
|
||||
self.agent(1, owner_user_id=0, builtin=1, visibility="global", kb_ids="[3]")
|
||||
self.agent(19, owner_user_id=3, status="disabled", visibility="private")
|
||||
self.agent(22, owner_user_id=3, visibility="private")
|
||||
self.skill(1, owner_user_id=0, builtin=1, visibility="global")
|
||||
self.skill(5, owner_user_id=3, visibility="private")
|
||||
self.skill(9, owner_user_id=1, visibility="private")
|
||||
self.db.executemany("INSERT INTO agentskillbinding VALUES (?,?,?)",
|
||||
[(101, 1, 1), (102, 1, 5), (103, 1, 9), (104, 19, 1), (105, 22, 1), (106, 22, 5)])
|
||||
template = {key: "" for key in TEMPLATE_COLUMNS}
|
||||
template.update(id=1, name="Source template", desc="Preserve template", version=1, enabled=1, builtin=1,
|
||||
organization_id=1, owner_user_id=0, visibility="global", constraints='["constraint"]',
|
||||
examples='[]', output_schema='{}', role_prompt="original-template-body",
|
||||
task_template="{{runtime_input}}", created_at="2026-09-15", updated_at="2026-09-15")
|
||||
self.insert("agentprompttemplate", TEMPLATE_COLUMNS, template)
|
||||
self.identities, self.kbs = {1: ADMIN_TARGET_ID, 3: USER_ID}, {3: KB_ID}
|
||||
|
||||
def insert(self, table, columns, values):
|
||||
self.db.execute("INSERT INTO " + table + " (" + ",".join(columns) + ") VALUES (" +
|
||||
",".join("?" for _ in columns) + ")", [values[k] for k in columns])
|
||||
|
||||
def agent(self, source_id, **overrides):
|
||||
values = {key: None for key in AGENT_COLUMNS}
|
||||
values.update(id=source_id, name=f"Agent {source_id}", role="production", desc="original description",
|
||||
content_tags='["source-tag"]', kb_ids='[]', fusion_agent_ids='[]', model_provider_id=None,
|
||||
model_name='deepseek-chat', model_catalog_id=None, prompt_template_id=1,
|
||||
allowed_tools='["knowledge_search","read_agent_memory"]', rules=" Original rules\n\n",
|
||||
rules_version=2, status="active", builtin=0, organization_id=1, owner_user_id=1,
|
||||
visibility="private", created_at="2026-09-15 00:00:00",
|
||||
runtime_snapshot='{"primary":{"rules":"frozen-original-rules"}}', persona_json='{"voice":"plain"}',
|
||||
digital_human_id=None, kb_category_ids='{}')
|
||||
values.update(overrides)
|
||||
self.insert("agent", AGENT_COLUMNS, values)
|
||||
|
||||
def skill(self, source_id, **overrides):
|
||||
values = dict(id=source_id, name=f"Skill {source_id}", desc="source description",
|
||||
instructions=" Original skill body\n\n", enabled=1, builtin=0, version=2,
|
||||
organization_id=1, owner_user_id=1, visibility="private", created_at="2026-09-15 00:00:00")
|
||||
values.update(overrides)
|
||||
self.insert("agentskill", SKILL_COLUMNS, values)
|
||||
|
||||
def plan(self):
|
||||
plan = asset_plan(self.db, self.identities, self.kbs, model_plan(self.db))
|
||||
plan["source_snapshot_sha256"] = "a" * 64
|
||||
return plan
|
||||
|
||||
def test_preserves_original_text_and_unsupported_configuration(self):
|
||||
plan = self.plan()
|
||||
self.assertEqual(plan["agents"][0]["target"]["config"]["system_prompt"], " Original rules\n\n")
|
||||
self.assertEqual(plan["skills"][0]["target"]["instructions"], " Original skill body\n\n")
|
||||
self.assertEqual(plan["agents"][0]["source"]["runtime_snapshot"]["primary"]["rules"], "frozen-original-rules")
|
||||
self.assertEqual(plan["prompt_templates"][0]["role_prompt"], "original-template-body")
|
||||
self.assertEqual(plan["agents"][0]["source"]["content_tags"], ["source-tag"])
|
||||
self.assertEqual(plan["agents"][0]["target"]["config"]["content_tags"], ["source-tag"])
|
||||
self.assertEqual(plan["agents"][0]["target"]["config"]["role"], "production")
|
||||
self.assertNotIn("role", plan["unsupported_fields"])
|
||||
self.assertNotIn("content_tags", plan["unsupported_fields"])
|
||||
self.assertEqual(plan["agents"][0]["target"]["config"]["fusion_agent_ids"], [])
|
||||
self.assertNotIn("fusion_agent_ids", plan["unsupported_fields"])
|
||||
|
||||
def test_rejects_invalid_fusion_configuration(self):
|
||||
for value in ('[1]', '[2,2]', '[true]', '["2"]', '[3]', '[0]'):
|
||||
with self.subTest(value=value):
|
||||
self.db.execute("UPDATE agent SET fusion_agent_ids=? WHERE id=1", (value,))
|
||||
with self.assertRaises(ValueError):
|
||||
self.plan()
|
||||
self.db.execute("UPDATE agent SET fusion_agent_ids='[]' WHERE id=1")
|
||||
|
||||
def test_maps_direct_fusions_without_adding_their_knowledge_or_recursion(self):
|
||||
self.db.execute("UPDATE agent SET fusion_agent_ids='[22,19]' WHERE id=1")
|
||||
self.db.execute("UPDATE agent SET fusion_agent_ids='[1]' WHERE id=22")
|
||||
plan = self.plan()
|
||||
config = plan["agents"][0]["target"]["config"]
|
||||
self.assertEqual(config["fusion_agent_ids"], [asset_id("agent", 22), asset_id("agent", 19)])
|
||||
self.assertEqual(config["knowledge_bases"], [KB_ID])
|
||||
self.db.execute("UPDATE agent SET role='qc' WHERE id=19")
|
||||
with self.assertRaises(ValueError):
|
||||
self.plan()
|
||||
|
||||
def test_rejects_invalid_native_duties_and_tags(self):
|
||||
for field, value in (("role", "unknown"), ("content_tags", '["same","same"]'),
|
||||
("content_tags", '[" "]'), ("content_tags", '[1]')):
|
||||
with self.subTest(field=field, value=value):
|
||||
self.db.execute(f"UPDATE agent SET {field}=? WHERE id=1", (value,))
|
||||
with self.assertRaises(ValueError):
|
||||
self.plan()
|
||||
self.db.execute("UPDATE agent SET role='production',content_tags='[\"source-tag\"]' WHERE id=1")
|
||||
|
||||
def test_replay_accepts_only_exact_pre_duty_snapshot(self):
|
||||
sql = asset_sql(self.plan())
|
||||
self.assertIn("NOT (mapping.native_snapshot->'config' ? 'role')", sql)
|
||||
self.assertIn("NOT (mapping.native_snapshot->'config' ? 'content_tags')", sql)
|
||||
self.assertIn("target #- '{config,role}' #- '{config,content_tags}'", sql)
|
||||
|
||||
def test_owner_zero_resolves_pinned_admin_and_private_owner_stays_separate(self):
|
||||
plan = self.plan()
|
||||
self.assertEqual(plan["agents"][0]["target"]["created_by"], ADMIN_TARGET_ID)
|
||||
self.assertEqual(plan["skills"][0]["target"]["created_by"], ADMIN_TARGET_ID)
|
||||
self.assertEqual(plan["agents"][1]["target"]["created_by"], USER_ID)
|
||||
self.assertEqual(plan["agents"][1]["target"]["visibility"], "private")
|
||||
self.assertFalse(plan["agents"][1]["target"]["enabled"])
|
||||
self.assertFalse(plan["agents"][0]["target"]["is_builtin"])
|
||||
|
||||
def test_owner_checks_include_template_owners_but_exclude_unrelated_identities(self):
|
||||
unrelated = "d574dd78-7ca7-41e1-ae19-e2874665a183"
|
||||
template_owner = "a986bcf9-3325-4ff0-aac7-cbc1d93f5b63"
|
||||
self.identities.update({2: unrelated, 8: template_owner})
|
||||
self.db.execute("UPDATE agentprompttemplate SET owner_user_id=8 WHERE id=1")
|
||||
|
||||
plan = self.plan()
|
||||
|
||||
self.assertEqual(plan["identity_mapping"], {"1": ADMIN_TARGET_ID, "3": USER_ID, "8": template_owner})
|
||||
self.assertEqual(self.identities[2], unrelated)
|
||||
self.assertNotIn(unrelated, asset_sql(plan))
|
||||
|
||||
def test_private_cross_owner_binding_is_preserved_but_not_effective(self):
|
||||
plan = self.plan()
|
||||
self.assertEqual(len(plan["bindings"]), 6)
|
||||
blocked = next(b for b in plan["bindings"] if b["source"]["id"] == 102)
|
||||
self.assertFalse(blocked["effective"])
|
||||
self.assertIn("blocked_private_owner_mismatch", blocked["review"])
|
||||
self.assertNotIn(asset_id("skill", 5), plan["agents"][0]["target"]["config"]["instruction_skill_ids"])
|
||||
self.assertIn(asset_id("skill", 9), plan["agents"][0]["target"]["config"]["instruction_skill_ids"])
|
||||
self.assertIn(asset_id("skill", 5), plan["agents"][2]["target"]["config"]["instruction_skill_ids"])
|
||||
|
||||
def test_global_skill_remains_bound_to_private_other_owner_agent(self):
|
||||
plan = self.plan()
|
||||
selected = plan["agents"][2]["target"]["config"]["instruction_skill_ids"]
|
||||
self.assertIn(asset_id("skill", 1), selected)
|
||||
binding = next(b for b in plan["bindings"] if b["source"]["id"] == 105)
|
||||
self.assertTrue(binding["effective"])
|
||||
self.assertEqual(binding["review"], [])
|
||||
|
||||
def test_native_skill_is_non_executable_and_disabled_flag_is_preserved(self):
|
||||
self.db.execute("UPDATE agentskill SET enabled=0 WHERE id=1")
|
||||
plan = self.plan()
|
||||
self.assertFalse(plan["skills"][0]["target"]["enabled"])
|
||||
self.assertTrue(plan["skills"][0]["target"]["instruction_only"])
|
||||
self.assertEqual(plan["skills"][0]["target"]["bundle_ref"], "")
|
||||
for agent in plan["agents"]:
|
||||
self.assertEqual(agent["target"]["config"]["allowed_tools"], [])
|
||||
self.assertEqual(agent["target"]["config"]["skills_selection_mode"], "none")
|
||||
|
||||
def test_native_model_uses_reviewed_legacy_fallback(self):
|
||||
plan = self.plan()
|
||||
for item in plan["agents"]:
|
||||
self.assertEqual(item["target"]["config"]["model_id"], model_id(3, "deepseek-flash"))
|
||||
self.assertTrue(item["model_resolution"]["model_fallback"])
|
||||
|
||||
def test_selected_kb_is_mapped_and_no_kb_becomes_none(self):
|
||||
plan = self.plan()
|
||||
self.assertEqual(plan["agents"][0]["target"]["config"]["knowledge_bases"], [KB_ID])
|
||||
self.assertEqual(plan["agents"][0]["target"]["config"]["kb_selection_mode"], "selected")
|
||||
self.assertEqual(plan["agents"][1]["target"]["config"]["kb_selection_mode"], "none")
|
||||
|
||||
def test_readonly_and_no_credentials_read(self):
|
||||
def authorize(action, table, column, database, trigger):
|
||||
if action == sqlite3.SQLITE_READ and table == "modelprovider" and column == "api_key":
|
||||
return sqlite3.SQLITE_DENY
|
||||
return sqlite3.SQLITE_OK
|
||||
self.db.set_authorizer(authorize)
|
||||
self.db.execute("PRAGMA query_only=ON")
|
||||
plan = self.plan()
|
||||
self.assertNotIn("DO-NOT-READ-SECRET", json.dumps(plan))
|
||||
|
||||
def test_bad_identity_or_organization_rejected(self):
|
||||
for identity in ({3: USER_ID}, {1: USER_ID, 3: ADMIN_TARGET_ID}, {1: ADMIN_TARGET_ID}):
|
||||
with self.subTest(identity=identity), self.assertRaises(ValueError):
|
||||
asset_plan(self.db, identity, self.kbs, model_plan(self.db))
|
||||
self.db.execute("UPDATE agent SET organization_id=2 WHERE id=19")
|
||||
with self.assertRaisesRegex(ValueError, "organization"):
|
||||
self.plan()
|
||||
|
||||
def test_unknown_visibility_rejected(self):
|
||||
self.db.execute("UPDATE agentskill SET visibility='shared' WHERE id=5")
|
||||
with self.assertRaisesRegex(ValueError, "visibility"):
|
||||
self.plan()
|
||||
|
||||
def test_explicit_grants_require_review(self):
|
||||
self.db.execute("INSERT INTO agentskillgrant VALUES (1,5,1,'use',1,'2026-09-15')")
|
||||
with self.assertRaisesRegex(ValueError, "explicit grants"):
|
||||
self.plan()
|
||||
|
||||
def test_missing_and_duplicate_binding_rejected(self):
|
||||
self.db.execute("INSERT INTO agentskillbinding VALUES (200,1,1)")
|
||||
with self.assertRaisesRegex(ValueError, "duplicated"):
|
||||
self.plan()
|
||||
self.db.execute("DELETE FROM agentskillbinding WHERE id=200")
|
||||
self.db.execute("INSERT INTO agentskillbinding VALUES (200,777,1)")
|
||||
with self.assertRaisesRegex(ValueError, "missing asset"):
|
||||
self.plan()
|
||||
|
||||
def test_unmapped_kb_or_category_scope_never_widens(self):
|
||||
self.db.execute("UPDATE agent SET kb_ids='[7]' WHERE id=1")
|
||||
with self.assertRaisesRegex(ValueError, "unmapped"):
|
||||
self.plan()
|
||||
self.db.execute("UPDATE agent SET kb_ids='[3]',kb_category_ids='{\"3\":[4]}' WHERE id=1")
|
||||
with self.assertRaisesRegex(ValueError, "category restrictions"):
|
||||
self.plan()
|
||||
|
||||
def test_missing_legacy_template_rejected(self):
|
||||
self.db.execute("DELETE FROM agentprompttemplate")
|
||||
with self.assertRaisesRegex(ValueError, "missing prompt template"):
|
||||
self.plan()
|
||||
|
||||
def test_plan_is_deterministic_and_source_ids_are_namespaced(self):
|
||||
self.assertEqual(self.plan(), self.plan())
|
||||
self.assertNotEqual(asset_id("agent", 1), asset_id("skill", 1))
|
||||
self.assertNotEqual(asset_id("agent", 1), asset_id("agent", 19))
|
||||
|
||||
def test_sql_requires_fingerprint(self):
|
||||
for digest in (None, "", "A" * 64, "a" * 63):
|
||||
plan = self.plan()
|
||||
plan["source_snapshot_sha256"] = digest
|
||||
with self.subTest(digest=digest), self.assertRaises(ValueError):
|
||||
asset_sql(plan)
|
||||
|
||||
def test_tampered_targets_or_access_fail_before_sql(self):
|
||||
mutations = [
|
||||
lambda p: p.update(target_tenant_id=42),
|
||||
lambda p: p.update(target_database="production"),
|
||||
lambda p: p["agents"][1]["target"].update(enabled=True),
|
||||
lambda p: p["agents"][1]["target"].update(visibility="global"),
|
||||
lambda p: p["skills"][1]["target"].update(instruction_only=False),
|
||||
lambda p: p["agents"][0]["target"]["config"].update(allowed_tools=["execute"]),
|
||||
lambda p: p["agents"][0]["target"]["config"].update(system_prompt="changed"),
|
||||
lambda p: p["bindings"][1].update(effective=True),
|
||||
lambda p: p["bindings"][1].update(review=[]),
|
||||
]
|
||||
for mutate in mutations:
|
||||
plan = self.plan()
|
||||
mutate(plan)
|
||||
with self.subTest(mutate=mutate), self.assertRaises(ValueError):
|
||||
asset_sql(plan)
|
||||
|
||||
def test_safe_summary_contains_no_prompt_bodies(self):
|
||||
summary = safe_asset_summary(self.plan())
|
||||
self.assertEqual(summary["counts"]["bindings"], 6)
|
||||
self.assertEqual(summary["effective_bindings"], 5)
|
||||
self.assertEqual(summary["blocked_bindings"], 1)
|
||||
self.assertEqual(summary["disabled_agent_ids"], [19])
|
||||
self.assertNotIn("Original", json.dumps(summary))
|
||||
self.assertNotIn("frozen", json.dumps(summary))
|
||||
self.assertNotIn("template-body", json.dumps(summary))
|
||||
|
||||
def test_sql_is_guarded_insert_only_and_keeps_tombstones_and_user_edits(self):
|
||||
sql = asset_sql(self.plan())
|
||||
self.assertIn("current_database()", sql)
|
||||
self.assertIn("pg_advisory_xact_lock", sql)
|
||||
self.assertIn("Asset owner durable identity or membership mismatch", sql)
|
||||
self.assertIn("Asset durable mapping conflict", sql)
|
||||
self.assertIn("refusing recreation", sql)
|
||||
self.assertIn("Native skill ID or owner/name collision", sql)
|
||||
self.assertIn("Native Agent ID or owner/name collision", sql)
|
||||
self.assertIn("effective boolean NOT NULL", sql)
|
||||
self.assertIn("SET LOCAL log_min_error_statement = 'panic'", sql)
|
||||
self.assertIn(f"CREATE TABLE IF NOT EXISTS {LEDGER}", sql)
|
||||
self.assertIn(f"CREATE TABLE IF NOT EXISTS {BINDING_LEDGER}", sql)
|
||||
self.assertNotIn("ON CONFLICT", sql.upper())
|
||||
self.assertNotIn("UPDATE CUSTOM_AGENTS", sql.upper())
|
||||
self.assertNotIn("UPDATE TENANT_SKILL_CATALOG", sql.upper())
|
||||
self.assertNotIn("COMMIT", sql.upper())
|
||||
self.assertNotIn("BEGIN TRANSACTION", sql.upper())
|
||||
self.assertNotIn("SELECT value FROM jsonb_each_text", sql)
|
||||
|
||||
def test_text_is_sql_escaped_without_semantic_transformation(self):
|
||||
value = " O'Brien\\line\n$assets$; SELECT 'opaque data'; "
|
||||
self.db.execute("UPDATE agent SET rules=? WHERE id=1", (value,))
|
||||
plan = self.plan()
|
||||
self.assertEqual(plan["agents"][0]["target"]["config"]["system_prompt"], value)
|
||||
sql = asset_sql(plan)
|
||||
self.assertIn("O''Brien", sql)
|
||||
self.assertIn("standard_conforming_strings = on", sql)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,273 @@
|
||||
"""Historical imports retain source facts and never start model execution."""
|
||||
import copy
|
||||
import base64
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from weknora_business_etl import (ADMIN_TARGET_ID, HISTORY_TABLES, JSON_FIELDS, LEDGER, RESOURCE_KINDS,
|
||||
business_id, business_plan, business_sql, safe_business_summary, postgres_plan)
|
||||
|
||||
USER = "ab043a7d-d10e-58fe-8417-3fd2cbab5f77"
|
||||
AGENT = "72a9a19f-65f3-4996-897b-efca38654b70"
|
||||
KB = "7e6ac485-f766-4d58-ae38-2f0456a022ef"
|
||||
|
||||
|
||||
class BusinessTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.root = Path(self.tmp.name)
|
||||
self.artifact = self.root / "seg_1_r1.md"
|
||||
self.artifact.write_text(" Original paragraph O'Brien\n\n", encoding="utf-8")
|
||||
self.db = sqlite3.connect(":memory:")
|
||||
self.addCleanup(self.db.close)
|
||||
shared = dict(organization_id=1, owner_user_id=3, visibility="private", created_at="2026-09-15 01:00:00")
|
||||
flow = dict(id=1, name="Flow", nodes=[{"node_id": "a", "agent_id": 9, "enabled": True, "x": 20, "y": 40}],
|
||||
edges=[], supervisor_agent_id=None, supervisor_mode="off", **shared)
|
||||
map_row = dict(id=2, owner_user_id=3, organization_id=1, topic="Course", kb_ids=[5], kb_category_ids={},
|
||||
requested_days=1, agent_id=9, version=2, confirmed_version=2,
|
||||
plan={"recommended_days": 1, "rationale": "Original rationale", "insufficiencies": [],
|
||||
"days": [{"day": 1, "title": "Day one", "source_ids": ["E1"], "support": "Original support", "prerequisite": "Prerequisite"}]},
|
||||
evidence=[{"id": "E1", "text": "Private evidence"}], coverage={"scope": "quick"},
|
||||
assessment_cancel_requested=0, maps={"1": {"day": 1, "title": "Daily framework", "children": [
|
||||
{"title": "Node title", "content": "Node content", "source_ids": ["E1"], "hook": "Original hook"}]}},
|
||||
status="confirmed", error="")
|
||||
campaign = dict(id=3, name="Task", mindmap_text="Original map text", objective="Legacy objective", success_criteria="Legacy criteria",
|
||||
canvas_snapshot={"nodes": flow["nodes"], "edges": []}, canvas_name="Flow", duration_minutes=60,
|
||||
qc_mode="full", previous_campaign_id=None, task_spec=[{"title": "Original paragraph"}],
|
||||
pending_confirms=["Original pending"], notes=[], status="human_review", has_failure=0,
|
||||
finished_at="", **shared)
|
||||
segment = dict(id=4, campaign_id=3, node_id="a", agent_id=9, agent_assignment_mode="manual", generation_history=[],
|
||||
fusion_agent_ids=[], runtime_snapshot={"primary": {"rules": "Private frozen rules"}},
|
||||
content_tag="Tag", title="Original paragraph", time_range="00:00-00:10", mindmap_node="Source node",
|
||||
product_ref="", status="approved", round=1, artifact_path=str(self.artifact), anchor_card={},
|
||||
qc_reports=[], failure={}, references=[{"chunk_id": "private-chunk", "text": "Private reference"}],
|
||||
kb_category_ids=[], kb_category_recommendation={}, updated_at="2026-09-15 02:00:00")
|
||||
run = dict(id=5, campaign_id=3, user_id=3, organization_id=1, status="completed", cancel_requested=0,
|
||||
pause_requested=0, checkpoint={"step": "review"}, config_snapshot={"rules": "Original runtime config"},
|
||||
trace_id="original-trace", runtime_version="v1", error="", created_at="2026-09-15 01:01:00",
|
||||
started_at="2026-09-15 01:01:00", finished_at="2026-09-15 02:00:00")
|
||||
report = dict(id=6, segment_id=4, gate="cross", checks=[{"result": "Original failed check"}], verdict="FAIL", run_id=5,
|
||||
trace_id="original-trace", runtime_snapshot={}, created_at="2026-09-15 01:40:00")
|
||||
event = dict(id=7, run_id=5, campaign_id=3, segment_id=4, node_id="a", trace_id="original-trace", span_id="span",
|
||||
parent_span_id="", actor_agent_id=9, actor_name="Agent", tool_name="knowledge_search", event_type="error",
|
||||
status="failed", payload={"error": "Original model error"}, created_at="2026-09-15 01:30:00")
|
||||
session = dict(id=8, run_id=5, campaign_id=3, segment_id=4, agent_id=9, agent_name="Agent", parent_session_id=None,
|
||||
phase="production", trace_id="original-trace", status="completed", turn=1, max_turns=6,
|
||||
messages=[{"role": "system", "content": "Private historical prompt"}], tool_state={},
|
||||
result_text="Original model result", result_meta={}, error="", created_at="2026-09-15 01:10:00",
|
||||
updated_at="2026-09-15 01:20:00", finished_at="2026-09-15 01:20:00")
|
||||
queue = dict(id=9, task_key="campaign-run:5", payload={"run_id": 5, "campaign_id": 3}, status="done", attempt=1)
|
||||
rows = {"canvas": [flow], "trainingmap": [map_row], "campaign": [campaign], "segment": [segment],
|
||||
"campaignrun": [run], "qcreport": [report], "runevent": [event], "agentsession": [session], "taskqueueitem": [queue]}
|
||||
for table in (*RESOURCE_KINDS, *HISTORY_TABLES):
|
||||
sample = rows.get(table, [{"id": 0}])[0]
|
||||
columns = ",".join('"' + key + '"' + (" INTEGER" if type(value) is int else " TEXT") for key, value in sample.items())
|
||||
self.db.execute(f'CREATE TABLE "{table}" ({columns})')
|
||||
for row in rows.get(table, []):
|
||||
self.insert(table, row)
|
||||
self.db.commit()
|
||||
self.db.execute("PRAGMA query_only=ON")
|
||||
self.identities = {"1": ADMIN_TARGET_ID, "3": USER}
|
||||
|
||||
def insert(self, table, row):
|
||||
columns = ",".join('"' + key + '"' for key in row)
|
||||
values = [json.dumps(value) if key in JSON_FIELDS.get(table, ()) else value for key, value in row.items()]
|
||||
self.db.execute(f'INSERT INTO "{table}" ({columns}) VALUES (' + ",".join("?" for _ in row) + ")", values)
|
||||
|
||||
def change(self, sql, values=()):
|
||||
self.db.execute("PRAGMA query_only=OFF")
|
||||
self.db.execute(sql, values)
|
||||
self.db.execute("PRAGMA query_only=ON")
|
||||
|
||||
def plan(self):
|
||||
result = business_plan(self.db, self.identities, {"9": AGENT}, {"5": KB}, self.root)
|
||||
result["source_snapshot_sha256"] = "a" * 64
|
||||
return result
|
||||
|
||||
def resource(self, plan, kind):
|
||||
return next(r["target"] for r in plan["resources"] if r["source_kind"] == kind)
|
||||
|
||||
def test_maps_agents_owners_and_knowledge_without_inference(self):
|
||||
plan = self.plan()
|
||||
flow = self.resource(plan, "canvas")
|
||||
self.assertEqual(flow["owner_id"], USER)
|
||||
self.assertEqual(flow["data"]["nodes"][0]["agent_id"], AGENT)
|
||||
self.assertEqual(flow["data"]["legacy_history"]["nodes"][0]["x"], 20)
|
||||
training = self.resource(plan, "trainingmap")
|
||||
self.assertEqual(training["data"]["knowledge_base_ids"], [KB])
|
||||
self.assertEqual(training["version"], 2)
|
||||
task = self.resource(plan, "campaign")
|
||||
self.assertEqual(task["data"]["map_id"], "")
|
||||
self.assertEqual(task["data"]["flow_id"], "")
|
||||
self.assertEqual(task["data"]["day"], 0)
|
||||
|
||||
def test_preserves_native_text_and_source_status(self):
|
||||
plan = self.plan()
|
||||
task = self.resource(plan, "campaign")
|
||||
segment = task["data"]["segments"][0]
|
||||
self.assertEqual(segment["content"], " Original paragraph O'Brien\n\n")
|
||||
self.assertEqual(segment["status"], "approved")
|
||||
self.assertEqual(task["status"], "human_review")
|
||||
self.assertEqual(task["data"]["composed"], "")
|
||||
self.assertIn('"verdict": "FAIL"', segment["review"])
|
||||
self.assertIn("Original failed check", task["data"]["review"])
|
||||
|
||||
def test_daily_framework_preserves_node_content_without_forging_confirmation(self):
|
||||
data = self.resource(self.plan(), "trainingmap")["data"]
|
||||
day = data["days"][0]
|
||||
self.assertEqual(day["nodes"][0]["label"], "Node title")
|
||||
self.assertIn("Node content", day["framework"])
|
||||
self.assertIn("Original hook", day["framework"])
|
||||
self.assertFalse(day["confirmed"])
|
||||
self.assertEqual(data["legacy_history"]["confirmed_version"], 2)
|
||||
self.assertEqual(data["legacy_history"]["plan"]["days"][0]["support"], "Original support")
|
||||
|
||||
def test_every_resource_blocks_automatic_execution(self):
|
||||
for resource in self.plan()["resources"]:
|
||||
marker = resource["target"]["data"]["legacy_import"]
|
||||
self.assertTrue(marker["read_only"])
|
||||
self.assertTrue(marker["requires_review"])
|
||||
self.assertEqual(marker["source_id"], resource["source_id"])
|
||||
task = self.resource(self.plan(), "campaign")
|
||||
self.assertTrue(task["data"]["legacy_import"]["needs_configuration"])
|
||||
|
||||
def test_missing_agent_and_category_scope_are_not_widened(self):
|
||||
self.change("UPDATE trainingmap SET agent_id=NULL,kb_category_ids=?", (json.dumps({"5": [9]}),))
|
||||
data = self.resource(self.plan(), "trainingmap")["data"]
|
||||
self.assertEqual(data["agent_id"], "")
|
||||
self.assertIn("knowledge_category_scope", data["legacy_import"]["missing_configuration"])
|
||||
self.assertTrue(data["legacy_import"]["read_only"])
|
||||
|
||||
def test_empty_canvas_remains_empty_and_requires_configuration(self):
|
||||
self.change("UPDATE canvas SET nodes='[]'")
|
||||
data = self.resource(self.plan(), "canvas")["data"]
|
||||
self.assertEqual(data["nodes"], [])
|
||||
self.assertIn("flow_nodes", data["legacy_import"]["missing_configuration"])
|
||||
|
||||
def test_unmapped_or_deleted_owner_is_not_reassigned_to_admin(self):
|
||||
self.change("UPDATE canvas SET owner_user_id=2")
|
||||
with self.assertRaisesRegex(ValueError, "no durable identity"):
|
||||
self.plan()
|
||||
|
||||
def test_invalid_identity_organization_or_snapshot_rejected(self):
|
||||
self.change("UPDATE canvas SET organization_id=2")
|
||||
with self.assertRaisesRegex(ValueError, "organization"):
|
||||
self.plan()
|
||||
self.change("UPDATE canvas SET organization_id=1")
|
||||
self.db.execute("PRAGMA query_only=OFF")
|
||||
with self.assertRaisesRegex(ValueError, "read-only"):
|
||||
self.plan()
|
||||
|
||||
def test_runtime_and_history_go_to_ledger_without_native_jobs_or_events(self):
|
||||
plan = self.plan()
|
||||
sources = {r["source_kind"]: r["source"] for r in plan["history"]}
|
||||
self.assertEqual(sources["agentsession"]["messages"][0]["content"], "Private historical prompt")
|
||||
self.assertEqual(sources["runevent"]["payload"]["error"], "Original model error")
|
||||
self.assertEqual(sources["taskqueueitem"]["status"], "done")
|
||||
sql = business_sql(plan).upper()
|
||||
for table in ("WORKBENCH_JOBS", "WORKBENCH_EVENTS", "WORKBENCH_REVISIONS", "KNOWLEDGE_USAGE", "POINTACCOUNT"):
|
||||
self.assertNotIn("INSERT INTO " + table, sql)
|
||||
|
||||
def test_conflicting_old_qc_run_is_preserved_but_not_displayed_as_current_qc(self):
|
||||
self.change("UPDATE qcreport SET run_id=999")
|
||||
plan = self.plan()
|
||||
report = next(i for i in plan["history"] if i["source_kind"] == "qcreport")
|
||||
self.assertEqual(report["source"]["run_id"], 999)
|
||||
self.assertIn("source_run_campaign_mismatch_ledger_only", report["relation_review"])
|
||||
self.assertEqual(self.resource(plan, "campaign")["data"]["segments"][0]["review"], "")
|
||||
self.assertEqual(self.resource(plan, "campaign")["data"]["review"], "")
|
||||
|
||||
def test_unrelated_legacy_sessions_are_not_copied(self):
|
||||
self.change("UPDATE agentsession SET campaign_id=999")
|
||||
plan = self.plan()
|
||||
self.assertFalse(any(i["source_kind"] == "agentsession" for i in plan["history"]))
|
||||
self.assertEqual(plan["excluded_history_counts"]["agentsession"], 1)
|
||||
|
||||
def test_all_historical_artifact_versions_are_preserved(self):
|
||||
older = self.root / "seg_1_r0.md"
|
||||
older.write_text("Previous version", encoding="utf-8")
|
||||
self.change("UPDATE segment SET generation_history=?", (json.dumps([{"round": 0, "artifact_path": str(older)}]),))
|
||||
plan = self.plan()
|
||||
self.assertEqual(len(plan["artifacts"]), 2)
|
||||
self.assertEqual(plan["artifacts"][str(older)]["body"], "Previous version")
|
||||
|
||||
def test_missing_artifact_is_explicit_and_does_not_fake_content(self):
|
||||
self.artifact.unlink()
|
||||
plan = self.plan()
|
||||
self.assertEqual(safe_business_summary(plan)["artifacts_missing"], 1)
|
||||
task = self.resource(plan, "campaign")["data"]
|
||||
self.assertEqual(task["segments"][0]["content"], "")
|
||||
self.assertIn("segment_artifact", task["legacy_import"]["missing_configuration"])
|
||||
|
||||
def test_artifact_cannot_escape_allowed_directory(self):
|
||||
self.change("UPDATE segment SET artifact_path='/etc/passwd'")
|
||||
with self.assertRaisesRegex(ValueError, "outside"):
|
||||
self.plan()
|
||||
|
||||
def test_report_omits_private_bodies_and_paths(self):
|
||||
summary = json.dumps(safe_business_summary(self.plan()))
|
||||
for forbidden in ("Original", "Private", "O'Brien", self.tmp.name, "historical prompt", "Node content"):
|
||||
self.assertNotIn(forbidden, summary)
|
||||
|
||||
def test_tampering_projection_or_artifact_fails(self):
|
||||
mutations = [lambda p: p.update(target_database="production"), lambda p: p.update(target_tenant_id=1),
|
||||
lambda p: p["resources"][0]["target"].update(owner_id=ADMIN_TARGET_ID),
|
||||
lambda p: p["resources"][0]["target"]["data"]["legacy_import"].update(read_only=False),
|
||||
lambda p: p["artifacts"][str(self.artifact)].update(body="Changed body"),
|
||||
lambda p: p["history"][0].update(target_id=AGENT)]
|
||||
for mutate in mutations:
|
||||
plan = self.plan()
|
||||
mutate(plan)
|
||||
with self.subTest(mutate=mutate), self.assertRaises(ValueError):
|
||||
business_sql(plan)
|
||||
|
||||
def test_bad_fingerprint_fails(self):
|
||||
for digest in (None, "", "X" * 64, "a" * 63):
|
||||
plan = self.plan()
|
||||
plan["source_snapshot_sha256"] = digest
|
||||
with self.subTest(digest=digest), self.assertRaises(ValueError):
|
||||
business_sql(plan)
|
||||
|
||||
def test_null_in_archived_evidence_is_losslessly_encoded_for_postgres(self):
|
||||
original = "source\x00evidence\ud800"
|
||||
self.change("UPDATE trainingmap SET evidence=?", (json.dumps([{"text": original}]),))
|
||||
plan = self.plan()
|
||||
encoded = postgres_plan(plan)
|
||||
item = next(r for r in encoded["resources"] if r["source_kind"] == "trainingmap")
|
||||
source = item["source"]["evidence"][0]["text"]
|
||||
self.assertEqual(base64.b64decode(source["body"]).decode("utf-8", errors="surrogatepass"), original)
|
||||
self.assertEqual(item["target"]["data"]["legacy_history"]["evidence"][0]["text"], source)
|
||||
self.assertEqual(self.resource(plan, "trainingmap")["data"]["legacy_history"]["evidence"][0]["text"], original)
|
||||
self.assertNotIn("\\u0000", business_sql(plan))
|
||||
self.assertNotIn("\\ud800", business_sql(plan))
|
||||
|
||||
def test_sql_is_insert_only_tenant_guarded_and_replay_preserves_edits(self):
|
||||
sql = business_sql(self.plan())
|
||||
for marker in ("current_database()", "yawan_native_acceptance_20260915", "pg_advisory_xact_lock(10059",
|
||||
"Business owner durable identity", "Business Agent durable mapping", "Business durable mapping conflict",
|
||||
"refusing recreation", "Native business ID or owner/name collision", "source_snapshot jsonb",
|
||||
"artifact_snapshot jsonb", "SET LOCAL log_min_error_statement = 'panic'", LEDGER):
|
||||
self.assertIn(marker, sql)
|
||||
for forbidden in ("UPDATE WORKBENCH_RESOURCES", "DELETE FROM", "ON CONFLICT", "COMMIT;", "BEGIN TRANSACTION"):
|
||||
self.assertNotIn(forbidden, sql.upper())
|
||||
self.assertIn("O''Brien", sql)
|
||||
|
||||
def test_namespaced_ids_and_plan_are_deterministic(self):
|
||||
self.assertEqual(self.plan(), self.plan())
|
||||
self.assertNotEqual(business_id("segment", 1), business_id("canvas", 1))
|
||||
self.assertNotEqual(business_id("trainingmap", 1, "1.0"), business_id("trainingmap", 1, "1.1"))
|
||||
|
||||
def test_duplicate_node_or_dangling_edge_fails(self):
|
||||
self.change("UPDATE canvas SET edges=?", (json.dumps([{"from": "a", "to": "missing"}]),))
|
||||
with self.assertRaisesRegex(ValueError, "missing node"):
|
||||
self.plan()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Category import regressions: tenant isolation, splits and tree ancestry."""
|
||||
import copy
|
||||
import unittest
|
||||
|
||||
from weknora_category_etl import TENANT, build_manifest, document_stem
|
||||
|
||||
|
||||
def fixture():
|
||||
source = {
|
||||
"source_mutated": False, "relation_validation": {"valid": True}, "source": {"sha256": "fixture"},
|
||||
"mapping": {
|
||||
"knowledge_bases": [{"legacy_id": 1, "native_id": "plan-kb", "name": "Library"}],
|
||||
"categories": [
|
||||
{"legacy_id": 1, "native_id": "plan-root", "knowledge_base_id": "plan-kb", "parent_id": None, "name": "Root", "sort_order": 0},
|
||||
{"legacy_id": 2, "native_id": "plan-child", "knowledge_base_id": "plan-kb", "parent_id": "plan-root", "name": "Child", "sort_order": 1},
|
||||
],
|
||||
"files": [{"legacy_id": 1, "knowledge_base_id": "plan-kb", "file_name": "Root__Child.jsonl", "category_id": "plan-child", "content_hash": "source-hash", "revision": 1}],
|
||||
},
|
||||
}
|
||||
state = {"tenant_id": TENANT, "knowledge_bases": {"1": {"id": "real-kb", "legacy_name": "Library", "complete": True, "failed": [], "uploaded": ["Root__Child.jsonl"]}}}
|
||||
target = {
|
||||
"tenant": {"id": TENANT, "status": "active"},
|
||||
"knowledge_bases": [{"id": "real-kb", "tenant_id": TENANT, "name": "Library"}],
|
||||
"tags": [], "relations": [],
|
||||
"files": [{"id": "doc", "tenant_id": TENANT, "knowledge_base_id": "real-kb", "file_name": "Root__Child.md", "file_hash": "target-hash", "file_size": 123, "parse_status": "completed", "folder_path": "", "custom_metadata": {"existing": "preserved"}}],
|
||||
}
|
||||
return source, state, target
|
||||
|
||||
|
||||
class CategoryMigrationTests(unittest.TestCase):
|
||||
def test_parent_tag_and_folder_tree_use_actual_kb(self):
|
||||
result = build_manifest(*fixture())
|
||||
self.assertEqual(result["files"][0]["folder_path"], "Root/Child")
|
||||
self.assertEqual(result["files"][0]["metadata"]["existing"], "preserved")
|
||||
self.assertEqual(len(result["relations"]), 2)
|
||||
self.assertTrue(all(tag["knowledge_base_id"] == "real-kb" for tag in result["tags"]))
|
||||
|
||||
def test_split_source_maps_both_parts(self):
|
||||
source, state, target = fixture()
|
||||
state["knowledge_bases"]["1"]["uploaded"] = ["Root__Child.part001.md", "Root__Child.part002.md"]
|
||||
file = target["files"][0]
|
||||
target["files"] = [{**file, "id": f"doc-{part}", "file_name": f"Root__Child.part{part:03}.md"} for part in (1, 2)]
|
||||
result = build_manifest(source, state, target)
|
||||
self.assertEqual(result["summary"]["source_files"], 1)
|
||||
self.assertEqual(result["summary"]["target_files"], 2)
|
||||
self.assertEqual(len(result["relations"]), 4)
|
||||
|
||||
def test_partial_split_rejected(self):
|
||||
source, state, target = fixture()
|
||||
state["knowledge_bases"]["1"]["uploaded"] = ["Root__Child.part001.md", "Root__Child.part003.md"]
|
||||
file = target["files"][0]
|
||||
target["files"] = [{**file, "id": f"doc-{part}", "file_name": f"Root__Child.part{part:03}.md"} for part in (1, 3)]
|
||||
with self.assertRaisesRegex(ValueError, "split"):
|
||||
build_manifest(source, state, target)
|
||||
|
||||
def test_missing_split_document_rejected(self):
|
||||
source, state, target = fixture()
|
||||
state["knowledge_bases"]["1"]["uploaded"] = ["Root__Child.part001.md", "Root__Child.part002.md"]
|
||||
target["files"][0]["file_name"] = "Root__Child.part001.md"
|
||||
with self.assertRaisesRegex(ValueError, "missing"):
|
||||
build_manifest(source, state, target)
|
||||
|
||||
def test_cross_tenant_and_wrong_mapping_rejected(self):
|
||||
for item in ("tenant", "knowledge_base", "file"):
|
||||
with self.subTest(item=item):
|
||||
source, state, target = fixture()
|
||||
if item == "tenant":
|
||||
target["tenant"]["id"] += 1
|
||||
elif item == "knowledge_base":
|
||||
target["knowledge_bases"][0]["tenant_id"] += 1
|
||||
else:
|
||||
target["files"][0]["tenant_id"] += 1
|
||||
with self.assertRaises(ValueError):
|
||||
build_manifest(source, state, target)
|
||||
|
||||
def test_same_name_does_not_adopt_existing_tag(self):
|
||||
source, state, target = fixture()
|
||||
target["tags"] = [{"id": "someone-elses-tag", "tenant_id": TENANT, "knowledge_base_id": "real-kb", "name": "Root", "sort_order": 0, "deleted_at": None}]
|
||||
with self.assertRaisesRegex(ValueError, "conflicts"):
|
||||
build_manifest(source, state, target)
|
||||
|
||||
def test_second_plan_is_stable_and_preserves_user_metadata(self):
|
||||
source, state, target = fixture()
|
||||
first = build_manifest(source, state, target)
|
||||
target["tags"] = [{**t, "deleted_at": None} for t in first["tags"]]
|
||||
target["files"][0]["folder_path"] = first["files"][0]["folder_path"]
|
||||
target["files"][0]["custom_metadata"] = first["files"][0]["metadata"]
|
||||
second = build_manifest(source, state, target)
|
||||
self.assertEqual(first["tags"], second["tags"])
|
||||
self.assertEqual(first["relations"], second["relations"])
|
||||
self.assertEqual(first["files"][0]["metadata"], second["files"][0]["metadata"])
|
||||
|
||||
def test_manual_changes_and_unmapped_files_are_not_overwritten(self):
|
||||
source, state, target = fixture()
|
||||
for change in ("folder", "unmapped", "duplicate"):
|
||||
changed = copy.deepcopy(target)
|
||||
if change == "folder":
|
||||
changed["files"][0]["folder_path"] = "User folder"
|
||||
elif change == "unmapped":
|
||||
changed["files"].append({**changed["files"][0], "id": "extra", "file_name": "Other.md"})
|
||||
else:
|
||||
changed["files"].append({**changed["files"][0], "id": "duplicate"})
|
||||
with self.subTest(change=change), self.assertRaises(ValueError):
|
||||
build_manifest(source, state, changed)
|
||||
|
||||
def test_path_traversal_rejected(self):
|
||||
with self.assertRaises(ValueError):
|
||||
document_stem("../outside.md")
|
||||
source, state, target = fixture()
|
||||
source["mapping"]["categories"][0]["name"] = ".."
|
||||
with self.assertRaises(ValueError):
|
||||
build_manifest(source, state, target)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Regression checks for the read-only SQLite snapshot used by weknora_etl."""
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from weknora_etl import consistent_snapshot
|
||||
|
||||
|
||||
class SnapshotTests(unittest.TestCase):
|
||||
def test_backup_includes_uncheckpointed_wal(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "legacy.sqlite"
|
||||
source = sqlite3.connect(path)
|
||||
source.execute("PRAGMA journal_mode=WAL")
|
||||
source.execute("CREATE TABLE item (id INTEGER PRIMARY KEY, value TEXT)")
|
||||
source.commit()
|
||||
source.execute("INSERT INTO item(value) VALUES ('only-in-wal')")
|
||||
source.commit()
|
||||
self.assertTrue(Path(str(path) + "-wal").exists())
|
||||
|
||||
with consistent_snapshot(path) as (snapshot, info):
|
||||
self.assertEqual(info["snapshot_method"], "sqlite_backup")
|
||||
self.assertEqual(snapshot.execute("SELECT value FROM item").fetchone()[0], "only-in-wal")
|
||||
self.assertEqual(snapshot.execute("PRAGMA query_only").fetchone()[0], 1)
|
||||
with self.assertRaises(sqlite3.OperationalError):
|
||||
snapshot.execute("CREATE TABLE forbidden (id INTEGER)")
|
||||
source.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Read-only planner and generated-SQL contract checks for identity migration."""
|
||||
import copy
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from weknora_identity_etl import (ADMIN_TARGET_ID, IDENTITY_LEDGER, ORGANIZATION_LEDGER,
|
||||
PREFERENCE_KEY, TARGET_TENANT, _stable_id, identity_plan,
|
||||
identity_sql, identity_verification_sql)
|
||||
|
||||
|
||||
class IdentityTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.db = sqlite3.connect(":memory:")
|
||||
self.addCleanup(self.db.close)
|
||||
self.db.executescript("""
|
||||
CREATE TABLE organization (id INTEGER, name TEXT, slug TEXT, enabled INTEGER);
|
||||
CREATE TABLE roledefinition (id INTEGER, role TEXT, label TEXT, description TEXT,
|
||||
permissions TEXT, builtin INTEGER, organization_id INTEGER, updated_at TEXT);
|
||||
CREATE TABLE appuser (id INTEGER, username TEXT, display_name TEXT, role TEXT,
|
||||
organization_id INTEGER, enabled INTEGER, password_hash TEXT);
|
||||
INSERT INTO organization VALUES (1, 'source org', 'default', 1);
|
||||
INSERT INTO roledefinition VALUES (1, 'admin', 'Admin', 'All', '["*"]', 1, 1, '2026-09-15');
|
||||
INSERT INTO roledefinition VALUES (2, 'editor', 'Editor', 'Edit', '["kb:read", "feature:knowledge", "tasks:write"]', 1, 1, '2026-09-15');
|
||||
INSERT INTO roledefinition VALUES (3, 'reviewer', 'Reviewer', 'Review', '["audit:read", "review"]', 1, 1, '2026-09-15');
|
||||
INSERT INTO roledefinition VALUES (4, 'viewer', 'Viewer', 'View', '["read"]', 1, 1, '2026-09-15');
|
||||
INSERT INTO appuser VALUES (1, 'yawan_admin', 'Legacy Admin', 'admin', 1, 1, 'secret-old-hash');
|
||||
INSERT INTO appuser VALUES (3, 'editor_one', 'Display One', 'editor', 1, 1, 'secret-old-hash');
|
||||
INSERT INTO appuser VALUES (8, 'wuzong', 'Display Two', 'editor', 1, 1, 'secret-old-hash');
|
||||
INSERT INTO appuser VALUES (5, 'deleted_test', 'Test', 'viewer', 1, 0, 'secret-old-hash');
|
||||
INSERT INTO appuser VALUES (6, 'deleted_test_two', 'Test Two', 'viewer', 1, 0, 'secret-old-hash');
|
||||
INSERT INTO appuser VALUES (7, 'deleted_test_three', 'Test Three', 'viewer', 1, 0, 'secret-old-hash');
|
||||
""")
|
||||
|
||||
def plan(self):
|
||||
plan = identity_plan(self.db, TARGET_TENANT)
|
||||
plan["source_snapshot_sha256"] = "a" * 64
|
||||
return plan
|
||||
|
||||
def test_three_enabled_users_and_disabled_manifest(self):
|
||||
plan = self.plan()
|
||||
self.assertEqual([u["source_user"]["id"] for u in plan["users"]], [1, 3, 8])
|
||||
self.assertEqual(plan["skipped"][0]["reason"], "source_account_disabled")
|
||||
self.assertEqual(plan["skipped"][0]["source_user"]["id"], 5)
|
||||
self.assertEqual(len(plan["skipped"]), 3)
|
||||
self.assertNotIn("secret-old-hash", json.dumps(plan))
|
||||
admin, first, second = plan["users"]
|
||||
self.assertEqual(admin["target_id"], ADMIN_TARGET_ID)
|
||||
self.assertEqual(admin["action"], "map_existing_admin")
|
||||
for user in (first, second):
|
||||
self.assertFalse(user["is_active"])
|
||||
self.assertFalse(user["is_system_admin"])
|
||||
self.assertFalse(user["can_access_all_tenants"])
|
||||
self.assertEqual(user["membership_status"], "suspended")
|
||||
self.assertEqual(user["target_membership_role"], "contributor")
|
||||
self.assertTrue(user["target_email"].endswith("@migration.invalid"))
|
||||
self.assertIn("permission_review_required", user["pending_activation"])
|
||||
self.assertIn("password_reset_required", user["pending_activation"])
|
||||
metadata = first["preferences"][PREFERENCE_KEY]
|
||||
self.assertEqual(metadata["source_user"]["display_name"], "Display One")
|
||||
self.assertIn("tasks:write", metadata["source_user"]["permissions"])
|
||||
|
||||
def test_all_source_roles_preserved_including_roles_without_active_users(self):
|
||||
roles = {role["role"]: role for role in self.plan()["source_role_definitions"]}
|
||||
self.assertEqual(set(roles), {"admin", "editor", "reviewer", "viewer"})
|
||||
self.assertEqual(roles["reviewer"]["permissions"], ["audit:read", "review"])
|
||||
self.assertEqual(roles["reviewer"]["description"], "Review")
|
||||
self.assertEqual(roles["reviewer"]["updated_at"], "2026-09-15")
|
||||
|
||||
def test_sql_requires_snapshot_fingerprint(self):
|
||||
for digest in (None, "", "not-a-digest", "A" * 64, "a" * 63):
|
||||
plan = self.plan()
|
||||
if digest is None:
|
||||
del plan["source_snapshot_sha256"]
|
||||
else:
|
||||
plan["source_snapshot_sha256"] = digest
|
||||
with self.subTest(digest=digest), self.assertRaises(ValueError):
|
||||
identity_sql(plan)
|
||||
|
||||
def test_planner_does_not_read_password_column_or_write(self):
|
||||
def authorize(action, first, second, database, trigger):
|
||||
if action == sqlite3.SQLITE_READ and first == "appuser" and second == "password_hash":
|
||||
return sqlite3.SQLITE_DENY
|
||||
return sqlite3.SQLITE_OK
|
||||
self.db.set_authorizer(authorize)
|
||||
self.db.execute("PRAGMA query_only=ON")
|
||||
self.assertEqual(len(self.plan()["users"]), 3)
|
||||
|
||||
def test_ids_are_deterministic_and_tenant_scoped(self):
|
||||
self.assertEqual(self.plan(), self.plan())
|
||||
self.assertNotEqual(_stable_id(3, TARGET_TENANT), _stable_id(3, TARGET_TENANT + 1))
|
||||
self.assertNotEqual(_stable_id(3, TARGET_TENANT), _stable_id(8, TARGET_TENANT))
|
||||
|
||||
def test_target_tenant_rejected(self):
|
||||
for tenant in (1, TARGET_TENANT + 1, str(TARGET_TENANT), True):
|
||||
with self.subTest(tenant=tenant), self.assertRaises(ValueError):
|
||||
identity_plan(self.db, tenant)
|
||||
|
||||
def test_foreign_source_organization_rejected(self):
|
||||
self.db.execute("INSERT INTO organization VALUES (2, 'other', 'other', 1)")
|
||||
with self.assertRaisesRegex(ValueError, "organization"):
|
||||
self.plan()
|
||||
|
||||
def test_cross_organization_user_rejected_even_if_disabled(self):
|
||||
self.db.execute("UPDATE appuser SET organization_id=2 WHERE id=5")
|
||||
with self.assertRaisesRegex(ValueError, "organization"):
|
||||
self.plan()
|
||||
|
||||
def test_cross_organization_role_rejected(self):
|
||||
self.db.execute("UPDATE roledefinition SET organization_id=2 WHERE role='editor'")
|
||||
with self.assertRaisesRegex(ValueError, "organization"):
|
||||
self.plan()
|
||||
|
||||
def test_unreviewed_enabled_account_rejected(self):
|
||||
self.db.execute("INSERT INTO appuser VALUES (10, 'extra', 'Extra', 'editor', 1, 1, '')")
|
||||
with self.assertRaisesRegex(ValueError, "reviewed migration batch"):
|
||||
self.plan()
|
||||
|
||||
def test_admin_mapping_cannot_silently_change(self):
|
||||
self.db.execute("UPDATE appuser SET username='other_admin' WHERE id=1")
|
||||
with self.assertRaisesRegex(ValueError, "explicitly mapped"):
|
||||
self.plan()
|
||||
|
||||
def test_unknown_enabled_role_rejected(self):
|
||||
self.db.execute("UPDATE appuser SET role='viewer' WHERE id=3")
|
||||
with self.assertRaisesRegex(ValueError, "Unsupported"):
|
||||
self.plan()
|
||||
|
||||
def test_duplicate_source_names_rejected(self):
|
||||
self.db.execute("UPDATE appuser SET username='EDITOR_ONE' WHERE id=8")
|
||||
with self.assertRaisesRegex(ValueError, "Conflicting"):
|
||||
self.plan()
|
||||
|
||||
def test_sql_rejects_tampered_scope_or_activation(self):
|
||||
base = self.plan()
|
||||
mutations = [
|
||||
lambda p: p.update(target_tenant_id=1),
|
||||
lambda p: p.update(source="other"),
|
||||
lambda p: p.update(target_database="production"),
|
||||
lambda p: p["users"][0].update(target_id="other-admin"),
|
||||
lambda p: p["users"][1].update(is_active=True),
|
||||
lambda p: p["users"][1].update(is_system_admin=True),
|
||||
lambda p: p["users"][1].update(can_access_all_tenants=True),
|
||||
lambda p: p["users"][1].update(membership_status="active"),
|
||||
lambda p: p["users"][1].update(target_membership_role="owner"),
|
||||
lambda p: p["users"][1].update(target_email="real@example.com"),
|
||||
lambda p: p["users"][1]["source_user"].update(organization_id=2),
|
||||
lambda p: p["users"][1]["preferences"][PREFERENCE_KEY].update(source="other"),
|
||||
]
|
||||
for mutation in mutations:
|
||||
plan = copy.deepcopy(base)
|
||||
mutation(plan)
|
||||
with self.subTest(mutation=mutation), self.assertRaises(ValueError):
|
||||
identity_sql(plan)
|
||||
|
||||
def test_sql_escapes_source_text_and_does_not_emit_a_password(self):
|
||||
self.db.execute("UPDATE appuser SET display_name=? WHERE id=3", ("O'Brien\\note $identity$",))
|
||||
sql = identity_sql(self.plan())
|
||||
self.assertIn("O''Brien", sql)
|
||||
self.assertIn("SET LOCAL standard_conforming_strings = on", sql)
|
||||
self.assertIn("gen_random_uuid()", sql)
|
||||
self.assertNotIn("secret-old-hash", sql)
|
||||
|
||||
def test_sql_contract_aborts_conflicts_and_preserves_exact_reruns(self):
|
||||
sql = identity_sql(self.plan())
|
||||
self.assertIn("current_database()", sql)
|
||||
self.assertIn("lower(btrim(username))", sql)
|
||||
self.assertIn("lower(btrim(email))", sql)
|
||||
self.assertIn("ID/provenance collision", sql)
|
||||
self.assertIn("username/email collision", sql)
|
||||
self.assertIn("Orphan or deleted membership conflicts", sql)
|
||||
self.assertIn("existing_user.preferences->'yawan_identity_migration' IS DISTINCT FROM", sql)
|
||||
self.assertIn("existing_member.status NOT IN ('suspended', 'active')", sql)
|
||||
self.assertIn("unexpected cross-tenant membership", sql)
|
||||
self.assertIn("Durable identity mapping conflict", sql)
|
||||
self.assertIn("IF NOT has_mapping AND (", sql)
|
||||
self.assertIn("refusing to recreate a deleted account", sql)
|
||||
self.assertNotIn("ON CONFLICT", sql.upper())
|
||||
self.assertNotIn("UPDATE USERS", sql.upper())
|
||||
self.assertNotIn("UPDATE TENANT_MEMBERS", sql.upper())
|
||||
self.assertNotIn("COMMIT", sql.upper())
|
||||
self.assertNotIn("BEGIN TRANSACTION", sql.upper())
|
||||
self.assertEqual(sql.count("INSERT INTO users"), 1)
|
||||
self.assertEqual(sql.count("INSERT INTO tenant_members"), 1)
|
||||
|
||||
def test_sql_stores_all_identity_mappings_without_modifying_existing_accounts(self):
|
||||
sql = identity_sql(self.plan())
|
||||
self.assertIn(f"CREATE TABLE IF NOT EXISTS {ORGANIZATION_LEDGER}", sql)
|
||||
self.assertIn(f"CREATE TABLE IF NOT EXISTS {IDENTITY_LEDGER}", sql)
|
||||
self.assertIn("source_role_definitions jsonb NOT NULL", sql)
|
||||
self.assertIn("source_snapshot_sha256 varchar(64) NOT NULL", sql)
|
||||
self.assertEqual(sql.count(f"INSERT INTO {IDENTITY_LEDGER}"), 1)
|
||||
self.assertGreater(sql.index(f"INSERT INTO {IDENTITY_LEDGER}"), sql.index("Pinned administrator"))
|
||||
self.assertNotIn("CONTINUE;", sql)
|
||||
self.assertNotIn("ON CONFLICT", sql.upper())
|
||||
|
||||
def test_readonly_verifier_reports_mappings_and_current_state_without_password(self):
|
||||
sql = identity_verification_sql(self.plan())
|
||||
for field in ("database_matches", "organization_mapping_matches", "identity_mapping_matches",
|
||||
"account_scope_matches", "membership_matches", "is_active", "membership_status"):
|
||||
self.assertIn(field, sql)
|
||||
for operation in ("INSERT INTO", "UPDATE USERS", "CREATE TABLE", "DELETE FROM", "password_hash"):
|
||||
self.assertNotIn(operation, sql)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,202 @@
|
||||
"""Safe planner, source routing, and generated-SQL model migration contracts."""
|
||||
import base64
|
||||
import copy
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from weknora_model_etl import (LEDGER, TARGET_TENANT, model_id, model_plan, model_sql,
|
||||
model_verification_sql, read_credentials, resolve_agent_model)
|
||||
|
||||
|
||||
def fake_encryption(value):
|
||||
# This verifies the encoder boundary only; it is never a production cipher.
|
||||
return "enc:v1:" + base64.urlsafe_b64encode(b"x" * (12 + len(value.encode()) + 16)).decode().rstrip("=")
|
||||
|
||||
|
||||
class ModelTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.db = sqlite3.connect(":memory:")
|
||||
self.addCleanup(self.db.close)
|
||||
self.db.executescript("""
|
||||
CREATE TABLE modelprovider (id INTEGER,name TEXT,base_url TEXT,api_key TEXT,models TEXT,
|
||||
default_model TEXT,builtin INTEGER,enabled INTEGER,organization_id INTEGER);
|
||||
CREATE TABLE modelcatalog (id INTEGER,provider_id INTEGER,model_name TEXT,display_name TEXT,
|
||||
capabilities TEXT,capability_status TEXT,enabled INTEGER,builtin INTEGER,rate_version TEXT,
|
||||
input_rate REAL,output_rate REAL);
|
||||
CREATE TABLE appsetting (key TEXT,value TEXT);
|
||||
INSERT INTO modelprovider VALUES (3,'deepseek','https://api.deepseek.com/v1','source-secret-UNIQUE',
|
||||
'["deepseek-flash","deepseek-v4-pro"]','deepseek-flash',0,1,1);
|
||||
INSERT INTO modelprovider VALUES (4,'images','https://images.invalid/v1','image-secret-UNIQUE',
|
||||
'["gpt-image-2"]','gpt-image-2',0,1,1);
|
||||
INSERT INTO modelcatalog VALUES (4,3,'deepseek-v4-flash','Disabled','[]','untested',0,0,'r1',1,2);
|
||||
INSERT INTO modelcatalog VALUES (6,3,'deepseek-v4-pro','Pro','[]','untested',1,0,'r1',1,2);
|
||||
INSERT INTO modelcatalog VALUES (7,3,'deepseek-flash','Flash','[]','untested',1,0,'r1',1,2);
|
||||
INSERT INTO modelcatalog VALUES (8,4,'gpt-image-2','Image','[]','untested',1,0,'r1',1,2);
|
||||
INSERT INTO appsetting VALUES ('default_model_provider_id','3');
|
||||
""")
|
||||
|
||||
def plan(self):
|
||||
plan = model_plan(self.db)
|
||||
plan["source_snapshot_sha256"] = "a" * 64
|
||||
return plan
|
||||
|
||||
def test_reviewed_models_only_without_native_image_generation_guess(self):
|
||||
plan = self.plan()
|
||||
self.assertEqual(len(plan["models"]), 2)
|
||||
self.assertEqual(plan["catalog_map"], {"6": model_id(3, "deepseek-v4-pro"), "7": model_id(3, "deepseek-flash")})
|
||||
self.assertEqual(plan["default_gen_model_id"], model_id(3, "deepseek-flash"))
|
||||
self.assertEqual({item["reason"] for item in plan["skipped"]},
|
||||
{"source_configuration_disabled", "image_generation_has_no_native_model_type"})
|
||||
for item in plan["models"]:
|
||||
self.assertFalse(item["is_default"])
|
||||
self.assertFalse(item["is_builtin"])
|
||||
self.assertEqual(item["source_catalog"]["capability_status"], "untested")
|
||||
self.assertIn("no_paid_probe_performed", item["compatibility"])
|
||||
self.assertNotIn("secret-UNIQUE", json.dumps(plan))
|
||||
|
||||
def test_planner_reads_no_credentials_and_never_writes(self):
|
||||
def authorize(action, first, second, database, trigger):
|
||||
if action == sqlite3.SQLITE_READ and first == "modelprovider" and second == "api_key":
|
||||
return sqlite3.SQLITE_DENY
|
||||
return sqlite3.SQLITE_OK
|
||||
self.db.set_authorizer(authorize)
|
||||
self.db.execute("PRAGMA query_only=ON")
|
||||
self.assertEqual(len(self.plan()["models"]), 2)
|
||||
|
||||
def test_source_runtime_fallback_is_explicit_not_catalog_guess(self):
|
||||
result = resolve_agent_model(self.plan(), {"model_provider_id": None, "model_name": "deepseek-chat"})
|
||||
self.assertEqual(result["effective_provider_id"], 3)
|
||||
self.assertEqual(result["effective_model_name"], "deepseek-flash")
|
||||
self.assertEqual(result["requested_model_name"], "deepseek-chat")
|
||||
self.assertTrue(result["provider_fallback"])
|
||||
self.assertTrue(result["model_fallback"])
|
||||
direct = resolve_agent_model(self.plan(), {"model_provider_id": 3, "model_name": "deepseek-v4-pro"})
|
||||
self.assertEqual(direct["target_model_id"], model_id(3, "deepseek-v4-pro"))
|
||||
self.assertFalse(direct["model_fallback"])
|
||||
|
||||
def test_unsupported_effective_agent_model_does_not_silently_fall_back(self):
|
||||
with self.assertRaisesRegex(ValueError, "no reviewed native mapping"):
|
||||
resolve_agent_model(self.plan(), {"model_provider_id": 4, "model_name": "gpt-image-2"})
|
||||
|
||||
def test_global_default_uses_source_builtin_and_id_order_when_setting_absent(self):
|
||||
self.db.execute("DELETE FROM appsetting")
|
||||
self.assertEqual(self.plan()["default_provider_id"], 3)
|
||||
self.db.execute("UPDATE modelprovider SET builtin=1 WHERE id=4")
|
||||
plan = self.plan()
|
||||
self.assertEqual(plan["default_provider_id"], 4)
|
||||
self.assertIsNone(plan["default_gen_model_id"])
|
||||
with self.assertRaisesRegex(ValueError, "no reviewed native mapping"):
|
||||
resolve_agent_model(plan, {"model_provider_id": None})
|
||||
|
||||
def test_provider_default_and_first_configured_model_match_legacy_runtime(self):
|
||||
self.db.execute("UPDATE modelprovider SET default_model='' WHERE id=3")
|
||||
resolved = resolve_agent_model(self.plan(), {"model_provider_id": 3, "model_name": ""})
|
||||
self.assertEqual(resolved["effective_model_name"], "deepseek-flash")
|
||||
|
||||
def test_credentials_read_only_for_imported_models(self):
|
||||
def authorize(action, first, second, database, trigger):
|
||||
return sqlite3.SQLITE_OK
|
||||
self.db.set_authorizer(authorize)
|
||||
credentials = read_credentials(self.db, self.plan())
|
||||
self.assertEqual(set(credentials), {3})
|
||||
self.assertEqual(credentials[3], "source-secret-UNIQUE")
|
||||
|
||||
def test_source_keychain_reference_needs_resolver(self):
|
||||
self.db.execute("UPDATE modelprovider SET api_key='keychain:approved:provider-3' WHERE id=3")
|
||||
with self.assertRaisesRegex(ValueError, "resolver"):
|
||||
read_credentials(self.db, self.plan())
|
||||
credentials = read_credentials(self.db, self.plan(), lambda value, provider_id: "resolved-in-memory")
|
||||
self.assertEqual(credentials, {3: "resolved-in-memory"})
|
||||
|
||||
def test_no_plaintext_sql_or_credential_ledger_and_no_existing_model_updates(self):
|
||||
plan = self.plan()
|
||||
sql = model_sql(plan, read_credentials(self.db, plan), fake_encryption)
|
||||
self.assertNotIn("source-secret-UNIQUE", sql)
|
||||
self.assertNotIn("image-secret-UNIQUE", sql)
|
||||
self.assertNotIn("UPDATE models", sql)
|
||||
self.assertNotIn("ON CONFLICT", sql)
|
||||
self.assertIn("item - 'stored_parameters'", sql)
|
||||
self.assertIn("current_database() <> 'yawan_native_acceptance_20260915'", sql)
|
||||
self.assertIn("Model ID collision without migration provenance", sql)
|
||||
self.assertIn("existing_model.deleted_at IS NOT NULL", sql)
|
||||
self.assertIn("mapped.source_snapshot IS DISTINCT FROM", sql)
|
||||
self.assertIn("SET LOCAL log_min_error_statement = 'panic'", sql)
|
||||
self.assertIn(LEDGER, sql)
|
||||
self.assertNotIn("stored_parameters", json.dumps(plan))
|
||||
|
||||
def test_native_cipher_required_and_callback_errors_redacted(self):
|
||||
plan = self.plan()
|
||||
for encrypt in (lambda x: x, lambda x: "enc:v1:invalid", lambda x: "enc:v1:has padding=="):
|
||||
with self.subTest(encrypt=encrypt), self.assertRaisesRegex(ValueError, "encryption failed"):
|
||||
model_sql(plan, {3: "source-secret-UNIQUE"}, encrypt)
|
||||
def broken(value):
|
||||
raise RuntimeError(value)
|
||||
try:
|
||||
model_sql(plan, {3: "source-secret-UNIQUE"}, broken)
|
||||
except ValueError as error:
|
||||
self.assertNotIn("source-secret", str(error))
|
||||
self.assertTrue(error.__suppress_context__)
|
||||
else:
|
||||
self.fail("encryption failure accepted")
|
||||
|
||||
def test_missing_secrets_and_snapshot_are_rejected(self):
|
||||
plan = self.plan()
|
||||
for secret in (None, "", "keychain:reference", "enc:v1:reference", "env:reference"):
|
||||
with self.subTest(secret=secret), self.assertRaises(ValueError):
|
||||
model_sql(plan, {3: secret}, fake_encryption)
|
||||
del plan["source_snapshot_sha256"]
|
||||
with self.assertRaisesRegex(ValueError, "fingerprint"):
|
||||
model_sql(plan, {3: "in-memory"}, fake_encryption)
|
||||
|
||||
def test_tenant_protocol_and_organization_guards(self):
|
||||
for tenant in (1, str(TARGET_TENANT), True):
|
||||
with self.subTest(tenant=tenant), self.assertRaises(ValueError):
|
||||
model_plan(self.db, tenant)
|
||||
self.db.execute("UPDATE modelprovider SET organization_id=2 WHERE id=4")
|
||||
with self.assertRaisesRegex(ValueError, "organization"):
|
||||
self.plan()
|
||||
|
||||
def test_unreviewed_endpoint_and_model_explicitly_skipped(self):
|
||||
self.db.execute("UPDATE modelprovider SET base_url='https://other.invalid/v1' WHERE id=3")
|
||||
plan = self.plan()
|
||||
self.assertEqual(plan["models"], [])
|
||||
self.assertIn("model_protocol_not_reviewed", {s["reason"] for s in plan["skipped"]})
|
||||
|
||||
def test_private_url_never_enters_plan(self):
|
||||
self.db.execute("UPDATE modelprovider SET base_url='https://example.invalid/v1?key=private' WHERE id=3")
|
||||
with self.assertRaisesRegex(ValueError, "private components"):
|
||||
self.plan()
|
||||
|
||||
def test_ids_and_input_mapping_are_deterministic(self):
|
||||
self.assertEqual(self.plan(), self.plan())
|
||||
self.assertNotEqual(model_id(3, "deepseek-flash"), model_id(4, "deepseek-flash"))
|
||||
self.assertNotEqual(model_id(3, "deepseek-flash"), model_id(3, "deepseek-flash", 10000))
|
||||
plan = copy.deepcopy(self.plan())
|
||||
plan["provider_model_map"]["3:deepseek-flash"] = "wrong"
|
||||
with self.assertRaisesRegex(ValueError, "mapping"):
|
||||
resolve_agent_model(plan, {})
|
||||
|
||||
def test_sql_rejects_scope_and_native_configuration_tampering(self):
|
||||
mutations = [lambda p: p.update(target_tenant_id=10000),
|
||||
lambda p: p.update(target_database="production"),
|
||||
lambda p: p["models"][0].update(is_default=True),
|
||||
lambda p: p["models"][0].update(is_builtin=True),
|
||||
lambda p: p["models"][0]["parameters"].update(api_key="must-not-be-in-plan")]
|
||||
for mutation in mutations:
|
||||
plan = copy.deepcopy(self.plan())
|
||||
mutation(plan)
|
||||
with self.subTest(mutation=mutation), self.assertRaises(ValueError):
|
||||
model_sql(plan, {3: "in-memory"}, fake_encryption)
|
||||
|
||||
def test_verification_emits_only_secret_presence(self):
|
||||
sql = model_verification_sql()
|
||||
self.assertIn("LIKE 'enc:v1:%'", sql)
|
||||
self.assertNotIn("row_to_json", sql)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,42 @@
|
||||
import unittest
|
||||
from weknora_resource_version_etl import make_plan, transaction, TARGET_DATABASE
|
||||
|
||||
|
||||
class ResourceVersionMigrationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.row = {"id": 1, "resource_type": "agent", "resource_id": 14, "owner_user_id": 3,
|
||||
"organization_id": 1, "status": "rejected", "payload": {"name": "Old version", "rules": "kept"},
|
||||
"review_note": "copy deleted", "created_at": "2026-09-10T01:00:00Z", "reviewed_at": ""}
|
||||
self.source = {"forks": [], "reviews": [self.row], "source_sha256": "a" * 64}
|
||||
self.target = {"database": TARGET_DATABASE, "agents": {"14": "agent-id"}, "identities": {"3": "owner-id"}}
|
||||
|
||||
def test_terminal_history_preserves_payload_without_restoring_copy(self):
|
||||
first = make_plan(self.source, self.target)
|
||||
self.assertEqual(first, make_plan(self.source, self.target))
|
||||
self.assertEqual(first["rows"][0]["proposed_snapshot"]["rules"], "kept")
|
||||
sql = transaction(first)
|
||||
self.assertIn("ROLLBACK", sql)
|
||||
self.assertNotIn("INSERT INTO custom_agents", sql)
|
||||
self.assertNotIn("INSERT INTO users", sql)
|
||||
|
||||
def test_active_forks_and_pending_reviews_require_real_migration(self):
|
||||
self.source["forks"] = [{"id": 1}]
|
||||
with self.assertRaises(ValueError):
|
||||
make_plan(self.source, self.target)
|
||||
self.source["forks"] = []
|
||||
self.row["status"] = "pending"
|
||||
with self.assertRaises(ValueError):
|
||||
make_plan(self.source, self.target)
|
||||
|
||||
def test_missing_owner_and_foreign_scope_rejected(self):
|
||||
self.target["identities"] = {}
|
||||
with self.assertRaises(ValueError):
|
||||
make_plan(self.source, self.target)
|
||||
self.target["identities"] = {"3": "owner-id"}
|
||||
self.row["organization_id"] = 2
|
||||
with self.assertRaises(ValueError):
|
||||
make_plan(self.source, self.target)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Executable
+74
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
#
|
||||
# 更新 Homebrew Formula 中的版本号和 sha256
|
||||
#
|
||||
# 用法:
|
||||
# ./scripts/update-homebrew-formula.sh v0.2.0
|
||||
#
|
||||
# 会自动从 GitHub Releases 下载 .sha256 文件来填充 Formula。
|
||||
# 在 CI 中被 release-lite workflow 调用。
|
||||
#
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
VERSION="${1:?Usage: $0 <version> (e.g. v0.2.0)}"
|
||||
VERSION_BARE="${VERSION#v}"
|
||||
FORMULA="${ROOT_DIR}/Formula/weknora-lite.rb"
|
||||
REPO="Tencent/WeKnora"
|
||||
BASE_URL="https://github.com/${REPO}/releases/download/${VERSION}"
|
||||
|
||||
if [ ! -f "${FORMULA}" ]; then
|
||||
echo "Error: Formula not found at ${FORMULA}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Updating Formula to version ${VERSION_BARE}..."
|
||||
|
||||
fetch_sha256() {
|
||||
local file="$1"
|
||||
local url="${BASE_URL}/${file}.sha256"
|
||||
local sha
|
||||
sha=$(curl -sSL "${url}" | awk '{print $1}')
|
||||
if [ -z "${sha}" ] || [ "${#sha}" -ne 64 ]; then
|
||||
echo "Error: Failed to fetch sha256 from ${url}" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "${sha}"
|
||||
}
|
||||
|
||||
SHA_DARWIN_ARM64=$(fetch_sha256 "WeKnora-lite_${VERSION}_darwin_arm64.tar.gz")
|
||||
SHA_DARWIN_AMD64=$(fetch_sha256 "WeKnora-lite_${VERSION}_darwin_amd64.tar.gz")
|
||||
SHA_LINUX_ARM64=$(fetch_sha256 "WeKnora-lite_${VERSION}_linux_arm64.tar.gz")
|
||||
SHA_LINUX_AMD64=$(fetch_sha256 "WeKnora-lite_${VERSION}_linux_amd64.tar.gz")
|
||||
|
||||
echo " darwin_arm64 : ${SHA_DARWIN_ARM64}"
|
||||
echo " darwin_amd64 : ${SHA_DARWIN_AMD64}"
|
||||
echo " linux_arm64 : ${SHA_LINUX_ARM64}"
|
||||
echo " linux_amd64 : ${SHA_LINUX_AMD64}"
|
||||
|
||||
# Use a temp file for portable sed
|
||||
TMP=$(mktemp)
|
||||
cp "${FORMULA}" "${TMP}"
|
||||
|
||||
# Update version
|
||||
sed -i.bak "s/^ version \".*\"/ version \"${VERSION_BARE}\"/" "${TMP}"
|
||||
|
||||
# Update sha256 values in order of appearance.
|
||||
# The formula has sha256 lines in this order:
|
||||
# 1. darwin arm64
|
||||
# 2. darwin amd64
|
||||
# 3. linux arm64
|
||||
# 4. linux amd64
|
||||
awk -v s1="${SHA_DARWIN_ARM64}" \
|
||||
-v s2="${SHA_DARWIN_AMD64}" \
|
||||
-v s3="${SHA_LINUX_ARM64}" \
|
||||
-v s4="${SHA_LINUX_AMD64}" \
|
||||
'BEGIN{n=0} /sha256 "/{n++; if(n==1) sub(/"[^"]*"$/,"\"" s1 "\""); else if(n==2) sub(/"[^"]*"$/,"\"" s2 "\""); else if(n==3) sub(/"[^"]*"$/,"\"" s3 "\""); else if(n==4) sub(/"[^"]*"$/,"\"" s4 "\"")} {print}' \
|
||||
"${TMP}" > "${FORMULA}"
|
||||
|
||||
rm -f "${TMP}" "${TMP}.bak"
|
||||
|
||||
echo "Formula updated: ${FORMULA}"
|
||||
@@ -0,0 +1,62 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
|
||||
const root = path.resolve(__dirname, '../..')
|
||||
const credentialsPath = path.join(root, 'acceptance/super-admin-login.json')
|
||||
const base = 'http://127.0.0.1:18080/api/v1'
|
||||
const literal = value => `'${String(value).replaceAll("'", "''")}'`
|
||||
const sql = statement => execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres', 'psql', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-v', 'ON_ERROR_STOP=1', '-At'], { input: statement, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim()
|
||||
|
||||
async function request(route, data, token) {
|
||||
const response = await fetch(base + route, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json', ...(token ? { Authorization: `Bearer ${token}` } : {}) },
|
||||
body: JSON.stringify(data), signal: AbortSignal.timeout(30000),
|
||||
})
|
||||
const body = await response.json()
|
||||
assert.ok(response.ok && body.success !== false, `${route}: HTTP ${response.status}`)
|
||||
return body
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const desired = JSON.parse(fs.readFileSync(0, 'utf8'))
|
||||
assert.match(desired.username, /^[a-zA-Z0-9_-]{2,50}$/)
|
||||
assert.ok(typeof desired.password === 'string' && [...desired.password].length >= 3)
|
||||
const credentials = JSON.parse(fs.readFileSync(credentialsPath, 'utf8'))
|
||||
const oldLogin = await request('/auth/login', { email: credentials.email, password: credentials.password })
|
||||
assert.equal(oldLogin.user.is_system_admin, true, 'Saved identity must already be a system administrator')
|
||||
assert.equal(oldLogin.user.is_active, true)
|
||||
const userID = oldLogin.user.id
|
||||
assert.match(userID, /^[a-f0-9-]{36}$/)
|
||||
assert.equal(sql(`SELECT count(*) FROM users WHERE id<>${literal(userID)} AND LOWER(username)=LOWER(${literal(desired.username)});`), '0', 'Requested account name already exists')
|
||||
|
||||
const backup = path.join(root, 'acceptance', `super-admin-login-before-${Date.now()}.json`)
|
||||
fs.writeFileSync(backup, JSON.stringify(credentials, null, 2) + '\n', { flag: 'wx', mode: 0o600 })
|
||||
// Keep the user ID and all ownership links; only change the login name.
|
||||
sql(`BEGIN;
|
||||
SELECT id FROM users WHERE id=${literal(userID)} AND is_system_admin=true AND is_active=true AND deleted_at IS NULL FOR UPDATE;
|
||||
UPDATE users SET username=${literal(desired.username)},updated_at=now() WHERE id=${literal(userID)} AND is_system_admin=true AND is_active=true AND deleted_at IS NULL;
|
||||
INSERT INTO audit_logs (tenant_id,actor_user_id,actor_role,action,target_type,target_id,target_user_id,outcome,details,created_at)
|
||||
SELECT tenant_id,id,'admin','system.user_updated','user',id,id,'success',${literal(JSON.stringify({ username_before: oldLogin.user.username, username_after: desired.username, source: 'authorized_account_update' }))}::jsonb,now() FROM users WHERE id=${literal(userID)};
|
||||
COMMIT;`)
|
||||
const renamed = await request('/auth/login', { username: desired.username, password: credentials.password })
|
||||
assert.equal(renamed.user.id, userID)
|
||||
if (credentials.password !== desired.password) {
|
||||
await request('/auth/change-password', { old_password: credentials.password, new_password: desired.password }, renamed.token)
|
||||
}
|
||||
const updated = { ...credentials, username: desired.username, password: desired.password }
|
||||
const temporary = credentialsPath + '.next'
|
||||
fs.writeFileSync(temporary, JSON.stringify(updated, null, 2) + '\n', { mode: 0o600 })
|
||||
fs.renameSync(temporary, credentialsPath)
|
||||
const current = await request('/auth/login', { username: desired.username, password: desired.password })
|
||||
assert.equal(current.user.id, userID)
|
||||
assert.equal(current.user.is_system_admin, true)
|
||||
if (credentials.password !== desired.password) {
|
||||
const revoked = await fetch(base + '/auth/me', { headers: { Authorization: `Bearer ${oldLogin.token}` }, signal: AbortSignal.timeout(15000) })
|
||||
assert.equal(revoked.status, 401, 'Previous login must be revoked after password rotation')
|
||||
}
|
||||
console.log(JSON.stringify({ passed: true, username: current.user.username, identity_preserved: true, system_admin: true, password_rotated: credentials.password !== desired.password }))
|
||||
}
|
||||
|
||||
main().catch(error => { console.error(error.message); process.exitCode = 1 })
|
||||
@@ -0,0 +1,279 @@
|
||||
const startedAt = new Date().toISOString()
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const crypto = require('node:crypto')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
|
||||
const base = (process.env.VERIFY_API_ORIGIN || 'http://127.0.0.1:18080') + '/api/v1'
|
||||
const origin = process.env.VERIFY_FRONTEND_ORIGIN || 'http://127.0.0.1:15173'
|
||||
const reportName = process.env.VERIFY_REPORT_NAME || 'access-control-live'
|
||||
assert.match(reportName, /^[a-zA-Z0-9_-]+$/)
|
||||
const prefix = `access-check-${Date.now()}`
|
||||
const checks = [], cleanup = [], failures = []
|
||||
let admin, member, browser
|
||||
const check = (name, value) => { checks.push({name, passed:!!value}); assert.ok(value, name) }
|
||||
const sql = statement => execFileSync('docker', ['--context','colima-yawan','exec','WeKnora-postgres','psql','-U','postgres','-d','yawan_native_acceptance_20260915','-v','ON_ERROR_STOP=1','-Atc',statement],{encoding:'utf8',stdio:['ignore','pipe','pipe']}).trim()
|
||||
const literal = value => `'${String(value).replaceAll("'", "''")}'`
|
||||
function validateResponse(route, method, result, status, expected, onUnexpectedSuccess) {
|
||||
const accepted=(Array.isArray(expected)?expected:[expected]).includes(status)
|
||||
if(!accepted&&status>=200&&status<300&&onUnexpectedSuccess)onUnexpectedSuccess(result)
|
||||
assert.ok(accepted, `${method} ${route}: HTTP ${status}; ${result.message || result.error?.message || ''}`)
|
||||
return result
|
||||
}
|
||||
function trackCreatedResource(collection, excludedIDs=[], actions=cleanup, remove=route=>api(route,'DELETE',undefined,admin,[200,202,204,404]), executeSQL=sql) {
|
||||
return result=>{
|
||||
const data=result.data
|
||||
const id=collection==='/knowledge-bases' ? data?.target_id || data?.knowledge_base?.id
|
||||
: collection==='/workbench/resources/flows' ? data?.personal_id : data?.id
|
||||
assert.ok(typeof id==='string'&&id&&!excludedIDs.includes(id),'Unexpected creation must return a new resource ID')
|
||||
actions.push(()=>remove(`${collection}/${encodeURIComponent(id)}?detach=true`))
|
||||
if(collection==='/workbench/resources/flows'){
|
||||
assert.ok(typeof data.id==='string'&&data.id&&data.id!==id,'Unexpected fork must return its separate metadata ID')
|
||||
actions.push(()=>executeSQL(`BEGIN;
|
||||
DELETE FROM workbench_resource_forks WHERE tenant_id=10059 AND kind='flows' AND id=${literal(data.id)} AND personal_id=${literal(id)};
|
||||
DELETE FROM workbench_access_publications WHERE tenant_id=10059 AND kind='flows' AND resource_id=${literal(id)};
|
||||
COMMIT;`))
|
||||
}
|
||||
}
|
||||
}
|
||||
async function finishVerification({browser,cleanup,failures,writeReport}) {
|
||||
if(browser)try{await browser.close()}catch(error){failures.push('browser close: '+error.message)}
|
||||
for(const action of [...cleanup].reverse())try{await action()}catch(error){failures.push('cleanup: '+error.message)}
|
||||
await writeReport()
|
||||
}
|
||||
async function api(route, method='GET', body, identity=admin, expected=200, onUnexpectedSuccess) {
|
||||
const form = body instanceof FormData
|
||||
const response = await fetch(base+route,{method,headers:{...(form?{}:{'Content-Type':'application/json'}),...(identity?{Authorization:`Bearer ${identity.token}`,'X-Tenant-ID':'10059'}:{})},body:body===undefined?undefined:form?body:JSON.stringify(body),signal:AbortSignal.timeout(30000)})
|
||||
const result = response.status===204 ? {} : await response.json()
|
||||
return validateResponse(route,method,result,response.status,expected,onUnexpectedSuccess)
|
||||
}
|
||||
async function browserContext(identity, viewport={width:1440,height:1000}) {
|
||||
const context = await browser.newContext({viewport})
|
||||
await context.addInitScript(a=>{
|
||||
const values={weknora_user:JSON.stringify({...a.user,tenant_id:String(a.user.tenant_id)}),weknora_tenant:JSON.stringify({...a.active_tenant,id:'10059'}),weknora_token:a.token,weknora_refresh_token:a.refresh_token,weknora_memberships:JSON.stringify(a.memberships)}
|
||||
for(const [key,value] of Object.entries(values))localStorage.setItem(key,value)
|
||||
localStorage.setItem('weknora:new-user-guide-done:v1','1')
|
||||
},identity)
|
||||
return context
|
||||
}
|
||||
async function main() {
|
||||
const credentials=JSON.parse(fs.readFileSync(`${directory}/super-admin-login.json`,'utf8'))
|
||||
admin=await api('/auth/login','POST',{email:credentials.email,password:credentials.password},null)
|
||||
const role=(await api('/access/roles','POST',{name:prefix,description:'Disposable permission verification',permissions:['content.read']})).data
|
||||
cleanup.push(()=>api(`/access/roles/${role.id}`,'DELETE'))
|
||||
check('new_role_uses_explicit_permission_version',role.permission_version===2)
|
||||
await api('/access/roles','POST',{name:prefix+'-invalid',permissions:['content.write']},admin,400,trackCreatedResource('/access/roles',[role.id]))
|
||||
await api(`/access/roles/${role.id}`,'PUT',{...role,version:0},admin,409)
|
||||
check('role_validation_and_optimistic_version',true)
|
||||
const password=crypto.randomBytes(18).toString('base64url')
|
||||
const userForm={username:prefix,email:`${prefix}@example.test`,password,role:'contributor',status:'active',custom_role_id:role.id}
|
||||
const user=(await api('/access/users','POST',userForm)).data
|
||||
let userDeleted=false
|
||||
cleanup.push(async()=>{if(!userDeleted)await api(`/access/users/${user.id}`,'DELETE')})
|
||||
member=await api('/auth/login','POST',{email:userForm.email,password},null)
|
||||
const initialAccess=(await api('/access/me','GET',undefined,member)).data
|
||||
check('created_user_can_login',initialAccess.restricted)
|
||||
check('new_role_has_exact_read_only_permissions',JSON.stringify(initialAccess.permissions)===JSON.stringify(['content.read']))
|
||||
await api('/access/users','GET',undefined,member,403)
|
||||
await api('/access/resources','GET',undefined,member,403)
|
||||
await api('/agents','GET',undefined,member,403)
|
||||
await api('/agents/draft','POST',{model_id:'unavailable',description:'This request must not invoke a model.'},member,403)
|
||||
await api('/knowledge-bases','GET',undefined,member,403)
|
||||
await api('/shared-agents','GET',undefined,member,403)
|
||||
await api('/shared-knowledge-bases','GET',undefined,member,403)
|
||||
await api('/workbench/resources/content-tasks','POST',{name:prefix+'-blocked',data:{}},member,403,trackCreatedResource('/workbench/resources/content-tasks'))
|
||||
await api('/workbench/resources/content-tasks','GET',undefined,member)
|
||||
await api(`/access/users/${admin.user.id}`,'DELETE',undefined,admin,400)
|
||||
await api(`/access/roles/${role.id}`,'DELETE',undefined,admin,409)
|
||||
check('feature_and_admin_routes_enforced',true)
|
||||
const {chromium}=require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
|
||||
browser=await chromium.launch({headless:true,channel:'chrome'})
|
||||
const restrictedContext=await browserContext(member), restrictedPage=await restrictedContext.newPage()
|
||||
const pageFailures=[]
|
||||
restrictedPage.on('pageerror',e=>pageFailures.push(e.message))
|
||||
restrictedPage.on('response',r=>{if(r.status()>=400 && new URL(r.url()).pathname.startsWith('/api/'))pageFailures.push(`${r.status()} ${new URL(r.url()).pathname}`)})
|
||||
await restrictedPage.goto(origin+'/platform/workbench/content?tab=tasks')
|
||||
await restrictedPage.waitForLoadState('networkidle')
|
||||
check('restricted_menu_hides_agents_and_access',await restrictedPage.locator('[data-guide="nav-agents"], [data-guide="nav-workbench/access"]').count()===0)
|
||||
check('read_only_content_has_no_create_button',await restrictedPage.getByRole('button',{name:'创建内容任务',exact:true}).count()===0)
|
||||
await restrictedPage.locator('[data-guide="user-menu"]').click()
|
||||
const memberItems = await restrictedPage.locator('.user-dropdown .menu-item').allTextContents()
|
||||
check('ordinary_user_menu_has_only_manual_and_logout',memberItems.length===2 && memberItems.some(text=>text.trim()==='操作手册') && !memberItems.some(text=>/设置|管理/.test(text)))
|
||||
await restrictedPage.locator('.dropdown-user-header').click()
|
||||
check('ordinary_user_identity_does_not_mount_settings',await restrictedPage.locator('.settings-overlay').count()===0)
|
||||
await restrictedPage.screenshot({path:`${directory}/${reportName}-member-menu.png`,animations:'disabled'})
|
||||
for(const section of ['general','userprofile','tenant','members','models','skills','runtime-queues']) {
|
||||
await restrictedPage.goto(origin+'/platform/settings?section='+section)
|
||||
await restrictedPage.waitForURL('**/platform/help')
|
||||
check('ordinary_user_settings_route_blocked:'+section,await restrictedPage.locator('.settings-overlay,.settings-modal').count()===0)
|
||||
}
|
||||
await restrictedPage.goto(origin+'/platform/workbench/access')
|
||||
await restrictedPage.waitForURL('**/platform/help')
|
||||
check('direct_page_access_redirects',true)
|
||||
check('real_member_help_hides_admin_chapters',await restrictedPage.locator('.help-section[id^="admin-"]').count()===0)
|
||||
check('restricted_page_no_failed_background_requests: '+pageFailures.join(', '),pageFailures.length===0)
|
||||
await restrictedContext.close()
|
||||
const resourcePermissions=['content.read','content.write','flows.read','flows.write','flows.delete','agents.read','agents.write','agents.skills','knowledge.read','knowledge.write','digital-humans.read']
|
||||
const updatedRole=(await api(`/access/roles/${role.id}`,'PUT',{...role,permissions:resourcePermissions})).data
|
||||
const updatedAccess=(await api('/access/me','GET',undefined,member)).data
|
||||
check('updated_role_keeps_explicit_permission_version',updatedRole.permission_version===2)
|
||||
check('updated_role_does_not_expand_write_permissions',JSON.stringify([...updatedAccess.permissions].sort())===JSON.stringify([...resourcePermissions].sort()))
|
||||
const moduleContext=await browserContext(member),modulePage=await moduleContext.newPage(),moduleFailures=[]
|
||||
modulePage.on('pageerror',e=>moduleFailures.push(e.message))
|
||||
modulePage.on('response',r=>{if(r.status()>=400 && new URL(r.url()).pathname.startsWith('/api/'))moduleFailures.push(`${r.status()} ${new URL(r.url()).pathname}`)})
|
||||
for(const route of ['workbench/flows','digital-humans','agents','knowledge-bases']){
|
||||
await modulePage.goto(origin+'/platform/'+route)
|
||||
await modulePage.waitForLoadState('networkidle')
|
||||
}
|
||||
check('permitted_modules_without_chat_or_sharing_load: '+moduleFailures.join(', '),moduleFailures.length===0)
|
||||
await moduleContext.close()
|
||||
const agents=(await api('/agents')).data
|
||||
const source=agents.find(a=>a.config?.model_id)
|
||||
assert.ok(source,'Configured model fixture')
|
||||
const agent=(await api('/agents','POST',{name:prefix+'-agent',config:{...source.config,knowledge_bases:[],knowledge_base_ids:[],kb_selection_mode:'none',skills:[],instruction_skill_ids:[],fusion_agent_ids:[],digital_human_id:''}},admin,201)).data
|
||||
cleanup.push(()=>api(`/agents/${agent.id}?detach=true`,'DELETE'))
|
||||
const flow=(await api('/workbench/resources/flows','POST',{name:prefix+'-flow',data:{nodes:[{id:'a',label:'A',agent_id:agent.id}],edges:[]}},admin,201)).data
|
||||
cleanup.push(()=>api(`/workbench/resources/flows/${flow.id}?detach=true`,'DELETE'))
|
||||
const form=new FormData();form.set('name',prefix+'-skill');form.set('file',new Blob(['# Review\nCheck structure and citations.'],{type:'text/markdown'}),'review.md')
|
||||
const skill=(await api('/skills/instructions/import','POST',form,admin,201)).data
|
||||
cleanup.push(()=>api(`/skills/instructions/${skill.id}?detach=true`,'DELETE'))
|
||||
const human=(await api('/digital-humans','POST',{name:prefix+'-human',model_id:source.config.model_id},admin,201)).data
|
||||
cleanup.push(()=>api(`/digital-humans/${human.id}?detach=true`,'DELETE'))
|
||||
const kb=(await api('/knowledge-bases','POST',{name:prefix+'-knowledge',type:'document',embedding_model_id:'a2c3ce32-6192-46c8-8072-910abfe8afed',summary_model_id:source.config.model_id,indexing_strategy:{vector_enabled:true,keyword_enabled:true,wiki_enabled:false,graph_enabled:false}},admin,[200,201])).data
|
||||
cleanup.push(()=>api(`/knowledge-bases/${kb.id}`,'DELETE',undefined,admin,[200,202]))
|
||||
await api(`/agents/${agent.id}`,'PUT',{name:agent.name,config:{...agent.config,kb_selection_mode:'selected',knowledge_bases:[kb.id],instruction_skill_ids:[skill.id]}})
|
||||
// A metadata-only fixture verifies document authorization without embedding or model calls.
|
||||
const knowledgeID=crypto.randomUUID()
|
||||
sql(`INSERT INTO knowledges (id,tenant_id,knowledge_base_id,type,title,source,parse_status,enable_status,created_at,updated_at) VALUES (${literal(knowledgeID)},10059,${literal(kb.id)},'manual',${literal(prefix+'-document')},'manual','completed','enabled',now(),now())`)
|
||||
cleanup.push(()=>sql(`DELETE FROM knowledges WHERE id=${literal(knowledgeID)}`))
|
||||
const fixtures=[{kind:'agents',id:agent.id,route:`/agents/${agent.id}`},{kind:'flows',id:flow.id,route:`/workbench/resources/flows/${flow.id}`},{kind:'skills',id:skill.id,route:`/skills/instructions/${skill.id}`},{kind:'digital-humans',id:human.id,route:`/digital-humans/${human.id}`},{kind:'knowledge',id:kb.id,route:`/knowledge-bases/${kb.id}`}]
|
||||
const resources=fixtures.map(({kind,id})=>({kind,id}))
|
||||
cleanup.push(async()=>{for(const table of ['workbench_access_grants','workbench_access_publications'])sql(`DELETE FROM ${table} WHERE tenant_id=10059 AND resource_id IN (${fixtures.map(f=>literal(f.id)).join(',')})`)})
|
||||
async function grants(action){await api('/access/grants','POST',{resources,user_ids:['grant','revoke'].includes(action)?[user.id]:[],action})}
|
||||
await grants('unpublish')
|
||||
for(const fixture of fixtures)await api(fixture.route,'GET',undefined,member,[403,404])
|
||||
await grants('grant')
|
||||
for(const fixture of fixtures)await api(fixture.route,'GET',undefined,member)
|
||||
await api(`/agents/${human.primary_agent_id}`,'GET',undefined,member)
|
||||
check('five_resource_types_and_derived_agent_grant',true)
|
||||
check('granted_document_batch_visible',(await api(`/knowledge/batch?ids=${knowledgeID}`,'GET',undefined,member)).data.some(k=>k.id===knowledgeID))
|
||||
await api(`/knowledge/${knowledgeID}/download`,'GET',undefined,member,403)
|
||||
await api(`/knowledge-bases/${kb.id}/faq/entries/export`,'GET',undefined,member,403)
|
||||
check('grants_do_not_allow_document_download_or_export',true)
|
||||
await api(`/agents/${agent.id}/copy`,'POST',{},member,403,trackCreatedResource('/agents',[agent.id]))
|
||||
await api(`/knowledge-bases/${kb.id}/duplicate`,'POST',{},member,403,trackCreatedResource('/knowledge-bases',[kb.id]))
|
||||
check('grants_do_not_allow_resource_duplication',true)
|
||||
await api(`/workbench/resources/flows/${flow.id}`,'PUT',{name:'forbidden',version:flow.version,data:flow.data},member,[403,404])
|
||||
await api(`/agents/${agent.id}`,'PUT',{name:'forbidden',config:agent.config},member,403)
|
||||
await api(`/knowledge-bases/${kb.id}/knowledge/manual`,'POST',{title:prefix+'-forbidden',content:'must not be created'},member,403,trackCreatedResource('/knowledge'))
|
||||
check('grants_do_not_transfer_ownership',true)
|
||||
await grants('revoke')
|
||||
for(const fixture of fixtures)await api(fixture.route,'GET',undefined,member,[403,404])
|
||||
await api(`/agents/${human.primary_agent_id}`,'GET',undefined,member,[403,404])
|
||||
check('revoked_document_batch_hidden',!(await api(`/knowledge/batch?ids=${knowledgeID}`,'GET',undefined,member)).data.some(k=>k.id===knowledgeID))
|
||||
await grants('publish')
|
||||
for(const fixture of fixtures)await api(fixture.route,'GET',undefined,member)
|
||||
await api(`/agents/${human.primary_agent_id}`,'GET',undefined,member)
|
||||
check('published_document_batch_visible',(await api(`/knowledge/batch?ids=${knowledgeID}`,'GET',undefined,member)).data.some(k=>k.id===knowledgeID))
|
||||
await api(`/knowledge/${knowledgeID}/download`,'GET',undefined,member,403)
|
||||
await api(`/knowledge-bases/${kb.id}/faq/entries/export`,'GET',undefined,member,403)
|
||||
check('global_publication_does_not_allow_document_export',true)
|
||||
await api(`/agents/${agent.id}/copy`,'POST',{},member,403,trackCreatedResource('/agents',[agent.id]))
|
||||
await api(`/knowledge-bases/${kb.id}/duplicate`,'POST',{},member,403,trackCreatedResource('/knowledge-bases',[kb.id]))
|
||||
await api(`/workbench/resources/flows/${flow.id}`,'DELETE',undefined,member,[403,404])
|
||||
await api(`/skills/instructions/${skill.id}?detach=true`,'DELETE',undefined,member,[403,404])
|
||||
await api(`/agents/${agent.id}`,'PUT',{name:'forbidden',config:agent.config},member,403)
|
||||
await api(`/resource-versions/flows/${flow.id}/fork`,'POST',{},member,410,trackCreatedResource('/workbench/resources/flows',[flow.id]))
|
||||
check('global_resources_deny_copy_mutation_and_retired_forks',true)
|
||||
const personalAgent=(await api('/agents','POST',{name:prefix+'-member-agent',config:{...source.config,knowledge_bases:[],knowledge_base_ids:[],kb_selection_mode:'none',skills:[],instruction_skill_ids:[],fusion_agent_ids:[],digital_human_id:''}},member,201)).data
|
||||
cleanup.push(()=>api(`/agents/${personalAgent.id}?detach=true`,'DELETE'))
|
||||
await api('/skills/instructions/mount','POST',{agent_ids:[personalAgent.id],skill_ids:[skill.id],mode:'add'},member)
|
||||
check('published_skill_mounts_on_members_own_agent',(await api(`/agents/${personalAgent.id}`,'GET',undefined,member)).data.config.instruction_skill_ids.includes(skill.id))
|
||||
const personalFlow=(await api('/workbench/resources/flows','POST',{name:prefix+'-member-flow',data:{nodes:[{id:'shared',label:'Shared Agent',agent_id:agent.id}],edges:[]}},member,201)).data
|
||||
cleanup.push(()=>api(`/workbench/resources/flows/${personalFlow.id}?detach=true`,'DELETE'))
|
||||
check('published_agent_usable_in_members_own_canvas',personalFlow.owner_id===member.user.id)
|
||||
const editedFlow=(await api(`/workbench/resources/flows/${personalFlow.id}`,'PUT',{name:prefix+'-admin-edited',version:personalFlow.version,data:personalFlow.data})).data
|
||||
check('administrator_can_edit_members_canvas',editedFlow.name===prefix+'-admin-edited'&&editedFlow.owner_id===member.user.id)
|
||||
const editedAgent=(await api(`/agents/${personalAgent.id}`,'PUT',{name:prefix+'-admin-agent',config:personalAgent.config})).data
|
||||
check('administrator_can_edit_members_agent',editedAgent.name===prefix+'-admin-agent'&&editedAgent.created_by===member.user.id)
|
||||
await api('/access/grants','POST',{resources:[{kind:'agents',id:agent.id},{kind:'knowledge',id:kb.id},{kind:'skills',id:skill.id}],user_ids:[],action:'unpublish'})
|
||||
for(const fixture of fixtures.filter(item=>['agents','knowledge','skills'].includes(item.kind)))await api(fixture.route,'GET',undefined,member,[403,404])
|
||||
const sharedFlowTask=(await api('/workbench/resources/content-tasks','POST',{name:prefix+'-shared-flow-task',data:{flow_id:flow.id,source_text:'第一天:识别主题;第二天:练习应用。'}},member,201)).data
|
||||
cleanup.push(()=>api(`/workbench/resources/content-tasks/${sharedFlowTask.id}?detach=true`,'DELETE'))
|
||||
check('member_can_create_task_using_canvas_private_dependencies',sharedFlowTask.owner_id===member.user.id&&sharedFlowTask.data.flow_id===flow.id&&sharedFlowTask.data.knowledge_base_ids.includes(kb.id))
|
||||
await grants('unpublish')
|
||||
for(const fixture of fixtures)await api(fixture.route,'GET',undefined,member,[403,404])
|
||||
await api(`/agents/${human.primary_agent_id}`,'GET',undefined,member,[403,404])
|
||||
check('unpublished_document_batch_hidden',!(await api(`/knowledge/batch?ids=${knowledgeID}`,'GET',undefined,member)).data.some(k=>k.id===knowledgeID))
|
||||
check('revoke_publish_unpublish_effective',true)
|
||||
const adminContext=await browserContext(admin),page=await adminContext.newPage(),adminErrors=[]
|
||||
page.on('pageerror',e=>adminErrors.push(e.name))
|
||||
page.on('response',r=>{if(r.status()>=400&&new URL(r.url()).pathname.startsWith('/api/'))adminErrors.push(`${r.status()} ${new URL(r.url()).pathname}`)})
|
||||
await page.goto(origin+'/platform/workbench/access')
|
||||
await page.getByRole('heading',{name:'访问控制',exact:true}).waitFor()
|
||||
await page.getByText('角色权限',{exact:true}).click()
|
||||
await page.getByRole('button',{name:'创建角色',exact:true}).click()
|
||||
const catalog=(await api('/access/catalog')).data
|
||||
const roleDrawer=page.locator('.t-drawer--open').filter({hasText:'创建角色'})
|
||||
await roleDrawer.waitFor({state:'visible'})
|
||||
// Groups mount asynchronously; wait for the rendered count instead of racing it.
|
||||
await page.waitForFunction(expected=>document.querySelectorAll('.t-drawer--open .permission-group-heading').length>=expected,catalog.length,{timeout:20000}).catch(()=>{})
|
||||
check('role_drawer_lists_every_sidebar_module',await roleDrawer.locator('.permission-group-heading').count()===catalog.length && catalog.length>=13)
|
||||
const adminGroups=catalog.filter(group=>group.admin_only)
|
||||
check('catalog_contains_administrator_only_groups',adminGroups.length>0)
|
||||
for(const group of adminGroups){
|
||||
const heading=page.getByText(group.name,{exact:true})
|
||||
const panel=roleDrawer.locator('.t-collapse-panel').filter({has:heading})
|
||||
const body=panel.locator('.t-collapse-panel__body')
|
||||
if(!await body.isVisible())await panel.locator('.t-collapse-panel__header').click()
|
||||
await body.locator('.admin-permission-note').waitFor({state:'visible'})
|
||||
check('administrator_group_has_no_grant_checkboxes:'+group.id,await body.locator('input[type="checkbox"],[role="checkbox"],.t-checkbox').count()===0)
|
||||
for(const permission of group.permissions)await body.locator('.permission-options').getByText(permission.name,{exact:true}).waitFor({state:'visible'})
|
||||
}
|
||||
await roleDrawer.locator('.admin-permission-note').first().scrollIntoViewIfNeeded()
|
||||
await page.screenshot({path:`${directory}/${reportName}-roles-desktop.png`,animations:'disabled'})
|
||||
if(process.env.VERIFY_PC_ONLY!=='1'){
|
||||
await page.setViewportSize({width:390,height:844})
|
||||
await roleDrawer.locator('.admin-permission-note').first().scrollIntoViewIfNeeded()
|
||||
await page.screenshot({path:`${directory}/${reportName}-roles-mobile.png`,animations:'disabled'})
|
||||
check('role_drawer_mobile_no_overflow',await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth+2))
|
||||
}
|
||||
await roleDrawer.getByRole('button',{name:'取消',exact:true}).click()
|
||||
await page.setViewportSize({width:1440,height:1000})
|
||||
await page.getByText('资源授权',{exact:true}).click()
|
||||
await page.getByText(prefix+'-agent',{exact:true}).waitFor()
|
||||
await page.screenshot({path:`${directory}/${reportName}-desktop.png`,animations:'disabled'})
|
||||
if(process.env.VERIFY_PC_ONLY!=='1'){
|
||||
await page.setViewportSize({width:390,height:844})
|
||||
await page.screenshot({path:`${directory}/${reportName}-mobile.png`,animations:'disabled'})
|
||||
check('access_mobile_no_horizontal_overflow',await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth+2))
|
||||
}
|
||||
check('admin_access_page_no_errors',adminErrors.length===0)
|
||||
await adminContext.close()
|
||||
await api(`/access/users/${user.id}`,'PUT',{...userForm,password:'',role:'viewer',custom_role_id:''})
|
||||
const viewerAccess=(await api('/access/me','GET',undefined,member)).data
|
||||
check('native_viewer_does_not_advertise_creation',!viewerAccess.permissions.includes('content.write')&&!viewerAccess.permissions.includes('digital-humans.write')&&!viewerAccess.permissions.includes('flows.write'))
|
||||
await api('/workbench/resources/content-tasks','POST',{name:prefix+'-viewer-blocked',data:{}},member,403,trackCreatedResource('/workbench/resources/content-tasks'))
|
||||
await api(`/access/users/${user.id}`,'PUT',{...userForm,password:'',status:'suspended'})
|
||||
await api('/access/me','GET',undefined,member,[401,403])
|
||||
check('suspend_revokes_existing_session',true)
|
||||
// Simulate a migrated inactive account using only the disposable fixture.
|
||||
sql(`UPDATE users SET is_active=false WHERE id=${literal(user.id)}`)
|
||||
await api(`/access/users/${user.id}`,'PUT',{...userForm,password:'',status:'active'},admin,400)
|
||||
const nextPassword=crypto.randomBytes(18).toString('base64url'),nextEmail=`updated-${prefix}@example.test`
|
||||
await api(`/access/users/${user.id}`,'PUT',{...userForm,email:nextEmail,password:nextPassword})
|
||||
member=await api('/auth/login','POST',{email:nextEmail,password:nextPassword},null)
|
||||
check('migrated_account_activation_requires_reset_and_allows_email_edit',true)
|
||||
check('deletion_fixture_has_membership_and_assignment',sql(`SELECT (SELECT count(*) FROM tenant_members WHERE tenant_id=10059 AND user_id=${literal(user.id)} AND deleted_at IS NULL), (SELECT count(*) FROM workbench_access_assignments WHERE tenant_id=10059 AND user_id=${literal(user.id)})`)==='1|1')
|
||||
await api(`/access/users/${user.id}`,'DELETE');userDeleted=true
|
||||
await api('/access/me','GET',undefined,member,[401,403])
|
||||
check('delete_removes_account_from_list',!(await api('/access/users')).data.some(u=>u.id===user.id))
|
||||
check('delete_removes_membership_and_assignments',sql(`SELECT (SELECT count(*) FROM tenant_members WHERE user_id=${literal(user.id)} AND deleted_at IS NULL), (SELECT count(*) FROM tenant_members WHERE tenant_id=10059 AND user_id=${literal(user.id)} AND deleted_at IS NOT NULL), (SELECT count(*) FROM workbench_access_assignments WHERE user_id=${literal(user.id)})`)==='0|1|0')
|
||||
}
|
||||
if(require.main===module)main().catch(e=>{failures.push(e.message);process.exitCode=1}).finally(()=>finishVerification({browser,cleanup,failures,writeReport:()=>{
|
||||
fs.writeFileSync(`${directory}/${reportName}.json`,JSON.stringify({started_at:startedAt,time:new Date().toISOString(),origin,api_origin:new URL(base).origin,checks,failures,passed:failures.length===0},null,2))
|
||||
console.log(JSON.stringify({checks,failures,passed:failures.length===0}))
|
||||
if(failures.length)process.exitCode=1
|
||||
}}))
|
||||
|
||||
module.exports={validateResponse,trackCreatedResource,finishVerification}
|
||||
@@ -0,0 +1,77 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
|
||||
|
||||
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
|
||||
const origin = process.env.VERIFY_FRONTEND_ORIGIN || 'https://agent.pducv.cn'
|
||||
const checks = [], failures = []
|
||||
let browser
|
||||
const check = (name, result) => { checks.push({name, passed: !!result}); assert.ok(result, name) }
|
||||
|
||||
async function main() {
|
||||
const credentials = JSON.parse(fs.readFileSync(`${directory}/super-admin-login.json`, 'utf8'))
|
||||
const response = await fetch(`${origin}/api/v1/auth/login`, {
|
||||
method: 'POST', headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({email: credentials.email, password: credentials.password}),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
})
|
||||
assert.equal(response.status, 200)
|
||||
const auth = await response.json()
|
||||
browser = await chromium.launch({headless: true, channel: 'chrome'})
|
||||
const context = await browser.newContext({viewport: {width: 1440, height: 1000}})
|
||||
await context.addInitScript(a => {
|
||||
const values = {
|
||||
weknora_user: JSON.stringify({...a.user, tenant_id: String(a.user.tenant_id)}),
|
||||
weknora_tenant: JSON.stringify({...a.active_tenant, id: '10059'}),
|
||||
weknora_token: a.token, weknora_refresh_token: a.refresh_token,
|
||||
weknora_memberships: JSON.stringify(a.memberships),
|
||||
}
|
||||
for (const [key, value] of Object.entries(values)) localStorage.setItem(key, value)
|
||||
localStorage.setItem('weknora:new-user-guide-done:v1', '1')
|
||||
}, auth)
|
||||
const page = await context.newPage(), errors = [], writes = []
|
||||
page.on('pageerror', e => errors.push(e.name))
|
||||
page.on('request', request => {
|
||||
if (request.method() === 'POST') writes.push(new URL(request.url()).pathname)
|
||||
})
|
||||
await page.goto(`${origin}/platform/agents`)
|
||||
await page.getByRole('button', {name: 'AI 创建', exact: true}).waitFor()
|
||||
await page.getByRole('button', {name: '手动创建', exact: true}).click()
|
||||
await page.locator('.settings-modal').waitFor()
|
||||
check('manual_creation_opens_editor_without_ai', !writes.includes('/api/v1/agents/draft'))
|
||||
await page.screenshot({path: `${directory}/public-agent-manual-desktop.png`, animations: 'disabled'})
|
||||
await page.locator('.settings-modal .close-btn').click()
|
||||
await page.locator('.settings-modal').waitFor({state: 'hidden'})
|
||||
await page.getByRole('button', {name: 'AI 创建', exact: true}).click()
|
||||
const dialog = page.locator('.t-dialog').filter({hasText: 'AI 创建 Agent'})
|
||||
await dialog.locator('textarea').fill('创建一个课程规划 Agent,面向新学员整理连续课程框架,每天衔接前一天的核心概念,并检查主题重复。只生成配置草稿。')
|
||||
const generate = dialog.getByRole('button', {name: '生成配置', exact: true})
|
||||
await page.setViewportSize({width: 390, height: 844})
|
||||
const draftResponse = page.waitForResponse(r => new URL(r.url()).pathname === '/api/v1/agents/draft', {timeout: 120000})
|
||||
await generate.click()
|
||||
await dialog.locator('progress').waitFor()
|
||||
check('ai_draft_has_visible_progress', await dialog.locator('progress').isVisible())
|
||||
await page.screenshot({path: `${directory}/public-agent-ai-progress-mobile.png`, animations: 'disabled'})
|
||||
check('ai_dialog_mobile_has_no_horizontal_overflow', await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 2))
|
||||
const result = await draftResponse
|
||||
check(`configured_model_generates_real_draft_http_${result.status()}`, result.status() === 200)
|
||||
const draft = (await result.json()).data
|
||||
check('draft_has_no_persisted_resource_id', !!draft?.name && !draft.id)
|
||||
await page.locator('.settings-modal').waitFor()
|
||||
const name = page.locator('[data-guide="agent-create-name"] input')
|
||||
await name.waitFor()
|
||||
await page.waitForFunction(expected => document.querySelector('[data-guide="agent-create-name"] input')?.value === expected, draft.name)
|
||||
check('generated_draft_is_editable_before_saving', await name.inputValue() === draft.name && await name.isEditable())
|
||||
await page.screenshot({path: `${directory}/public-agent-ai-draft-mobile.png`, animations: 'disabled'})
|
||||
await page.setViewportSize({width: 1440, height: 1000})
|
||||
await page.screenshot({path: `${directory}/public-agent-ai-draft-desktop.png`, animations: 'disabled'})
|
||||
check('ai_draft_does_not_auto_create_agent', !writes.includes('/api/v1/agents'))
|
||||
check('agent_creation_has_no_javascript_errors', errors.length === 0)
|
||||
}
|
||||
|
||||
main().catch(error => {failures.push(error.message); process.exitCode = 1}).finally(async () => {
|
||||
if (browser) await browser.close()
|
||||
const report = {checked_at: new Date().toISOString(), origin, checks, failures, passed: failures.length === 0}
|
||||
fs.writeFileSync(`${directory}/public-agent-creation.json`, JSON.stringify(report, null, 2))
|
||||
console.log(JSON.stringify(report))
|
||||
})
|
||||
@@ -0,0 +1,120 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
|
||||
const origin = process.env.VERIFY_FRONTEND_ORIGIN || 'http://127.0.0.1:15173'
|
||||
const apiOrigin = process.env.VERIFY_API_ORIGIN || 'http://127.0.0.1:18080'
|
||||
const reportName = process.env.VERIFY_REPORT_NAME || 'agent-creation-ui'
|
||||
assert.match(reportName, /^[a-zA-Z0-9_-]+$/)
|
||||
const startedAt = new Date().toISOString()
|
||||
|
||||
// Draft AI output is a fixture; other reads use the running local API. No resources are saved.
|
||||
async function main() {
|
||||
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
|
||||
const credentials = JSON.parse(fs.readFileSync(`${directory}/super-admin-login.json`, 'utf8'))
|
||||
const response = await fetch(`${apiOrigin}/api/v1/auth/login`, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ email: credentials.email, password: credentials.password }),
|
||||
})
|
||||
assert.equal(response.status, 200)
|
||||
const auth = await response.json()
|
||||
const browser = await chromium.launch({ headless: true, channel: 'chrome' })
|
||||
const checks = [], errors = [], failures = [], requests = [], failedRequests = [], slowRequests = []
|
||||
const pendingRequests = new Map()
|
||||
let finalPath = ''
|
||||
const check = (name, passed) => { checks.push({ name, passed: !!passed }); assert.ok(passed, name) }
|
||||
try {
|
||||
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } })
|
||||
await context.addInitScript(a => {
|
||||
const entries = {
|
||||
weknora_user: JSON.stringify({ ...a.user, tenant_id: String(a.user.tenant_id) }),
|
||||
weknora_tenant: JSON.stringify({ ...a.active_tenant, id: '10059' }),
|
||||
weknora_token: a.token, weknora_refresh_token: a.refresh_token,
|
||||
weknora_memberships: JSON.stringify(a.memberships),
|
||||
'weknora:new-user-guide-done:v1': '1',
|
||||
}
|
||||
for (const [key, value] of Object.entries(entries)) localStorage.setItem(key, value)
|
||||
}, auth)
|
||||
let releaseDraft, draftRequest
|
||||
await context.route('**/api/v1/agents/draft', route => new Promise(resolve => {
|
||||
draftRequest = route.request().postDataJSON()
|
||||
releaseDraft = async () => {
|
||||
await route.fulfill({ status: 200, json: { success: true, data: {
|
||||
name: '课程框架规划助手', description: '按用户目标整理课程结构,确保前后衔接。', avatar: '',
|
||||
config: { role: 'training_map', agent_type: 'custom', agent_mode: 'quick-answer', model_id: draftRequest.model_id, system_prompt: '规划课程框架并说明每一天的前置条件。', kb_selection_mode: 'none', knowledge_bases: [], mcp_selection_mode: 'none', skills_selection_mode: 'none', selected_skills: [], instruction_skill_ids: [] },
|
||||
} } })
|
||||
resolve()
|
||||
}
|
||||
}))
|
||||
const page = await context.newPage()
|
||||
page.on('request', request => {
|
||||
requests.push({ path: new URL(request.url()).pathname, method: request.method() })
|
||||
pendingRequests.set(request, Date.now())
|
||||
})
|
||||
page.on('requestfinished', request => {
|
||||
const started = pendingRequests.get(request)
|
||||
pendingRequests.delete(request)
|
||||
const duration = started === undefined ? 0 : Date.now() - started
|
||||
if (duration >= 3000) slowRequests.push({ path: new URL(request.url()).pathname, duration_ms: duration })
|
||||
})
|
||||
page.on('requestfailed', request => {
|
||||
pendingRequests.delete(request)
|
||||
const failure = { path: new URL(request.url()).pathname, error: request.failure()?.errorText || '' }
|
||||
failedRequests.push(failure)
|
||||
if (failure.error !== 'net::ERR_ABORTED') errors.push(failure)
|
||||
})
|
||||
page.on('pageerror', error => errors.push({ stage: checks.at(-1)?.name, error: error.stack || error.message }))
|
||||
await page.goto(`${origin}/platform/agents`)
|
||||
const ai = page.getByRole('button', { name: 'AI 创建', exact: true })
|
||||
await ai.waitFor()
|
||||
check('manual_and_ai_creation_visible', await page.getByRole('button', { name: '手动创建', exact: true }).isVisible())
|
||||
await page.locator('.agent-installations').first().waitFor()
|
||||
check('installation_status_visible_on_agent_cards', await page.locator('.agent-installations').count() > 0)
|
||||
await page.waitForLoadState('networkidle', { timeout: 15000 })
|
||||
check('hidden_agent_editor_not_requested', !requests.some(item => /\/AgentEditorModal-[^/]+\.js$/.test(item.path)))
|
||||
check('hidden_draft_dialog_not_requested', !requests.some(item => /\/AgentDraftDialog-[^/]+\.js$/.test(item.path)))
|
||||
check('retired_resource_versions_menu_absent', await page.locator('.sidebar').getByText('资源版本', { exact: true }).count() === 0)
|
||||
await page.screenshot({ path: `${directory}/agent-creation-list-desktop.png`, animations: 'disabled' })
|
||||
await ai.click()
|
||||
const dialog = page.locator('.t-dialog:visible')
|
||||
await dialog.getByPlaceholder('例如:面向课程学员,负责将每日课程框架整理为有前后衔接的讲稿,重点解释概念和案例。').fill('请创建负责课程规划与每日框架衔接的导图助手。')
|
||||
await dialog.getByRole('button', { name: '生成配置', exact: true }).click()
|
||||
await dialog.locator('progress').waitFor()
|
||||
check('ai_generation_progress_visible', await dialog.locator('progress').isVisible())
|
||||
check('ai_double_submit_prevented', await dialog.getByRole('button', { name: /生成配置/ }).isDisabled())
|
||||
await page.screenshot({ path: `${directory}/agent-draft-progress-desktop.png`, animations: 'disabled' })
|
||||
await page.setViewportSize({ width: 390, height: 844 })
|
||||
check('draft_dialog_fits_mobile', await dialog.evaluate(element => { const rect = element.getBoundingClientRect(); return rect.left >= 0 && rect.right <= innerWidth + 2 }))
|
||||
await page.screenshot({ path: `${directory}/agent-draft-progress-mobile.png`, animations: 'disabled' })
|
||||
await releaseDraft()
|
||||
const name = page.locator('[data-guide="agent-create-name"] input')
|
||||
await name.waitFor()
|
||||
await page.waitForFunction(() => document.querySelector('[data-guide="agent-create-name"] input')?.value === '课程框架规划助手')
|
||||
check('ai_draft_loaded_into_editable_editor', await name.inputValue() === '课程框架规划助手')
|
||||
check('draft_request_preserves_requirement', draftRequest.requirement === '请创建负责课程规划与每日框架衔接的导图助手。')
|
||||
await page.screenshot({ path: `${directory}/agent-draft-editor-mobile.png`, animations: 'disabled' })
|
||||
// Restore normal browser caching after the one mocked AI request.
|
||||
await context.unroute('**/api/v1/agents/draft')
|
||||
await page.goto(`${origin}/platform/workbench/resource-versions`)
|
||||
await ai.waitFor()
|
||||
check('retired_route_redirects_to_agents', new URL(page.url()).pathname === '/platform/agents')
|
||||
await page.getByRole('button', { name: '手动创建', exact: true }).click()
|
||||
await name.waitFor()
|
||||
check('manual_creation_does_not_reuse_ai_draft', await name.inputValue() !== '课程框架规划助手')
|
||||
check('no_business_resources_saved', !requests.some(item => ['POST', 'PUT', 'PATCH', 'DELETE'].includes(item.method) && item.path.startsWith('/api/') && !item.path.startsWith('/api/v1/auth/') && item.path !== '/api/v1/agents/draft'))
|
||||
check('no_vue_runtime_errors', errors.length === 0)
|
||||
} catch (error) {
|
||||
failures.push(error.message)
|
||||
const pages = browser.contexts().flatMap(context => context.pages())
|
||||
await pages[0]?.screenshot({ path: `${directory}/agent-creation-ui-failure.png` })
|
||||
} finally {
|
||||
finalPath = browser.contexts().flatMap(context => context.pages()).map(page => new URL(page.url()).pathname).join(',')
|
||||
const diagnostics = { path: finalPath, slow_requests: slowRequests, failed_requests: failedRequests,
|
||||
pending_requests: [...pendingRequests].map(([request, started]) => ({ path: new URL(request.url()).pathname, duration_ms: Date.now() - started })) }
|
||||
await browser.close()
|
||||
const report = { started_at: startedAt, checked_at: new Date().toISOString(), origin, checks, errors, failures, diagnostics, ai_output_mocked: true, passed: failures.length === 0 && errors.length === 0 && checks.every(item => item.passed) }
|
||||
fs.writeFileSync(`${directory}/${reportName}.json`, JSON.stringify(report, null, 2))
|
||||
console.log(JSON.stringify(report))
|
||||
if (!report.passed) process.exitCode = 1
|
||||
}
|
||||
}
|
||||
main().catch(error => { console.error(error.message); process.exitCode = 1 })
|
||||
@@ -0,0 +1,97 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const crypto = require('node:crypto')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
|
||||
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
|
||||
const base = 'http://127.0.0.1:18080/api/v1'
|
||||
const prefix = `fusion-check-${Date.now()}`
|
||||
const checks = [], failures = [], users = [], roles = [], agents = []
|
||||
let admin, browser
|
||||
const literal = value => `'${String(value).replaceAll("'", "''")}'`
|
||||
const sql = statement => execFileSync('docker', ['--context', 'colima-yawan', 'exec', 'WeKnora-postgres', 'psql', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-v', 'ON_ERROR_STOP=1', '-Atc', statement], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim()
|
||||
const check = (name, value) => { checks.push({ name, passed: !!value }); assert.ok(value, name) }
|
||||
async function api(route, method = 'GET', body, identity = admin, expected = 200) {
|
||||
const response = await fetch(base + route, { method, headers: { 'Content-Type': 'application/json', ...(identity ? { Authorization: `Bearer ${identity.token}`, 'X-Tenant-ID': '10059' } : {}) }, body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(15000) })
|
||||
const result = await response.json()
|
||||
assert.ok((Array.isArray(expected) ? expected : [expected]).includes(response.status), `${method} ${route}: ${response.status} ${result.message || result.error?.message || ''}`)
|
||||
return result
|
||||
}
|
||||
async function main() {
|
||||
const credentials = JSON.parse(fs.readFileSync(`${directory}/super-admin-login.json`, 'utf8'))
|
||||
admin = await api('/auth/login', 'POST', { email: credentials.email, password: credentials.password }, null)
|
||||
const role = (await api('/access/roles', 'POST', { name: prefix, permissions: ['agents.read', 'agents.write'] })).data
|
||||
roles.push(role.id)
|
||||
async function member(label) {
|
||||
const password = crypto.randomBytes(18).toString('base64url'), email = `${prefix}-${label}@example.test`
|
||||
const user = (await api('/access/users', 'POST', { username: `${prefix}-${label}`, email, password, role: 'contributor', status: 'active', custom_role_id: role.id })).data
|
||||
users.push(user.id)
|
||||
return api('/auth/login', 'POST', { email, password }, null)
|
||||
}
|
||||
const alice = await member('alice'), bob = await member('bob')
|
||||
async function agent(label, identity, role = 'production') {
|
||||
const row = (await api('/agents', 'POST', { name: `${prefix}-${label}`, config: { role, agent_mode: 'quick-answer', model_id: '404e005b-83ae-5e2e-a3b3-999f9d5cd47e', kb_selection_mode: 'none', knowledge_bases: [], fusion_agent_ids: [], system_prompt: '保持原有身份与表达语气' } }, identity, 201)).data
|
||||
agents.push(row.id)
|
||||
return row
|
||||
}
|
||||
const primary = await agent('primary', alice), helper = await agent('private-helper', bob), second = await agent('second-helper', alice), qc = await agent('qc', alice, 'qc')
|
||||
check('new_agents_are_private_by_default', [primary, helper, second, qc].every(row => row.visibility === 'private'))
|
||||
const update = (ids, expected = 200) => api(`/agents/${primary.id}`, 'PUT', { name: primary.name, config: { ...primary.config, fusion_agent_ids: ids } }, alice, expected)
|
||||
const denied = await update([helper.id], 400)
|
||||
check('private_other_user_helper_is_rejected_without_name_leak', !JSON.stringify(denied).includes(helper.name))
|
||||
await update([primary.id], 400)
|
||||
await update([qc.id], 400)
|
||||
await update([second.id, second.id], 400)
|
||||
await update(Array.from({ length: 9 }, () => crypto.randomUUID()), 400)
|
||||
check('self_wrong_role_duplicate_and_over_limit_rejected', true)
|
||||
async function grant(action) { await api('/access/grants', 'POST', { resources: [{ kind: 'agents', id: helper.id }], user_ids: [alice.user.id], action }) }
|
||||
await grant('grant')
|
||||
await update([helper.id, second.id])
|
||||
check('direct_grant_can_configure_multiple_helpers', JSON.stringify((await api(`/agents/${primary.id}`, 'GET', undefined, alice)).data.config.fusion_agent_ids) === JSON.stringify([helper.id, second.id]))
|
||||
await grant('revoke')
|
||||
await update([helper.id], 400)
|
||||
await api(`/agents/${primary.id}/copy`, 'POST', undefined, alice, 400)
|
||||
check('revoked_access_blocks_update_and_copy', true)
|
||||
await grant('grant')
|
||||
browser = await chromium.launch({ headless: true, channel: 'chrome' })
|
||||
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } })
|
||||
await context.addInitScript(auth => {
|
||||
for (const [key, value] of Object.entries({ weknora_user: JSON.stringify({ ...auth.user, tenant_id: String(auth.user.tenant_id) }), weknora_tenant: JSON.stringify({ ...auth.active_tenant, id: '10059' }), weknora_token: auth.token, weknora_refresh_token: auth.refresh_token, weknora_memberships: JSON.stringify(auth.memberships), 'weknora:new-user-guide-done:v1': '1' })) localStorage.setItem(key, value)
|
||||
}, alice)
|
||||
const page = await context.newPage(), errors = []
|
||||
page.on('pageerror', error => errors.push(error.message))
|
||||
await page.goto(`http://127.0.0.1:15173/platform/agents?edit=${primary.id}`)
|
||||
const row = page.locator('.setting-row').filter({ has: page.getByText('融合表达 Agent', { exact: true }) })
|
||||
await row.waitFor()
|
||||
await row.scrollIntoViewIfNeeded()
|
||||
check('editor_displays_saved_multi_selection', await row.getByText(helper.name, { exact: true }).count() > 0 && await row.getByText(second.name, { exact: true }).count() > 0)
|
||||
await page.screenshot({ path: `${directory}/agent-fusion-desktop.png`, animations: 'disabled' })
|
||||
await page.setViewportSize({ width: 390, height: 844 })
|
||||
await row.scrollIntoViewIfNeeded()
|
||||
await page.screenshot({ path: `${directory}/agent-fusion-mobile.png`, animations: 'disabled' })
|
||||
const mobileLayout = await page.evaluate(() => ({ width: innerWidth, documentWidth: document.documentElement.scrollWidth,
|
||||
overflow: [...document.querySelectorAll('body *')].filter(element => { const rect = element.getBoundingClientRect(); return rect.width > 0 && rect.right > innerWidth + 2 }).slice(0, 15).map(element => ({ tag: element.tagName, className: element.className, right: element.getBoundingClientRect().right })) }))
|
||||
if (mobileLayout.documentWidth > mobileLayout.width + 2) console.log(JSON.stringify(mobileLayout))
|
||||
check('mobile_no_document_overflow', mobileLayout.documentWidth <= mobileLayout.width + 2)
|
||||
check('browser_no_runtime_errors', errors.length === 0)
|
||||
await api(`/agents/${helper.id}?detach=true`, 'DELETE', undefined, bob)
|
||||
const after = (await api(`/agents/${primary.id}`, 'GET', undefined, alice)).data
|
||||
check('deletion_unbinds_reference_and_preserves_other_helper', JSON.stringify(after.config.fusion_agent_ids) === JSON.stringify([second.id]))
|
||||
}
|
||||
main().catch(error => { failures.push(error.message); process.exitCode = 1 }).finally(async () => {
|
||||
await browser?.close()
|
||||
if (admin) {
|
||||
for (const id of agents) try { await api(`/agents/${id}?detach=true`, 'DELETE', undefined, admin, [200, 404]) } catch (error) { failures.push(`cleanup agent: ${error.message}`) }
|
||||
if (agents.length) {
|
||||
const ids = agents.map(literal).join(',')
|
||||
sql(`DELETE FROM workbench_access_grants WHERE tenant_id=10059 AND kind='agents' AND resource_id IN (${ids}); DELETE FROM workbench_access_publications WHERE tenant_id=10059 AND kind='agents' AND resource_id IN (${ids}); DELETE FROM custom_agents WHERE tenant_id=10059 AND id IN (${ids})`)
|
||||
}
|
||||
for (const id of users) try { await api(`/access/users/${id}`, 'DELETE') } catch (error) { failures.push(`cleanup user: ${error.message}`) }
|
||||
for (const id of roles) try { await api(`/access/roles/${id}`, 'DELETE') } catch (error) { failures.push(`cleanup role: ${error.message}`) }
|
||||
check('no_live_fixture_agents_remain', sql(`SELECT count(*) FROM custom_agents WHERE tenant_id=10059 AND name LIKE ${literal(prefix + '%')}`) === '0')
|
||||
}
|
||||
const report = { time: new Date().toISOString(), prefix, checks, failures, passed: failures.length === 0 }
|
||||
fs.writeFileSync(`${directory}/agent-fusion-live.json`, JSON.stringify(report, null, 2))
|
||||
console.log(JSON.stringify(report, null, 2))
|
||||
if (failures.length) process.exitCode = 1
|
||||
})
|
||||
@@ -0,0 +1,52 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const crypto = require('node:crypto')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
|
||||
const schema = `fusion_migration_test_${crypto.randomBytes(6).toString('hex')}`
|
||||
const migration = fs.readFileSync(`${__dirname}/../migrations/versioned/000208_agent_fusions.up.sql`, 'utf8')
|
||||
const checks = []
|
||||
const sql = query => execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres',
|
||||
'psql', '-X', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-At', '-v', 'ON_ERROR_STOP=1'],
|
||||
{ input: `SET search_path TO ${schema}, public;\n${query}`, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim()
|
||||
const check = (name, condition) => { assert.ok(condition, name); checks.push(name) }
|
||||
const value = text => `'${JSON.stringify(text).replaceAll("'", "''")}'::jsonb`
|
||||
function fixture(id, fusion, role = 'production', native = { role: 'production', sentinel: 'preserved' }) {
|
||||
sql(`INSERT INTO custom_agents VALUES ('agent-${id}',10059,${value(native)},NULL);
|
||||
INSERT INTO yawan_legacy_asset_mappings VALUES ('yawan-workbench',1,'agent',${id},10059,'agent-${id}',${value({ role, fusion_agent_ids: fusion })},'{}');`)
|
||||
}
|
||||
const run = () => sql(`BEGIN; ${migration} COMMIT;`)
|
||||
try {
|
||||
sql(`CREATE SCHEMA ${schema};
|
||||
CREATE TABLE custom_agents (id TEXT PRIMARY KEY,tenant_id BIGINT,config JSONB,deleted_at TIMESTAMPTZ);
|
||||
CREATE TABLE yawan_legacy_asset_mappings (source_system TEXT,source_organization_id BIGINT,source_kind TEXT,source_id BIGINT,target_tenant_id BIGINT,target_id TEXT,source_snapshot JSONB,native_snapshot JSONB);`)
|
||||
fixture(1, [3, 2]); fixture(2, [1]); fixture(3, [])
|
||||
fixture(4, [2], 'production', { role: 'production', fusion_agent_ids: [], sentinel: 'explicit-empty' })
|
||||
fixture(5, [2], 'production', { role: 'qc', sentinel: 'native-duty-edited' })
|
||||
run()
|
||||
const data = JSON.parse(sql('SELECT jsonb_object_agg(id,config) FROM custom_agents').split('\n').at(-1))
|
||||
check('ordered_direct_ids_preserved_without_recursive_cycle_expansion', JSON.stringify(data['agent-1'].fusion_agent_ids) === JSON.stringify(['agent-3', 'agent-2']) && JSON.stringify(data['agent-2'].fusion_agent_ids) === JSON.stringify(['agent-1']))
|
||||
check('empty_selection_and_native_duty_edits_preserved', data['agent-4'].fusion_agent_ids.length === 0 && !('fusion_agent_ids' in data['agent-5']))
|
||||
check('unrelated_configuration_preserved', data['agent-1'].sentinel === 'preserved')
|
||||
sql(`UPDATE custom_agents SET config=config || '{"fusion_agent_ids":[],"sentinel":"owner-edit"}' WHERE id='agent-1';`)
|
||||
const before = sql('SELECT jsonb_agg(to_jsonb(a) ORDER BY id) FROM custom_agents a')
|
||||
run()
|
||||
check('rerun_does_not_restore_user_removed_bindings', before === sql('SELECT jsonb_agg(to_jsonb(a) ORDER BY id) FROM custom_agents a'))
|
||||
const cases = [[6, [999], 'missing-target'], [7, [2, 2], 'duplicate'], [8, [8], 'self'], [9, ['2'], 'invalid-id'], [10, [2], 'primary-wrong-role']]
|
||||
for (const [id, fusions, label] of cases) {
|
||||
fixture(id, fusions, label === 'primary-wrong-role' ? 'qc' : 'production')
|
||||
const baseline = sql('SELECT jsonb_agg(to_jsonb(a) ORDER BY id) FROM custom_agents a')
|
||||
let rejected = false
|
||||
try { run() } catch { rejected = true }
|
||||
check(`${label}_rejected_atomically`, rejected && baseline === sql('SELECT jsonb_agg(to_jsonb(a) ORDER BY id) FROM custom_agents a'))
|
||||
sql(`DELETE FROM custom_agents WHERE id='agent-${id}'; DELETE FROM yawan_legacy_asset_mappings WHERE source_id=${id};`)
|
||||
}
|
||||
fixture(11, [2])
|
||||
sql(`UPDATE yawan_legacy_asset_mappings SET source_snapshot=jsonb_set(source_snapshot,'{role}','"qc"') WHERE source_id=2;`)
|
||||
let rejected = false
|
||||
try { run() } catch { rejected = true }
|
||||
check('nonproduction_target_rejected', rejected)
|
||||
console.log(JSON.stringify({ passed: true, checks }, null, 2))
|
||||
} finally {
|
||||
sql(`DROP SCHEMA IF EXISTS ${schema} CASCADE;`)
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify migrated assets through acceptance HTTP without printing bodies."""
|
||||
import argparse
|
||||
import json
|
||||
import secrets
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from apply_weknora_assets import target_snapshot
|
||||
from apply_weknora_identities import sql_run
|
||||
from weknora_asset_etl import LEDGER, asset_id
|
||||
from weknora_category_etl import literal, save_json
|
||||
from weknora_identity_etl import TARGET_DATABASE, TARGET_TENANT
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--credentials', type=Path, required=True)
|
||||
parser.add_argument('--out', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.out.exists():
|
||||
parser.error('Report exists')
|
||||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
checks, fixture_user, fixture_agent = [], None, None
|
||||
completed = False
|
||||
|
||||
def request(path, token=None, data=None, method=None, tenant=TARGET_TENANT):
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
if token:
|
||||
headers.update(Authorization='Bearer ' + token)
|
||||
if tenant:
|
||||
headers['X-Tenant-ID'] = str(tenant)
|
||||
req = urllib.request.Request('http://127.0.0.1:18080' + path, headers=headers,
|
||||
data=json.dumps(data).encode() if data is not None else None,
|
||||
method=method)
|
||||
try:
|
||||
with opener.open(req, timeout=30) as response:
|
||||
return response.status, json.load(response)
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, {}
|
||||
|
||||
def check(name, passed):
|
||||
checks.append({'check': name, 'passed': bool(passed)})
|
||||
|
||||
before = target_snapshot()
|
||||
if before['database'] != TARGET_DATABASE:
|
||||
raise RuntimeError('Acceptance database mismatch')
|
||||
credentials = json.loads(args.credentials.read_text())
|
||||
status, auth = request('/api/v1/auth/login', data={k: credentials[k] for k in ('email', 'password')})
|
||||
if status != 200 or not auth.get('token'):
|
||||
raise RuntimeError('Acceptance administrator login failed')
|
||||
token = auth['token']
|
||||
expected = json.loads(sql_run(f"SELECT json_agg(json_build_object('kind',source_kind,'source_id',source_id,'id',target_id,'native',native_snapshot)) FROM {LEDGER} WHERE target_tenant_id={TARGET_TENANT} AND source_kind IN ('agent','skill');"))
|
||||
agents = [a for a in expected if a['kind'] == 'agent']
|
||||
skills = [a for a in expected if a['kind'] == 'skill']
|
||||
try:
|
||||
status, response = request('/api/v1/agents', token)
|
||||
listed = {a['id']: a for a in response.get('data', [])}
|
||||
check('administrator_sees_all_15_agents', status == 200 and len(agents) == 15 and all(a['id'] in listed for a in agents))
|
||||
for item in agents:
|
||||
status, response = request('/api/v1/agents/' + item['id'], token)
|
||||
native, current = item['native'], response.get('data', {})
|
||||
check('agent_' + str(item['source_id']) + '_body_model_bindings_and_owner', status == 200 and
|
||||
all(current.get(k) == native[k] for k in ('name', 'created_by', 'visibility', 'enabled')) and
|
||||
all(current.get('config', {}).get(k, []) == native['config'][k]
|
||||
for k in ('system_prompt', 'model_id', 'knowledge_bases', 'instruction_skill_ids')))
|
||||
status, response = request('/api/v1/skills/instructions', token)
|
||||
listed = {a['id']: a for a in response.get('data', [])}
|
||||
check('administrator_sees_all_9_skills', status == 200 and len(skills) == 9 and all(s['id'] in listed for s in skills))
|
||||
for item in skills:
|
||||
status, response = request('/api/v1/skills/instructions/' + item['id'], token)
|
||||
current, native = response.get('data', {}), item['native']
|
||||
check('skill_' + str(item['source_id']) + '_body_and_owner', status == 200 and
|
||||
all(current.get(k) == native[k] for k in ('name', 'instructions', 'created_by', 'visibility', 'enabled')))
|
||||
for endpoint in ('/api/v1/agents', '/api/v1/skills/instructions'):
|
||||
status, _ = request(endpoint)
|
||||
check('anonymous_' + endpoint.rsplit('/', 1)[-1] + '_denied', status in (401, 403))
|
||||
|
||||
suffix = secrets.token_hex(6)
|
||||
password = secrets.token_urlsafe(18) + 'aA1!'
|
||||
email = 'asset-check-' + suffix + '@migration.invalid'
|
||||
status, response = request('/api/v1/system/admin/users/create', token,
|
||||
{'username': 'asset-check-' + suffix, 'email': email, 'password': password})
|
||||
fixture_user = response.get('user', {}).get('id')
|
||||
if status != 201 or not fixture_user or str(uuid.UUID(fixture_user)) != fixture_user:
|
||||
raise RuntimeError(f'Temporary acceptance user provisioning failed (HTTP {status})')
|
||||
sql_run(f"INSERT INTO tenant_members(user_id,tenant_id,role,status) VALUES ({literal(fixture_user)},{TARGET_TENANT},'contributor','active');")
|
||||
status, member_auth = request('/api/v1/auth/login', data={'email': email, 'password': password})
|
||||
if status != 200 or not member_auth.get('token'):
|
||||
raise RuntimeError('Temporary acceptance user login failed')
|
||||
member_token = member_auth['token']
|
||||
status, response = request('/api/v1/agents', member_token)
|
||||
visible = {a['id'] for a in response.get('data', [])}
|
||||
check('member_sees_13_global_agents_and_no_private_agents', status == 200 and
|
||||
all((a['id'] in visible) == (a['native']['visibility'] == 'global' and a['native']['enabled']) for a in agents))
|
||||
status, response = request('/api/v1/skills/instructions', member_token)
|
||||
visible = {a['id'] for a in response.get('data', [])}
|
||||
check('member_sees_7_global_skills_and_no_private_skills', status == 200 and
|
||||
all((s['id'] in visible) == (s['native']['visibility'] == 'global') for s in skills))
|
||||
for kind, number, endpoint in (('agent', 22, '/api/v1/agents/'), ('skill', 5, '/api/v1/skills/instructions/'), ('skill', 9, '/api/v1/skills/instructions/')):
|
||||
status, _ = request(endpoint + asset_id(kind, number), member_token)
|
||||
check('private_' + kind + '_' + str(number) + '_direct_read_denied', status in (403, 404))
|
||||
# A valid no-op payload avoids modifying the original even if a guard regresses.
|
||||
first = agents[0]
|
||||
status, _ = request('/api/v1/agents/' + first['id'], member_token,
|
||||
{'name': first['native']['name'], 'config': first['native']['config']}, 'PUT')
|
||||
check('member_cannot_update_global_original_agent', status in (403, 404))
|
||||
status, _ = request('/api/v1/skills/instructions/mount', member_token,
|
||||
{'agent_ids': [first['id']], 'skill_ids': [asset_id('skill', 1)], 'mode': 'add'})
|
||||
check('member_cannot_mount_on_another_owners_agent', status in (403, 404))
|
||||
status, response = request('/api/v1/agents', member_token, {
|
||||
'name': 'asset-check-' + suffix, 'config': {'agent_mode': 'quick-answer', 'kb_selection_mode': 'none',
|
||||
'skills_selection_mode': 'none', 'model_id': first['native']['config']['model_id']}})
|
||||
fixture_agent = response.get('data', {}).get('id')
|
||||
if status != 201 or not fixture_agent or str(uuid.UUID(fixture_agent)) != fixture_agent:
|
||||
raise RuntimeError('Temporary Agent creation failed')
|
||||
status, response = request('/api/v1/skills/instructions/mount', member_token,
|
||||
{'agent_ids': [fixture_agent], 'skill_ids': [asset_id('skill', 1)], 'mode': 'add'})
|
||||
check('member_can_mount_global_skill_on_own_agent', status == 200)
|
||||
status, response = request('/api/v1/agents/' + fixture_agent, member_token)
|
||||
check('mounted_global_skill_persisted', status == 200 and asset_id('skill', 1) in response.get('data', {}).get('config', {}).get('instruction_skill_ids', []))
|
||||
status, _ = request('/api/v1/skills/instructions/mount', member_token,
|
||||
{'agent_ids': [fixture_agent], 'skill_ids': [asset_id('skill', 5)], 'mode': 'add'})
|
||||
check('member_cannot_mount_private_skill', status in (403, 404))
|
||||
status, _ = request('/api/v1/agents/' + first['id'], member_token, tenant=10060)
|
||||
check('unrelated_tenant_header_denied', status in (403, 404))
|
||||
completed = True
|
||||
finally:
|
||||
if fixture_user:
|
||||
# Scope cleanup by the exact user returned by this run, never by name prefixes.
|
||||
sql_run(f"BEGIN; DELETE FROM custom_agents WHERE created_by={literal(fixture_user)}; DELETE FROM tenant_members WHERE user_id={literal(fixture_user)} AND tenant_id={TARGET_TENANT}; DELETE FROM auth_tokens WHERE user_id={literal(fixture_user)}; UPDATE users SET is_active=false,updated_at=now() WHERE id={literal(fixture_user)}; COMMIT;")
|
||||
check('existing_assets_models_and_knowledge_unchanged', before == target_snapshot())
|
||||
check('all_api_checks_completed', completed)
|
||||
result = {'passed': all(c['passed'] for c in checks), 'checks': checks,
|
||||
'fixture_user_deactivated': bool(fixture_user), 'fixture_agent_removed': bool(fixture_agent)}
|
||||
save_json(args.out, result)
|
||||
print(json.dumps(result))
|
||||
return 0 if result['passed'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run asset migration regressions in acceptance PostgreSQL and roll them back.
|
||||
|
||||
Source rules, skill bodies, and generated SQL stay in memory. Reports contain
|
||||
only check names, booleans, counts, and the fixed acceptance scope.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from apply_weknora_assets import build_plan, target_snapshot
|
||||
from apply_weknora_identities import sql_run
|
||||
from verify_identity_etl import rejected_case
|
||||
from weknora_asset_etl import (BINDING_LEDGER, LEDGER, SOURCE, TARGET_DATABASE,
|
||||
TARGET_TENANT, asset_sql, safe_asset_summary)
|
||||
from weknora_category_etl import literal, save_json
|
||||
from weknora_etl import consistent_snapshot
|
||||
from weknora_identity_etl import IDENTITY_LEDGER
|
||||
|
||||
|
||||
def supporting_snapshot():
|
||||
tables = (LEDGER, BINDING_LEDGER, IDENTITY_LEDGER, "users", "tenant_members",
|
||||
"tenants", "knowledge_bases")
|
||||
fields = []
|
||||
for table in tables:
|
||||
fields.extend((literal(table),
|
||||
f"(SELECT coalesce(json_agg(fingerprint ORDER BY fingerprint),'[]') "
|
||||
f"FROM (SELECT md5(to_jsonb(r)::text) AS fingerprint FROM {table} r) s)"))
|
||||
return json.loads(sql_run("SELECT json_build_object(" + ",".join(fields) + ");"))
|
||||
|
||||
|
||||
def unchanged_check(name, pairs):
|
||||
conditions = []
|
||||
for table, baseline in pairs:
|
||||
conditions.extend((f"EXISTS (SELECT * FROM {table} EXCEPT SELECT * FROM {baseline})",
|
||||
f"EXISTS (SELECT * FROM {baseline} EXCEPT SELECT * FROM {table})"))
|
||||
return f"""
|
||||
DO $asset_unchanged$
|
||||
BEGIN
|
||||
IF {' OR '.join(conditions)} THEN
|
||||
RAISE EXCEPTION 'Asset regression changed rows unexpectedly';
|
||||
END IF;
|
||||
END $asset_unchanged$;
|
||||
SELECT json_build_object('check',{literal(name)},'passed',true);
|
||||
"""
|
||||
|
||||
|
||||
def regression_sql(plan, query):
|
||||
agent, skill = plan["agents"][0]["target"], plan["skills"][0]["target"]
|
||||
aid, sid = literal(agent["id"]), literal(skill["id"])
|
||||
other_owner = literal(next(uid for uid in plan["identity_mapping"].values()
|
||||
if uid != agent["created_by"]))
|
||||
other_skill_owner = literal(next(uid for uid in plan["identity_mapping"].values()
|
||||
if uid != skill["created_by"]))
|
||||
asset_scope = f"source_system={literal(SOURCE)} AND source_organization_id=1"
|
||||
agent_ledger = f"{asset_scope} AND source_kind='agent' AND target_id={aid}"
|
||||
skill_ledger = f"{asset_scope} AND source_kind='skill' AND target_id={sid}"
|
||||
binding = plan["bindings"][0]
|
||||
binding_where = f"{asset_scope} AND source_binding_id={binding['source']['id']}"
|
||||
kb = literal(next(iter(plan["kb_mapping"].values())))
|
||||
model = literal(agent["config"]["model_id"])
|
||||
owner = literal(agent["created_by"])
|
||||
foreign_tenant = TARGET_TENANT + 1
|
||||
cases = [
|
||||
("mapped_agent_wrong_tenant_rejected", f"UPDATE custom_agents SET tenant_id={foreign_tenant} WHERE id={aid};",
|
||||
"Mapped native asset removed; refusing recreation"),
|
||||
("mapped_skill_wrong_tenant_rejected", f"UPDATE tenant_skill_catalog SET tenant_id={foreign_tenant} WHERE id={sid};",
|
||||
"Mapped native asset removed; refusing recreation"),
|
||||
("mapped_agent_wrong_owner_rejected", f"UPDATE custom_agents SET created_by={other_owner} WHERE id={aid};",
|
||||
"Mapped native asset removed; refusing recreation"),
|
||||
("mapped_skill_wrong_owner_rejected", f"UPDATE tenant_skill_catalog SET created_by={other_skill_owner} WHERE id={sid};",
|
||||
"Mapped native asset removed; refusing recreation"),
|
||||
("mapped_skill_kind_change_rejected", f"UPDATE tenant_skill_catalog SET instruction_only=false WHERE id={sid};",
|
||||
"Mapped native asset removed; refusing recreation"),
|
||||
("asset_ledger_wrong_tenant_rejected", f"UPDATE {LEDGER} SET target_tenant_id={foreign_tenant} WHERE {agent_ledger};",
|
||||
"Asset durable mapping conflict"),
|
||||
("asset_ledger_wrong_id_rejected", f"UPDATE {LEDGER} SET target_id='rollback-only-id' WHERE {agent_ledger};",
|
||||
"Asset durable mapping conflict"),
|
||||
("asset_ledger_source_change_rejected", f"UPDATE {LEDGER} SET source_snapshot='{{}}'::jsonb WHERE {skill_ledger};",
|
||||
"Asset durable mapping conflict"),
|
||||
("asset_ledger_native_plan_change_rejected", f"UPDATE {LEDGER} SET native_snapshot='{{}}'::jsonb WHERE {agent_ledger};",
|
||||
"Asset durable mapping conflict"),
|
||||
("asset_ledger_changed_duty_rejected", f"UPDATE {LEDGER} SET native_snapshot=jsonb_set(native_snapshot,'{{config,role}}','\"invalid-role\"'::jsonb) WHERE {agent_ledger};",
|
||||
"Asset durable mapping conflict"),
|
||||
("hard_deleted_agent_not_recreated", f"DELETE FROM custom_agents WHERE id={aid};",
|
||||
"Mapped native asset removed; refusing recreation"),
|
||||
("hard_deleted_skill_not_recreated", f"DELETE FROM tenant_skill_catalog WHERE id={sid};",
|
||||
"Mapped native asset removed; refusing recreation"),
|
||||
("agent_id_without_ledger_rejected", f"DELETE FROM {LEDGER} WHERE {agent_ledger};",
|
||||
"Native Agent ID or owner/name collision"),
|
||||
("skill_id_without_ledger_rejected", f"DELETE FROM {LEDGER} WHERE {skill_ledger};",
|
||||
"Native skill ID or owner/name collision"),
|
||||
("agent_owner_name_without_ledger_rejected", f"DELETE FROM {LEDGER} WHERE {agent_ledger}; "
|
||||
f"UPDATE custom_agents SET id='rollback-only-agent-id' WHERE id={aid};",
|
||||
"Native Agent ID or owner/name collision"),
|
||||
("skill_owner_name_without_ledger_rejected", f"DELETE FROM {LEDGER} WHERE {skill_ledger}; "
|
||||
f"UPDATE tenant_skill_catalog SET id='rollback-only-skill-id' WHERE id={sid};",
|
||||
"Native skill ID or owner/name collision"),
|
||||
("inactive_target_tenant_rejected", f"UPDATE tenants SET status='suspended' WHERE id={TARGET_TENANT};",
|
||||
"Asset migration requires active target tenant"),
|
||||
("owner_identity_mapping_change_rejected", f"UPDATE {IDENTITY_LEDGER} SET target_user_id='rollback-only-owner' "
|
||||
f"WHERE {asset_scope} AND target_user_id={owner};",
|
||||
"Asset owner durable identity or membership mismatch"),
|
||||
("owner_membership_deletion_rejected", f"UPDATE tenant_members SET deleted_at=now() WHERE tenant_id={TARGET_TENANT} AND user_id={owner};",
|
||||
"Asset owner durable identity or membership mismatch"),
|
||||
("knowledge_base_wrong_tenant_rejected", f"UPDATE knowledge_bases SET tenant_id={foreign_tenant} WHERE id={kb};",
|
||||
"Asset knowledge base mapping missing or outside tenant"),
|
||||
("deleted_model_rejected", f"UPDATE models SET deleted_at=now() WHERE id={model};",
|
||||
"Asset model mapping missing or not usable"),
|
||||
("binding_wrong_tenant_rejected", f"UPDATE {BINDING_LEDGER} SET target_tenant_id={foreign_tenant} WHERE {binding_where};",
|
||||
"Legacy skill binding provenance conflict"),
|
||||
("binding_agent_change_rejected", f"UPDATE {BINDING_LEDGER} SET target_agent_id='rollback-only-agent' WHERE {binding_where};",
|
||||
"Legacy skill binding provenance conflict"),
|
||||
("binding_source_change_rejected", f"UPDATE {BINDING_LEDGER} SET source_snapshot='{{}}'::jsonb WHERE {binding_where};",
|
||||
"Legacy skill binding provenance conflict"),
|
||||
("binding_access_review_change_rejected", f"UPDATE {BINDING_LEDGER} SET access_review='[\"rollback-only-review\"]'::jsonb WHERE {binding_where};",
|
||||
"Legacy skill binding provenance conflict"),
|
||||
("binding_effective_flag_change_rejected", f"UPDATE {BINDING_LEDGER} SET effective=NOT effective WHERE {binding_where};",
|
||||
"Legacy skill binding provenance conflict"),
|
||||
]
|
||||
if plan["prompt_templates"]:
|
||||
template_id = plan["prompt_templates"][0]["id"]
|
||||
cases.append(("template_source_change_rejected",
|
||||
f"UPDATE {LEDGER} SET source_snapshot='{{}}'::jsonb WHERE {asset_scope} "
|
||||
f"AND source_kind='template' AND source_id={template_id};",
|
||||
"Legacy prompt template provenance conflict"))
|
||||
tables = ("custom_agents", "tenant_skill_catalog", LEDGER, BINDING_LEDGER, "models")
|
||||
pairs = [(table, f"_asset_before_{index}") for index, table in enumerate(tables)]
|
||||
sql = """BEGIN ISOLATION LEVEL SERIALIZABLE;
|
||||
SET LOCAL statement_timeout='30s';
|
||||
SET LOCAL lock_timeout='5s';
|
||||
SET LOCAL log_statement='none';
|
||||
SET LOCAL log_min_error_statement='panic';
|
||||
"""
|
||||
for table, baseline in pairs:
|
||||
sql += f"CREATE TEMP TABLE {baseline} ON COMMIT DROP AS SELECT * FROM {table};\n"
|
||||
sql += query + unchanged_check("exact_repeat_and_existing_models_unchanged", pairs)
|
||||
sql += f"""
|
||||
SELECT json_build_object('check','all_39_source_bindings_retained_only_30_effective','passed',
|
||||
(SELECT count(*)=39 AND count(*) FILTER (WHERE effective)=30 FROM {BINDING_LEDGER}
|
||||
WHERE {asset_scope} AND target_tenant_id={TARGET_TENANT})
|
||||
AND (SELECT count(*)=15 FROM {LEDGER} WHERE {asset_scope} AND source_kind='agent')
|
||||
AND (SELECT count(*)=9 FROM {LEDGER} WHERE {asset_scope} AND source_kind='skill')
|
||||
AND NOT EXISTS (
|
||||
(SELECT b.target_agent_id,b.target_skill_id FROM {BINDING_LEDGER} b
|
||||
WHERE b.source_system={literal(SOURCE)} AND b.source_organization_id=1 AND b.effective
|
||||
EXCEPT
|
||||
SELECT a.id,s.value FROM custom_agents a JOIN {LEDGER} l
|
||||
ON l.source_kind='agent' AND l.target_id=a.id AND l.target_tenant_id=a.tenant_id
|
||||
CROSS JOIN LATERAL jsonb_array_elements_text(a.config->'instruction_skill_ids') s(value)
|
||||
WHERE l.source_system={literal(SOURCE)} AND l.source_organization_id=1)
|
||||
UNION ALL
|
||||
(SELECT a.id,s.value FROM custom_agents a JOIN {LEDGER} l
|
||||
ON l.source_kind='agent' AND l.target_id=a.id AND l.target_tenant_id=a.tenant_id
|
||||
CROSS JOIN LATERAL jsonb_array_elements_text(a.config->'instruction_skill_ids') s(value)
|
||||
WHERE l.source_system={literal(SOURCE)} AND l.source_organization_id=1
|
||||
EXCEPT
|
||||
SELECT b.target_agent_id,b.target_skill_id FROM {BINDING_LEDGER} b
|
||||
WHERE b.source_system={literal(SOURCE)} AND b.source_organization_id=1 AND b.effective)));
|
||||
SELECT json_build_object('check','cross_owner_private_skills_blocked','passed',
|
||||
(SELECT count(*)=9 FROM {BINDING_LEDGER} WHERE {asset_scope} AND NOT effective)
|
||||
AND NOT EXISTS (SELECT 1 FROM {BINDING_LEDGER} b
|
||||
JOIN custom_agents a ON a.id=b.target_agent_id AND a.tenant_id=b.target_tenant_id
|
||||
JOIN tenant_skill_catalog s ON s.id=b.target_skill_id AND s.tenant_id=b.target_tenant_id
|
||||
WHERE b.source_system={literal(SOURCE)} AND b.source_organization_id=1
|
||||
AND b.effective IS DISTINCT FROM (s.visibility='global' OR s.created_by=a.created_by)));
|
||||
"""
|
||||
for name, mutation, expected in cases:
|
||||
sql += rejected_case(name, mutation, expected, query)
|
||||
sql += f"""
|
||||
SAVEPOINT pre_duty_replay;
|
||||
UPDATE {LEDGER} SET native_snapshot=native_snapshot #- '{{config,role}}' #- '{{config,content_tags}}' #- '{{config,fusion_agent_ids}}'
|
||||
WHERE {asset_scope} AND source_kind='agent';
|
||||
CREATE TEMP TABLE _pre_duty_ledger ON COMMIT DROP AS SELECT * FROM {LEDGER};
|
||||
"""
|
||||
sql += query + unchanged_check("pre_duty_ledger_replay_preserves_native_edits_and_audit",
|
||||
[("custom_agents", pairs[0][1]), (LEDGER, "_pre_duty_ledger")])
|
||||
sql += "ROLLBACK TO SAVEPOINT pre_duty_replay;\n"
|
||||
duty_migration = (Path(__file__).resolve().parents[1] / "migrations/versioned/000205_agent_duties.up.sql").read_text()
|
||||
sql += f"""
|
||||
SAVEPOINT duty_backfill;
|
||||
UPDATE custom_agents SET config=config - 'role' - 'content_tags' WHERE id={aid};
|
||||
{duty_migration}
|
||||
SELECT json_build_object('check','missing_duties_backfilled_without_other_changes','passed',
|
||||
(SELECT a.config->>'role'=l.source_snapshot->>'role'
|
||||
AND a.config->'content_tags'=l.source_snapshot->'content_tags'
|
||||
AND (to_jsonb(a) #- '{{config,role}}' #- '{{config,content_tags}}') =
|
||||
(to_jsonb(b) #- '{{config,role}}' #- '{{config,content_tags}}')
|
||||
FROM custom_agents a JOIN {LEDGER} l ON l.target_id=a.id AND l.source_kind='agent'
|
||||
JOIN {pairs[0][1]} b ON b.id=a.id WHERE a.id={aid}));
|
||||
UPDATE custom_agents SET config=config || '{{"role":"general","content_tags":[]}}'::jsonb WHERE id={aid};
|
||||
CREATE TEMP TABLE _duty_edited_agents ON COMMIT DROP AS SELECT * FROM custom_agents;
|
||||
{duty_migration}
|
||||
"""
|
||||
sql += unchanged_check("duty_backfill_preserves_explicit_native_edits", [("custom_agents", "_duty_edited_agents")])
|
||||
sql += "ROLLBACK TO SAVEPOINT duty_backfill;\n"
|
||||
sql += f"""
|
||||
UPDATE custom_agents SET name='Rollback-only native Agent edit',description='local edit',
|
||||
enabled=false,visibility='private',
|
||||
config=jsonb_set(jsonb_set(config,'{{system_prompt}}','\"Rollback-only native prompt\"'::jsonb),
|
||||
'{{instruction_skill_ids}}','[]'::jsonb) || '{{"role":"general","content_tags":[]}}'::jsonb WHERE id={aid} AND tenant_id={TARGET_TENANT};
|
||||
UPDATE tenant_skill_catalog SET name='Rollback-only native skill edit',description='local edit',
|
||||
instructions='Rollback-only native instructions',version='77',enabled=false,visibility='private'
|
||||
WHERE id={sid} AND tenant_id={TARGET_TENANT};
|
||||
CREATE TEMP TABLE _edited_agents ON COMMIT DROP AS SELECT * FROM custom_agents;
|
||||
CREATE TEMP TABLE _edited_skills ON COMMIT DROP AS SELECT * FROM tenant_skill_catalog;
|
||||
"""
|
||||
sql += query + unchanged_check("native_edits_disabling_and_binding_removal_preserved",
|
||||
[("custom_agents", "_edited_agents"), ("tenant_skill_catalog", "_edited_skills"), *pairs[2:]])
|
||||
sql += f"""
|
||||
UPDATE custom_agents SET deleted_at=now() WHERE id={aid} AND tenant_id={TARGET_TENANT};
|
||||
UPDATE tenant_skill_catalog SET deleted_at=now() WHERE id={sid} AND tenant_id={TARGET_TENANT};
|
||||
CREATE TEMP TABLE _deleted_agents ON COMMIT DROP AS SELECT * FROM custom_agents;
|
||||
CREATE TEMP TABLE _deleted_skills ON COMMIT DROP AS SELECT * FROM tenant_skill_catalog;
|
||||
"""
|
||||
sql += query + unchanged_check("soft_deleted_assets_not_revived",
|
||||
[("custom_agents", "_deleted_agents"), ("tenant_skill_catalog", "_deleted_skills"), *pairs[2:]])
|
||||
sql += "ROLLBACK;\n"
|
||||
return sql, len(cases) + 8
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--sqlite", type=Path, required=True)
|
||||
parser.add_argument("--kb-mapping", type=Path, required=True)
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.out.exists():
|
||||
parser.error("Output already exists")
|
||||
manifest = json.loads(args.manifest.read_text())
|
||||
with consistent_snapshot(args.sqlite) as (db, info):
|
||||
plan = build_plan(db, args.kb_mapping)
|
||||
expected = {k: v for k, v in manifest.items() if k != "source_snapshot_sha256"}
|
||||
if safe_asset_summary(plan) != expected or info["source_mutated"]:
|
||||
raise RuntimeError("Asset source differs from the reviewed manifest")
|
||||
if (len(plan["agents"]), len(plan["skills"]), len(plan["bindings"]),
|
||||
sum(b["effective"] for b in plan["bindings"])) != (15, 9, 39, 30):
|
||||
raise RuntimeError("Asset counts differ from the reviewed acceptance batch")
|
||||
plan["source_snapshot_sha256"] = info["sha256"]
|
||||
query, expected_checks = regression_sql(plan, asset_sql(plan))
|
||||
before, supporting_before = target_snapshot(), supporting_snapshot()
|
||||
output = sql_run(query)
|
||||
checks = [json.loads(line) for line in output.splitlines() if line.startswith("{")]
|
||||
after, supporting_after = target_snapshot(), supporting_snapshot()
|
||||
checks.extend((
|
||||
{"check": "all_regressions_executed", "passed": len(checks) == expected_checks},
|
||||
{"check": "all_test_changes_rolled_back", "passed": before == after and supporting_before == supporting_after},
|
||||
{"check": "acceptance_scope", "passed": before["database"] == after["database"] == TARGET_DATABASE},
|
||||
))
|
||||
report = {"checked_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"database": TARGET_DATABASE, "tenant_id": TARGET_TENANT,
|
||||
"committed": False, "source_mutated": info["source_mutated"],
|
||||
"checks": checks, "passed": all(c["passed"] for c in checks)}
|
||||
save_json(args.out, report)
|
||||
print(json.dumps(report))
|
||||
return 0 if report["passed"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify historical workbench imports through pinned local acceptance HTTP.
|
||||
|
||||
Run only after the business import and read-only protection deployment finish.
|
||||
Reports contain only boolean checks and counts, never resource or login bodies.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import Counter
|
||||
import json
|
||||
import secrets
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from apply_weknora_business import target_snapshot
|
||||
from apply_weknora_identities import sql_run
|
||||
from weknora_business_etl import LEDGER, RESOURCE_KINDS, SOURCE
|
||||
from weknora_category_etl import literal, save_json
|
||||
from weknora_identity_etl import TARGET_DATABASE, TARGET_TENANT
|
||||
|
||||
|
||||
BASE_URL = 'http://127.0.0.1:18080'
|
||||
ROOT = '/api/v1/workbench/resources/'
|
||||
EXPECTED_COUNTS = {'canvas': 3, 'trainingmap': 8, 'campaign': 2}
|
||||
ACTIONS = {
|
||||
'flows': ('generate',), # Flows have no executable action in the native API.
|
||||
'training-maps': ('generate', 'revise', 'generate-day', 'confirm-day', 'confirm'),
|
||||
'content-tasks': ('decompose', 'generate', 'revise', 'accept', 'skip', 'review', 'compose'),
|
||||
}
|
||||
READ_ONLY_TEXT = '\u53ea\u8bfb\u5386\u53f2\u8d44\u6599'
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--credentials', type=Path, required=True)
|
||||
parser.add_argument('--out', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.out.exists():
|
||||
parser.error('Report exists')
|
||||
|
||||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
checks, counts = {}, Counter()
|
||||
before, fixture_user, fixture_tenant = None, None, None
|
||||
|
||||
def check(name, passed):
|
||||
checks[name] = checks.get(name, True) and bool(passed)
|
||||
counts[name] += 1
|
||||
return bool(passed)
|
||||
|
||||
def require(name, passed):
|
||||
if not check(name, passed):
|
||||
raise RuntimeError('Acceptance precondition failed')
|
||||
|
||||
def request(path, token=None, data=None, method=None, tenant=TARGET_TENANT):
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
if tenant is not None:
|
||||
headers['X-Tenant-ID'] = str(tenant)
|
||||
if token:
|
||||
headers['Authorization'] = 'Bearer ' + token
|
||||
body = json.dumps(data, ensure_ascii=False).encode() if data is not None else None
|
||||
if body is not None and len(body) > 2 * 1024 * 1024:
|
||||
raise RuntimeError('Verification payload exceeds the native request limit')
|
||||
req = urllib.request.Request(BASE_URL + path, headers=headers, data=body, method=method)
|
||||
counts['http_requests'] += 1
|
||||
try:
|
||||
response = opener.open(req, timeout=30)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
payload = json.load(response)
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError('Unexpected acceptance response shape')
|
||||
return response.status, payload
|
||||
|
||||
def listed(kind, token):
|
||||
result = {}
|
||||
for page in range(1, 1001):
|
||||
status, body = request(ROOT + kind + f'?page={page}&page_size=100', token)
|
||||
if status != 200 or not isinstance(body.get('data'), list):
|
||||
return False, result
|
||||
rows, total = body['data'], body.get('total')
|
||||
if type(total) is not int or total < 0:
|
||||
return False, result
|
||||
old_count = len(result)
|
||||
for row in rows:
|
||||
result[row['id']] = row
|
||||
if len(result) >= total:
|
||||
return len(result) == total, result
|
||||
if len(result) == old_count:
|
||||
return False, result
|
||||
return False, result
|
||||
|
||||
def denied(name, path, token, data=None, method=None, tenant=TARGET_TENANT,
|
||||
statuses=(403, 404), readonly=False):
|
||||
status, body = request(path, token, data, method, tenant)
|
||||
passed = status in statuses and (not readonly or READ_ONLY_TEXT in body.get('message', ''))
|
||||
check(name, passed)
|
||||
# A successful mutation ends the run immediately; never attempt repairs.
|
||||
if method in ('PUT', 'DELETE', 'POST') and 200 <= status < 300:
|
||||
raise RuntimeError('A protected mutation unexpectedly succeeded')
|
||||
|
||||
try:
|
||||
before = target_snapshot()
|
||||
require('pinned_acceptance_database', before['database'] == TARGET_DATABASE)
|
||||
expected = json.loads(sql_run(
|
||||
f"SELECT coalesce(json_agg(json_build_object('kind',source_kind,'id',target_id,"
|
||||
f"'native',native_snapshot) ORDER BY source_kind,source_id),'[]') FROM {LEDGER} "
|
||||
f"WHERE source_system={literal(SOURCE)} AND source_organization_id=1 "
|
||||
f"AND target_tenant_id={TARGET_TENANT} AND source_kind IN ('canvas','trainingmap','campaign');"))
|
||||
require('exact_13_migrated_resources_in_ledger',
|
||||
Counter(item['kind'] for item in expected) == EXPECTED_COUNTS
|
||||
and len({item['id'] for item in expected}) == 13)
|
||||
for source_kind, count in EXPECTED_COUNTS.items():
|
||||
counts['migrated_' + RESOURCE_KINDS[source_kind]] = count
|
||||
require('all_13_native_snapshots_are_read_only', all(
|
||||
item['native']['data'].get('legacy_import', {}).get('read_only') is True
|
||||
and item['native']['tenant_id'] == TARGET_TENANT
|
||||
and item['native']['id'] == item['id']
|
||||
and item['native']['kind'] == RESOURCE_KINDS[item['kind']]
|
||||
for item in expected))
|
||||
|
||||
credentials = json.loads(args.credentials.read_text())
|
||||
status, auth = request('/api/v1/auth/login', data={k: credentials[k] for k in ('email', 'password')})
|
||||
require('administrator_login_succeeded', status == 200 and bool(auth.get('token')))
|
||||
token, administrator_id = auth['token'], auth.get('user', {}).get('id')
|
||||
require('administrator_identity_available', bool(administrator_id))
|
||||
counts['migrated_distinct_owners'] = len({item['native']['owner_id'] for item in expected})
|
||||
counts['migrated_administrator_owned_resources'] = sum(
|
||||
item['native']['owner_id'] == administrator_id for item in expected)
|
||||
counts['migrated_other_owner_resources'] = len(expected) - counts['migrated_administrator_owned_resources']
|
||||
|
||||
for kind in RESOURCE_KINDS.values():
|
||||
ok, resources = listed(kind, token)
|
||||
selected = [item for item in expected if item['native']['kind'] == kind]
|
||||
require('administrator_lists_all_migrated_resources',
|
||||
ok and all(item['id'] in resources for item in selected))
|
||||
denied('anonymous_list_denied', ROOT + kind, None, statuses=(401, 403))
|
||||
|
||||
for item in expected:
|
||||
native = item['native']
|
||||
path = ROOT + native['kind'] + '/' + item['id']
|
||||
status, body = request(path, token)
|
||||
current = body.get('data', {})
|
||||
require('native_data_owner_status_and_version_match', status == 200 and all(
|
||||
current.get(field) == native[field]
|
||||
for field in ('id', 'tenant_id', 'owner_id', 'kind', 'name', 'status', 'version', 'data')))
|
||||
status, body = request(path + '/jobs?page_size=100', token)
|
||||
require('imported_resources_have_no_jobs', status == 200 and body.get('data') == [] and body.get('total') == 0)
|
||||
for suffix in ('', '/jobs?page_size=100'):
|
||||
denied('anonymous_resource_and_jobs_read_denied', path + suffix, None, statuses=(401, 403))
|
||||
|
||||
payload = {key: native[key] for key in ('name', 'data', 'version')}
|
||||
owned = native['owner_id'] == administrator_id
|
||||
# PUT checks historical protection before locking. DELETE and supported
|
||||
# actions lock by owner first, even for a tenant administrator.
|
||||
denied('all_13_updates_rejected_as_read_only', path, token, payload, 'PUT', statuses=(400,), readonly=True)
|
||||
for detach, label in (('', 'deletes'), ('?detach=true', 'detaching_deletes')):
|
||||
denied(('owned_' + label + '_rejected_as_read_only') if owned else
|
||||
('other_owner_' + label + '_rejected_by_ownership'), path + detach, token,
|
||||
method='DELETE', statuses=(400,) if owned else (403, 404), readonly=owned)
|
||||
for action in ACTIONS[native['kind']]:
|
||||
parameters = {'version': native['version'], 'day': 1}
|
||||
if action == 'revise':
|
||||
parameters['advice'] = 'Verification only; preserve historical content.'
|
||||
segments = native['data'].get('segments', [])
|
||||
if segments:
|
||||
parameters['segment_id'] = segments[0]['id']
|
||||
is_flow = native['kind'] == 'flows'
|
||||
action_check = 'flow_actions_unsupported' if is_flow else (
|
||||
'owned_supported_actions_rejected_as_read_only' if owned else
|
||||
'other_owner_supported_actions_rejected_by_ownership')
|
||||
denied(action_check, path + '/actions/' + action, token, parameters, 'POST',
|
||||
statuses=(400,) if is_flow or owned else (403, 404), readonly=owned and not is_flow)
|
||||
denied('anonymous_update_denied', path, None, payload, 'PUT', statuses=(401, 403))
|
||||
denied('anonymous_delete_denied', path, None, method='DELETE', statuses=(401, 403))
|
||||
denied('anonymous_action_denied', path + '/actions/generate', None,
|
||||
{'version': native['version']}, 'POST', statuses=(401, 403))
|
||||
require('resources_and_execution_unchanged_after_admin_checks', before == target_snapshot())
|
||||
|
||||
suffix = secrets.token_hex(6)
|
||||
password = secrets.token_urlsafe(18) + 'aA1!'
|
||||
email = 'business-check-' + suffix + '@migration.invalid'
|
||||
existing_tenants = set(json.loads(sql_run("SELECT coalesce(json_agg(id),'[]') FROM tenants;")))
|
||||
status, body = request('/api/v1/system/admin/users/create', token, {
|
||||
'username': 'business-check-' + suffix, 'email': email, 'password': password})
|
||||
candidate = body.get('user', {}).get('id')
|
||||
require('temporary_user_created', status == 201 and candidate and str(uuid.UUID(candidate)) == candidate)
|
||||
fixture_user = candidate
|
||||
home = int(sql_run(f"SELECT tenant_id FROM users WHERE id={literal(fixture_user)};"))
|
||||
if home and home not in existing_tenants:
|
||||
fixture_tenant = home
|
||||
require('temporary_user_home_is_new_or_tenantless', home == 0 or fixture_tenant == home)
|
||||
sql_run(f"INSERT INTO tenant_members(user_id,tenant_id,role,status) VALUES "
|
||||
f"({literal(fixture_user)},{TARGET_TENANT},'contributor','active');")
|
||||
status, auth = request('/api/v1/auth/login', data={'email': email, 'password': password})
|
||||
require('contributor_login_succeeded', status == 200 and bool(auth.get('token')))
|
||||
member_token = auth['token']
|
||||
fixture_scope = json.loads(sql_run(f"SELECT json_build_object("
|
||||
f"'is_admin',is_system_admin,'all_tenants',can_access_all_tenants,"
|
||||
f"'memberships',(SELECT count(*) FROM tenant_members WHERE user_id=u.id AND deleted_at IS NULL),"
|
||||
f"'has_personal_membership',EXISTS(SELECT 1 FROM tenant_members WHERE user_id=u.id "
|
||||
f"AND tenant_id={fixture_tenant or 0} AND role='owner' AND status='active' AND deleted_at IS NULL),"
|
||||
f"'is_contributor',EXISTS(SELECT 1 FROM tenant_members WHERE user_id=u.id "
|
||||
f"AND tenant_id={TARGET_TENANT} AND role='contributor' AND status='active' AND deleted_at IS NULL)) "
|
||||
f"FROM users u WHERE id={literal(fixture_user)};"))
|
||||
require('temporary_user_is_isolated_contributor', fixture_scope == {
|
||||
'is_admin': False, 'all_tenants': False, 'memberships': 1 + bool(fixture_tenant),
|
||||
'has_personal_membership': bool(fixture_tenant), 'is_contributor': True})
|
||||
|
||||
for kind in RESOURCE_KINDS.values():
|
||||
ok, resources = listed(kind, member_token)
|
||||
require('contributor_lists_no_other_owners_resources', ok and not resources)
|
||||
denied('cross_tenant_list_denied', ROOT + kind, member_token, tenant=10060)
|
||||
for item in expected:
|
||||
native = item['native']
|
||||
path = ROOT + native['kind'] + '/' + item['id']
|
||||
payload = {key: native[key] for key in ('name', 'data', 'version')}
|
||||
for suffix in ('', '/jobs?page_size=100'):
|
||||
denied('contributor_direct_resource_and_jobs_read_denied', path + suffix, member_token)
|
||||
denied('cross_tenant_direct_resource_and_jobs_read_denied', path + suffix, member_token, tenant=10060)
|
||||
denied('contributor_update_denied', path, member_token, payload, 'PUT')
|
||||
denied('contributor_delete_denied', path, member_token, method='DELETE')
|
||||
denied('cross_tenant_update_denied', path, member_token, payload, 'PUT', tenant=10060)
|
||||
denied('cross_tenant_delete_denied', path, member_token, method='DELETE', tenant=10060)
|
||||
denied('cross_tenant_action_denied', path + '/actions/generate', member_token,
|
||||
{'version': native['version']}, 'POST', tenant=10060)
|
||||
if native['kind'] != 'flows':
|
||||
denied('contributor_direct_action_denied', path + '/actions/generate', member_token,
|
||||
{'version': native['version']}, 'POST')
|
||||
else:
|
||||
denied('contributor_flow_action_unsupported', path + '/actions/generate', member_token,
|
||||
{'version': native['version']}, 'POST', statuses=(400,))
|
||||
check('all_api_checks_completed', True)
|
||||
except Exception:
|
||||
# Transport/database exceptions can contain private paths or response data.
|
||||
check('all_api_checks_completed', False)
|
||||
finally:
|
||||
if fixture_user:
|
||||
try:
|
||||
disable_tenant = (f"UPDATE tenants SET status='inactive',updated_at=now() "
|
||||
f"WHERE id={fixture_tenant}; ") if fixture_tenant else ''
|
||||
sql_run(f"BEGIN; DELETE FROM tenant_members WHERE user_id={literal(fixture_user)}; "
|
||||
f"DELETE FROM auth_tokens WHERE user_id={literal(fixture_user)}; "
|
||||
f"UPDATE users SET is_active=false,updated_at=now() WHERE id={literal(fixture_user)}; "
|
||||
+ disable_tenant + "COMMIT;")
|
||||
cleaned = sql_run(f"SELECT NOT is_active AND NOT EXISTS(SELECT 1 FROM tenant_members "
|
||||
f"WHERE user_id=u.id) AND NOT EXISTS(SELECT 1 FROM auth_tokens WHERE user_id=u.id) "
|
||||
f"FROM users u WHERE id={literal(fixture_user)};") == 't'
|
||||
check('temporary_user_deactivated_membership_and_tokens_removed', cleaned)
|
||||
if fixture_tenant:
|
||||
check('temporary_personal_tenant_deactivated', sql_run(
|
||||
f"SELECT status='inactive' FROM tenants WHERE id={fixture_tenant};") == 't')
|
||||
except Exception:
|
||||
check('temporary_user_deactivated_membership_and_tokens_removed', False)
|
||||
if before is not None:
|
||||
try:
|
||||
check('all_existing_resources_jobs_events_revisions_assets_and_knowledge_unchanged', before == target_snapshot())
|
||||
except Exception:
|
||||
check('all_existing_resources_jobs_events_revisions_assets_and_knowledge_unchanged', False)
|
||||
result = {'passed': bool(checks) and all(checks.values()), 'checks': checks, 'counts': dict(counts),
|
||||
'fixture_user_created': fixture_user is not None,
|
||||
'fixture_personal_tenant_created': fixture_tenant is not None}
|
||||
save_json(args.out, result)
|
||||
print(json.dumps(result))
|
||||
return 0 if result['passed'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,153 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
|
||||
|
||||
async function main() {
|
||||
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(directory, 'business-apply-20260916.manifest.json'), 'utf8'))
|
||||
const credentials = JSON.parse(fs.readFileSync(path.join(directory, 'super-admin-login.json'), 'utf8'))
|
||||
const login = await fetch('http://127.0.0.1:18080/api/v1/auth/login', {
|
||||
method: 'POST', headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({email: credentials.email, password: credentials.password}),
|
||||
})
|
||||
assert.equal(login.status, 200, 'Administrator login')
|
||||
const auth = await login.json()
|
||||
const headers = {'Content-Type': 'application/json', Authorization: `Bearer ${auth.token}`, 'X-Tenant-ID': '10059'}
|
||||
async function request(url, method = 'GET', data) {
|
||||
const response = await fetch(`http://127.0.0.1:18080/api/v1${url}`, {method, headers, body: data === undefined ? undefined : JSON.stringify(data)})
|
||||
assert.ok(response.ok, `Acceptance HTTP ${response.status}`)
|
||||
return response.json()
|
||||
}
|
||||
const checks = [], errors = []
|
||||
const browser = await chromium.launch({headless: true, channel: 'chrome'})
|
||||
let fixture, page, stage = 'starting'
|
||||
const check = (name, passed) => {checks.push({check: name, passed: !!passed}); assert.ok(passed, name)}
|
||||
try {
|
||||
const context = await browser.newContext({viewport: {width: 1440, height: 1000}})
|
||||
await context.addInitScript(a => {
|
||||
const tenant = {...a.active_tenant, id: '10059'}
|
||||
const data = {weknora_user: JSON.stringify({...a.user, tenant_id: String(a.user.tenant_id)}),
|
||||
weknora_tenant: JSON.stringify(tenant), weknora_token: a.token,
|
||||
weknora_refresh_token: a.refresh_token, weknora_memberships: JSON.stringify(a.memberships)}
|
||||
for (const [key, value] of Object.entries(data)) localStorage.setItem(key, value)
|
||||
localStorage.setItem('weknora:new-user-guide-done:v1', '1')
|
||||
}, auth)
|
||||
page = await context.newPage()
|
||||
page.setDefaultTimeout(15000)
|
||||
page.on('pageerror', () => errors.push('browser_error'))
|
||||
async function visit(url) {
|
||||
await page.goto('http://127.0.0.1:15173' + url)
|
||||
await page.locator('.workbench-resource').first().waitFor({timeout: 30000})
|
||||
const guide = page.locator('.guide__close')
|
||||
if (await guide.isVisible()) await guide.click()
|
||||
}
|
||||
async function resize(width, height) {
|
||||
stage = 'resize_viewport'
|
||||
await page.setViewportSize({width, height})
|
||||
stage = 'wait_sidebar_collapse'
|
||||
if (width <= 640) await page.locator('.aside_box--collapsed').waitFor()
|
||||
stage = 'wait_page_dimensions'
|
||||
await page.waitForFunction(() => {
|
||||
const bounds = document.querySelector('.workbench-page')?.getBoundingClientRect()
|
||||
return bounds && bounds.width >= Math.min(300, innerWidth - 80) && bounds.right <= innerWidth + 2
|
||||
}, undefined, {polling: 100})
|
||||
stage = 'wait_layout_transition'
|
||||
await page.evaluate(async () => {
|
||||
await Promise.race([
|
||||
Promise.all(document.getAnimations()
|
||||
.filter(animation => Number.isFinite(animation.effect?.getComputedTiming().endTime))
|
||||
.map(animation => animation.finished.catch(() => {}))),
|
||||
new Promise(resolve => setTimeout(resolve, 1000)),
|
||||
])
|
||||
})
|
||||
}
|
||||
async function mobileCheck(kind) {
|
||||
await resize(390, 844)
|
||||
check(`${kind}_mobile_usable_width`, await page.locator('.workbench-page').evaluate(element => {
|
||||
const bounds = element.getBoundingClientRect()
|
||||
const detail = element.querySelector('.workbench-detail')?.getBoundingClientRect()
|
||||
return bounds.width >= 300 && bounds.right <= innerWidth + 2 && detail && detail.width >= 260 && detail.right <= innerWidth + 2
|
||||
}))
|
||||
await page.locator('.workbench-page').evaluate(element => { element.scrollTop = 0 })
|
||||
await page.screenshot({path: path.join(directory, `business-${kind}-mobile.png`), animations: 'disabled'})
|
||||
check(`${kind}_mobile_no_overflow`, await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 2))
|
||||
}
|
||||
for (const [kind, route, tab] of [
|
||||
['trainingmap', '/platform/workbench/content', 'maps'],
|
||||
['campaign', '/platform/workbench/content', 'tasks'],
|
||||
]) {
|
||||
const ids = Object.values(manifest.resource_map[kind])
|
||||
await visit(`${route}?tab=${tab}&resource=${ids.at(-1)}`)
|
||||
check(`${kind}_list_count`, await page.locator('.workbench-resource').count() === ids.length)
|
||||
check(`${kind}_history_warning`, await page.locator('.workbench-detail .t-alert').isVisible())
|
||||
check(`${kind}_delete_hidden`, await page.locator('.workbench-detail').getByRole('button', {name: '删除', exact: true}).count() === 0)
|
||||
const aiButtons = page.locator('.workbench-detail').getByRole('button', {name: kind === 'campaign' ? '重新拆解' : '重新生成方案', exact: true})
|
||||
check(`${kind}_execution_disabled`, await aiButtons.isDisabled())
|
||||
await page.screenshot({path: path.join(directory, `business-${kind}-desktop.png`), animations: 'disabled'})
|
||||
await page.getByRole('button', {name: '全屏查看', exact: true}).click()
|
||||
await page.locator('.t-dialog:visible .workbench-preview').waitFor()
|
||||
check(`${kind}_fullscreen_readable`, await page.locator('.t-dialog:visible .workbench-preview').isVisible())
|
||||
stage = 'close_desktop_preview'
|
||||
await page.locator('.t-dialog:visible .t-dialog__close').click()
|
||||
stage = 'wait_desktop_preview_closed'
|
||||
await page.locator('.t-dialog:visible').waitFor({state: 'hidden'})
|
||||
await mobileCheck(kind)
|
||||
await page.getByRole('button', {name: '全屏查看', exact: true}).click()
|
||||
await page.locator('.t-dialog:visible .workbench-preview').waitFor()
|
||||
check(`${kind}_mobile_fullscreen_fits`, await page.locator('.t-dialog:visible').evaluate(element => {
|
||||
const bounds = element.getBoundingClientRect()
|
||||
return bounds.left >= 0 && bounds.right <= innerWidth + 2
|
||||
}))
|
||||
await page.screenshot({path: path.join(directory, `business-${kind}-mobile-fullscreen.png`), animations: 'disabled'})
|
||||
await page.locator('.t-dialog:visible .t-dialog__close').click()
|
||||
await page.locator('.t-dialog:visible').waitFor({state: 'hidden'})
|
||||
await resize(1440, 1000)
|
||||
}
|
||||
await visit('/platform/workbench/flows')
|
||||
check('three_historical_flows_listed', await page.locator('.workbench-resource').count() === 3)
|
||||
await page.locator('.workbench-resource').first().click()
|
||||
check('flow_history_warning', await page.locator('.workbench-detail .t-alert').isVisible())
|
||||
check('flow_delete_and_save_hidden', await page.locator('.workbench-detail').getByRole('button', {name: /^(保存|解绑并删除)$/}).count() === 0)
|
||||
await page.screenshot({path: path.join(directory, 'business-flow-desktop.png'), animations: 'disabled'})
|
||||
await mobileCheck('flow')
|
||||
await resize(1440, 1000)
|
||||
|
||||
const agents = (await request('/agents')).data
|
||||
const agent = agents.find(a => a.enabled && a.config.model_id && a.created_by === auth.user.id)
|
||||
assert.ok(agent, 'Runnable owned Agent available for branch editor fixture')
|
||||
const edges = [{source: 'a', target: 'b'}, {source: 'a', target: 'c'}]
|
||||
fixture = (await request('/workbench/resources/flows', 'POST', {
|
||||
name: `Migration branch check ${Date.now()}`,
|
||||
data: {description: 'Disposable branch preservation check',
|
||||
nodes: ['a', 'b', 'c'].map(id => ({id, label: `Branch ${id}`, agent_id: agent.id})), edges},
|
||||
})).data
|
||||
await visit('/platform/workbench/flows')
|
||||
await page.locator('.workbench-resource').filter({hasText: fixture.name}).click()
|
||||
await page.locator('.flow-node').nth(2).getByRole('button').nth(0).click()
|
||||
const saved = page.waitForResponse(response => response.url().endsWith(fixture.id) && response.request().method() === 'PUT')
|
||||
await page.getByRole('button', {name: '保存', exact: true}).click()
|
||||
check('native_branch_save_success', (await saved).status() === 200)
|
||||
const current = (await request(`/workbench/resources/flows/${fixture.id}`)).data
|
||||
check('node_display_reordered', current.data.nodes.map(n => n.id).join(',') === 'a,c,b')
|
||||
check('branch_edges_preserved_after_reorder_and_save', JSON.stringify(current.data.edges) === JSON.stringify(edges))
|
||||
check('no_browser_javascript_errors', errors.length === 0)
|
||||
} finally {
|
||||
if (checks.length < 25 && page && !page.isClosed()) {
|
||||
console.log(JSON.stringify({stage, layout: await page.evaluate(() => ({
|
||||
dialogs: document.querySelectorAll('.t-dialog').length,
|
||||
collapsed: !!document.querySelector('.aside_box--collapsed'),
|
||||
width: innerWidth,
|
||||
page: document.querySelector('.workbench-page')?.getBoundingClientRect().toJSON(),
|
||||
}))}))
|
||||
await page.screenshot({path: path.join(directory, 'business-browser-debug.png'), animations: 'disabled'})
|
||||
}
|
||||
if (fixture) await request(`/workbench/resources/flows/${fixture.id}?detach=true`, 'DELETE')
|
||||
await browser.close()
|
||||
fs.writeFileSync(path.join(directory, 'business-browser-20260916.json'), JSON.stringify({
|
||||
passed: checks.length === 25 && checks.every(c => c.passed), checks, fixture_removed: !!fixture,
|
||||
}, null, 2) + '\n')
|
||||
}
|
||||
console.log(JSON.stringify({checks, passed: checks.every(c => c.passed)}))
|
||||
}
|
||||
main().catch(error => { console.error(error instanceof assert.AssertionError ? error.message : `Business browser verification failed (${error.name})`); process.exitCode = 1 })
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise historical business imports in PostgreSQL and roll back every case."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from apply_weknora_business import build_plan, target_snapshot, PRESERVED_TABLES
|
||||
from apply_weknora_identities import sql_run
|
||||
from verify_asset_etl import unchanged_check
|
||||
from verify_identity_etl import rejected_case
|
||||
from weknora_business_etl import LEDGER, TARGET_DATABASE, TARGET_TENANT, business_sql
|
||||
from weknora_category_etl import literal, save_json
|
||||
from weknora_etl import consistent_snapshot
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--sqlite', type=Path, required=True)
|
||||
parser.add_argument('--kb-mapping', type=Path, required=True)
|
||||
parser.add_argument('--artifact-root', type=Path, required=True)
|
||||
parser.add_argument('--out', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.out.exists():
|
||||
parser.error('Report exists')
|
||||
with consistent_snapshot(args.sqlite) as (db, info):
|
||||
plan = build_plan(db, args.kb_mapping, args.artifact_root)
|
||||
plan['source_snapshot_sha256'] = info['sha256']
|
||||
query = business_sql(plan)
|
||||
first = plan['resources'][0]['target']
|
||||
rid = literal(first['id'])
|
||||
owner = literal(next(uid for uid in plan['identity_mapping'].values() if uid != first['owner_id']))
|
||||
scope = f"target_id={rid} AND target_tenant_id={TARGET_TENANT}"
|
||||
cases = [
|
||||
('wrong_resource_tenant_rejected', f'UPDATE workbench_resources SET tenant_id={TARGET_TENANT + 1} WHERE id={rid};', 'Mapped business resource removed'),
|
||||
('wrong_resource_owner_rejected', f'UPDATE workbench_resources SET owner_id={owner} WHERE id={rid};', 'Mapped business resource removed'),
|
||||
('deleted_resource_not_recreated', f'DELETE FROM workbench_resources WHERE id={rid};', 'Mapped business resource removed'),
|
||||
('ledger_target_conflict_rejected', f"UPDATE {LEDGER} SET target_id='rollback-only' WHERE {scope};", 'Business durable mapping conflict'),
|
||||
('ledger_source_conflict_rejected', f"UPDATE {LEDGER} SET source_snapshot='{{}}' WHERE {scope};", 'Business durable mapping conflict'),
|
||||
('ledger_projection_conflict_rejected', f"UPDATE {LEDGER} SET native_snapshot='{{}}' WHERE {scope};", 'Business durable mapping conflict'),
|
||||
('id_without_provenance_rejected', f'DELETE FROM {LEDGER} WHERE {scope};', 'Native business ID or owner/name collision'),
|
||||
('inactive_tenant_rejected', f"UPDATE tenants SET status='suspended' WHERE id={TARGET_TENANT};", 'Business migration requires active target tenant'),
|
||||
]
|
||||
before = target_snapshot()
|
||||
ledger_before = sql_run(f"SELECT to_regclass('public.{LEDGER}') IS NOT NULL;")
|
||||
sql = "BEGIN ISOLATION LEVEL SERIALIZABLE; SET LOCAL statement_timeout='45s'; SET LOCAL lock_timeout='5s';\n"
|
||||
preserved = [(table, f'_preserved_{i}') for i, table in enumerate(PRESERVED_TABLES)]
|
||||
for table, name in preserved:
|
||||
sql += f'CREATE TEMP TABLE {name} ON COMMIT DROP AS SELECT * FROM {table};\n'
|
||||
sql += query
|
||||
pairs = [('workbench_resources', '_resource_repeat'), (LEDGER, '_ledger_repeat')]
|
||||
for table, name in pairs:
|
||||
sql += f'CREATE TEMP TABLE {name} ON COMMIT DROP AS SELECT * FROM {table};\n'
|
||||
sql += query + unchanged_check('exact_repeat_noop', pairs + preserved)
|
||||
for name, mutation, expected in cases:
|
||||
sql += rejected_case(name, mutation, expected, query)
|
||||
sql += f"UPDATE workbench_resources SET name='Rollback native edit',status='paused',version=99,data=jsonb_set(data,'{{description}}','\"Local edit\"'::jsonb) WHERE id={rid};\n"
|
||||
sql += 'CREATE TEMP TABLE _native_edit ON COMMIT DROP AS SELECT * FROM workbench_resources;\n'
|
||||
sql += query + unchanged_check('native_edits_preserved', [('workbench_resources', '_native_edit'), pairs[1], *preserved])
|
||||
sql += 'ROLLBACK;'
|
||||
checks = [json.loads(line) for line in sql_run(sql).splitlines() if line.startswith('{')]
|
||||
checks.extend([
|
||||
{'check': 'all_cases_executed', 'passed': len(checks) == len(cases) + 2},
|
||||
{'check': 'all_regressions_rolled_back', 'passed': before == target_snapshot() and ledger_before == sql_run(f"SELECT to_regclass('public.{LEDGER}') IS NOT NULL;")},
|
||||
{'check': 'source_unchanged', 'passed': not info['source_mutated']},
|
||||
])
|
||||
result = {'database': TARGET_DATABASE, 'committed': False, 'passed': all(c['passed'] for c in checks), 'checks': checks}
|
||||
save_json(args.out, result)
|
||||
print(json.dumps(result))
|
||||
return 0 if result['passed'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,164 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const crypto = require('node:crypto')
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
|
||||
|
||||
const root = path.resolve(__dirname, '../..')
|
||||
const origin = process.env.VERIFY_FRONTEND_ORIGIN || 'https://agent.pducv.cn'
|
||||
const apiOrigin = process.env.VERIFY_API_ORIGIN || 'http://127.0.0.1:18080'
|
||||
const kbID = '16c3e59c-4bb6-41cf-ae0a-6d0941b3d032'
|
||||
const report = { checked_at: new Date().toISOString(), origin, api_origin: apiOrigin, checks: [], failures: [] }
|
||||
let auth, browser
|
||||
|
||||
function check(name, value) {
|
||||
report.checks.push({ name, passed: !!value })
|
||||
assert.ok(value, name)
|
||||
}
|
||||
|
||||
async function api(route, requestOrigin = apiOrigin) {
|
||||
const response = await fetch(`${requestOrigin}/api/v1${route}`, {
|
||||
headers: { Authorization: `Bearer ${auth.token}`, 'X-Tenant-ID': '10059' },
|
||||
signal: AbortSignal.timeout(30000),
|
||||
})
|
||||
assert.equal(response.status, 200, `GET ${route}: HTTP ${response.status}`)
|
||||
const result = await response.json()
|
||||
assert.notEqual(result.success, false, `GET ${route} failed`)
|
||||
return result.data
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const credentials = JSON.parse(fs.readFileSync(path.join(root, 'acceptance/super-admin-login.json'), 'utf8'))
|
||||
const login = await fetch(`${apiOrigin}/api/v1/auth/login`, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(credentials),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
})
|
||||
assert.equal(login.status, 200, 'authentication failed')
|
||||
const loginResult = await login.json()
|
||||
auth = loginResult.data || loginResult
|
||||
|
||||
const kbs = await api('/knowledge-bases')
|
||||
report.knowledge_bases = []
|
||||
for (const kb of kbs) {
|
||||
const config = await api(`/initialization/config/${kb.id}`)
|
||||
const initialized = !!(kb.summary_model_id && kb.embedding_model_id && config.llm?.modelName && config.embedding?.modelName)
|
||||
report.knowledge_bases.push({ id: kb.id, name: kb.name, initialized, graph_enabled: !!kb.extract_config?.enabled,
|
||||
llm_model: config.llm?.modelName, embedding_model: config.embedding?.modelName })
|
||||
check(`models_resolve:${kb.name}`, initialized)
|
||||
}
|
||||
const overview = await api(`/knowledge-bases/${kbID}/evidence-graph`)
|
||||
report.graph = { counts: overview.counts, sources: overview.sources, mapped_sources: overview.mapped_sources }
|
||||
check('product_has_extracted_entities_and_relations', overview.counts.knowledgeentity > 2 && overview.counts.knowledgerelation > 0)
|
||||
const nodes = [...overview.nodes]
|
||||
let cursor = overview.next_cursor
|
||||
for (let pages = 0; cursor && pages < 50; pages++) {
|
||||
const next = await api(`/knowledge-bases/${kbID}/evidence-graph?cursor=${cursor}`)
|
||||
nodes.push(...next.nodes)
|
||||
assert.notEqual(next.next_cursor, cursor, 'graph cursor did not advance')
|
||||
cursor = next.next_cursor
|
||||
}
|
||||
assert.equal(cursor, 0, 'graph exceeded verification pagination bound')
|
||||
check('all_entity_ids_are_exact_in_browser', nodes.every(node => Number.isSafeInteger(node.id)))
|
||||
const nativeNodes = nodes.filter(node => node.payload.origin === 'native_extraction_v1')
|
||||
check('native_extraction_is_visible', nativeNodes.length > 0)
|
||||
check('example_entities_are_absent', !nativeNodes.some(node => ['基础概念', '实践练习'].includes(node.payload.canonical_name)))
|
||||
const chunks = new Map(), documents = new Map(), relations = new Map(), evidenceIDs = new Set()
|
||||
let selected
|
||||
for (const node of nativeNodes) {
|
||||
const detail = await api(`/knowledge-bases/${kbID}/evidence-graph?entity=${node.id}`)
|
||||
assert.equal(detail.nodes[0]?.id, node.id, 'entity detail must resolve exact ID')
|
||||
const nativeEvidence = detail.evidence.filter(item => item.payload.origin === 'native_extraction_v1')
|
||||
assert.ok(nativeEvidence.length, `native entity ${node.id} has no evidence`)
|
||||
if (!selected && detail.relations.length) selected = { node, evidence: nativeEvidence[0] }
|
||||
for (const relation of detail.relations) relations.set(relation.id, relation)
|
||||
for (const evidence of nativeEvidence) {
|
||||
assert.equal(evidence.mapping_state, 'native_extraction', 'evidence no longer maps to its original chunk')
|
||||
assert.ok(evidence.native_chunk_id && evidence.native_knowledge_id, 'evidence source link is missing')
|
||||
if (!chunks.has(evidence.native_chunk_id)) chunks.set(evidence.native_chunk_id, await api(`/chunks/by-id/${evidence.native_chunk_id}`))
|
||||
if (!documents.has(evidence.native_knowledge_id)) documents.set(evidence.native_knowledge_id, await api(`/knowledge/${evidence.native_knowledge_id}`))
|
||||
const chunk = chunks.get(evidence.native_chunk_id), document = documents.get(evidence.native_knowledge_id)
|
||||
assert.equal(chunk.knowledge_base_id, kbID, 'chunk belongs to another knowledge base')
|
||||
assert.equal(chunk.knowledge_id, evidence.native_knowledge_id, 'chunk does not belong to the source document')
|
||||
assert.equal(document.knowledge_base_id, kbID, 'source document belongs to another knowledge base')
|
||||
assert.equal(crypto.createHash('sha256').update(chunk.content).digest('hex'), evidence.payload.source_content_hash,
|
||||
'evidence content hash differs from actual chunk')
|
||||
assert.ok(evidence.payload.excerpt && chunk.content.includes(evidence.payload.excerpt), 'evidence excerpt is not from the original chunk')
|
||||
evidenceIDs.add(evidence.id)
|
||||
}
|
||||
}
|
||||
check('native_evidence_resolves_to_unchanged_source', evidenceIDs.size > 0)
|
||||
check('example_relation_is_absent', [...relations.values()].every(relation =>
|
||||
!(relation.payload.from_name === '基础概念' && relation.payload.to_name === '实践练习')))
|
||||
const nativeRelations = [...relations.values()].filter(relation => relation.payload.origin === 'native_extraction_v1')
|
||||
for (const relation of nativeRelations) {
|
||||
assert.ok(typeof relation.payload.evidence === 'string' && relation.payload.evidence.trim(),
|
||||
`native relation ${relation.id} has no source quotation`)
|
||||
assert.ok([...chunks.values()].some(chunk => chunk.content.includes(relation.payload.evidence)),
|
||||
`native relation ${relation.id} quotation is absent from verified source chunks`)
|
||||
}
|
||||
check('native_relations_have_verbatim_source_quotations', nativeRelations.length > 0)
|
||||
report.verified_native = { entities: nativeNodes.length, entity_evidence: evidenceIDs.size,
|
||||
relations: nativeRelations.length, quoted_relations: nativeRelations.length,
|
||||
chunks: chunks.size, documents: documents.size }
|
||||
report.sample_relations = [...relations.values()].slice(0, 5).map(relation => ({
|
||||
from: relation.payload.from_name, relation: relation.payload.relation_type, to: relation.payload.to_name,
|
||||
}))
|
||||
const publicOverview = await api(`/knowledge-bases/${kbID}/evidence-graph`, origin)
|
||||
check('public_graph_api_exposes_completed_extractions', publicOverview.counts.knowledgeentity >= overview.counts.knowledgeentity)
|
||||
const denied = await fetch(`${origin}/api/v1/knowledge-bases/${kbID}/evidence-graph`, { signal: AbortSignal.timeout(30000) })
|
||||
check('public_graph_api_requires_auth', denied.status === 401)
|
||||
|
||||
browser = await chromium.launch({ headless: true, channel: 'chrome' })
|
||||
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } })
|
||||
await context.addInitScript(value => {
|
||||
const local = { weknora_user: JSON.stringify({ ...value.user, tenant_id: String(value.user.tenant_id) }),
|
||||
weknora_tenant: JSON.stringify({ ...value.active_tenant, id: '10059' }), weknora_token: value.token,
|
||||
weknora_refresh_token: value.refresh_token, weknora_memberships: JSON.stringify(value.memberships), weknora_selected_tenant_id: '10059' }
|
||||
for (const [key, item] of Object.entries(local)) localStorage.setItem(key, item)
|
||||
localStorage.setItem('weknora:new-user-guide-done:v1', '1')
|
||||
}, auth)
|
||||
const page = await context.newPage(), errors = []
|
||||
page.on('pageerror', error => errors.push(error.message))
|
||||
page.on('response', response => {
|
||||
const pathname = new URL(response.url()).pathname
|
||||
if (response.status() >= 400 && /\/(?:api|assets)\//.test(pathname)) errors.push(`${response.status()} ${pathname}`)
|
||||
})
|
||||
await page.goto(`${origin}/platform/knowledge-bases/${kbID}?tab=evidence`)
|
||||
await page.locator('.entity-row').first().waitFor({ timeout: 30000 })
|
||||
assert.ok(selected, 'missing native entity with relation for browser verification')
|
||||
await page.getByRole('textbox', { name: '搜索实体' }).fill(selected.node.payload.canonical_name)
|
||||
await page.getByRole('button', { name: '搜索', exact: true }).click()
|
||||
await page.locator('.entity-row').filter({ hasText: `#${selected.node.id}` }).click()
|
||||
await page.locator('.evidence-row').first().waitFor({ timeout: 30000 })
|
||||
check('native_entity_relations_visible', await page.locator('.relation-list li').count() > 0)
|
||||
check('source_evidence_marked_verified', await page.getByText('已核验原生分块', { exact: true }).count() > 0)
|
||||
await page.screenshot({ path: path.join(root, 'acceptance/business-graph-live-desktop.png'), animations: 'disabled' })
|
||||
await page.getByRole('button', { name: '查看原生文档', exact: true }).first().click()
|
||||
await page.locator('.t-drawer--open').first().waitFor({ timeout: 30000 })
|
||||
const originalDocument = documents.get(selected.evidence.native_knowledge_id)
|
||||
const drawer = page.locator('.t-drawer--open').first()
|
||||
const rawTitle = originalDocument.file_name || originalDocument.title || originalDocument.source
|
||||
const extensionIndex = rawTitle.lastIndexOf('.')
|
||||
const displayTitle = originalDocument.type === 'file' && extensionIndex > 0 ? rawTitle.slice(0, extensionIndex) : rawTitle
|
||||
await page.waitForFunction(title => [...document.querySelectorAll('.t-drawer--open .doc-drawer-header-title')]
|
||||
.some(element => element.textContent.trim() === title), displayTitle, { timeout: 30000 })
|
||||
check('source_document_opens_in_drawer', (await drawer.locator('.doc-drawer-header-title').innerText()).trim() === displayTitle)
|
||||
await page.screenshot({ path: path.join(root, 'acceptance/business-graph-live-source.png'), animations: 'disabled' })
|
||||
await page.keyboard.press('Escape')
|
||||
await page.setViewportSize({ width: 390, height: 844 })
|
||||
await page.screenshot({ path: path.join(root, 'acceptance/business-graph-live-mobile.png'), animations: 'disabled' })
|
||||
check('mobile_graph_has_no_overflow', await page.locator('.evidence-graph').evaluate(element => element.scrollWidth <= element.clientWidth + 2))
|
||||
check('mobile_page_has_no_overflow', await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 2))
|
||||
await page.goto(`${origin}/platform/knowledge-bases`)
|
||||
await page.waitForLoadState('networkidle', { timeout: 30000 })
|
||||
check('knowledge_base_list_has_no_initialization_warning', await page.locator('.warning-banner').count() === 0 && await page.locator('.uninitialized').count() === 0)
|
||||
report.browser_errors = errors
|
||||
check('browser_has_no_api_or_runtime_errors', errors.length === 0)
|
||||
}
|
||||
|
||||
main().catch(error => { report.failures.push(error.message); process.exitCode = 1 }).finally(async () => {
|
||||
if (browser) await browser.close()
|
||||
report.passed = report.failures.length === 0
|
||||
fs.writeFileSync(path.join(root, 'acceptance/business-graph-live.json'), JSON.stringify(report, null, 2) + '\n')
|
||||
console.log(JSON.stringify(report))
|
||||
})
|
||||
@@ -0,0 +1,183 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const crypto = require('node:crypto')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
|
||||
const kb = '16c3e59c-4bb6-41cf-ae0a-6d0941b3d032'
|
||||
const problemChunk = '1144acfb-18de-4687-8bf5-df0e9c8f1dd5'
|
||||
const outputPath = path.resolve(__dirname, '../../acceptance/business-graph-semantics.json')
|
||||
const report = { checked_at: new Date().toISOString(), production_write: false, checks: [], failures: [] }
|
||||
const query = `BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY;
|
||||
SELECT json_build_object(
|
||||
'config',(SELECT extract_config FROM knowledge_bases WHERE tenant_id=10059 AND id='${kb}' AND deleted_at IS NULL),
|
||||
'chunks',COALESCE((SELECT json_agg(json_build_object('id',c.id,'knowledge_id',c.knowledge_id,'content',c.content,'file_name',k.file_name) ORDER BY c.id)
|
||||
FROM chunks c JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id
|
||||
WHERE c.tenant_id=10059 AND c.knowledge_base_id='${kb}' AND c.chunk_type='text' AND c.is_enabled=true AND c.deleted_at IS NULL AND k.deleted_at IS NULL),'[]'),
|
||||
'records',COALESCE((SELECT json_agg(json_build_object('id',legacy_id,'table',source_table,'payload',payload) ORDER BY source_table,legacy_id)
|
||||
FROM native_graph_records WHERE tenant_id=10059 AND knowledge_base_id='${kb}' AND payload->>'origin'='native_extraction_v1'),'[]'),
|
||||
'sources',COALESCE((SELECT json_agg(json_build_object('source_chunk_id',source_chunk_id,'native_chunk_id',native_chunk_id,'native_knowledge_id',native_knowledge_id,'source_content_hash',source_content_hash,'native_content_sha256',native_content_sha256,'mapping_state',mapping_state))
|
||||
FROM native_graph_sources WHERE tenant_id=10059 AND knowledge_base_id='${kb}' AND mapping_state='native_extraction'),'[]')
|
||||
);
|
||||
COMMIT;`
|
||||
|
||||
function check(name, passed) {
|
||||
report.checks.push({ name, passed: !!passed })
|
||||
if (!passed) report.failures.push(name)
|
||||
}
|
||||
|
||||
try {
|
||||
const input = JSON.parse(execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres',
|
||||
'psql', '-X', '-qAt', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915'],
|
||||
{ input: query, encoding: 'utf8', maxBuffer: 20 * 1024 * 1024 }))
|
||||
const chunks = new Map(input.chunks.map(chunk => [chunk.id, chunk]))
|
||||
const hashes = new Map(input.chunks.map(chunk => [chunk.id, crypto.createHash('sha256').update(chunk.content).digest('hex')]))
|
||||
const sources = new Map(input.sources.map(source => [source.source_chunk_id, source]))
|
||||
const entities = new Map(input.records.filter(record => record.table === 'knowledgeentity').map(record => [record.id, record]))
|
||||
const relations = input.records.filter(record => record.table === 'knowledgerelation')
|
||||
const states = input.records.filter(record => record.table === 'knowledgegraphchunkstate')
|
||||
const evidence = input.records.filter(record => record.table === 'knowledgegraphevidence')
|
||||
const labels = new Set(input.config.tags)
|
||||
report.counts = { source_chunks: chunks.size, checkpoints: states.length, entities: entities.size,
|
||||
relations: relations.length, evidence_records: evidence.length }
|
||||
check('all_51_product_chunks_are_present', chunks.size === 51)
|
||||
check('all_51_product_chunks_are_checkpointed', states.length === 51 && new Set(states.map(state => state.payload.native_chunk_id)).size === 51)
|
||||
for (const state of states) {
|
||||
const chunk = chunks.get(state.payload.native_chunk_id)
|
||||
check(`checkpoint_current:${state.payload.native_chunk_id}`, chunk
|
||||
&& state.payload.native_knowledge_id === chunk.knowledge_id
|
||||
&& state.payload.source_content_hash === hashes.get(chunk.id)
|
||||
&& state.payload.extraction_status === 'complete')
|
||||
}
|
||||
const relationEvidence = new Map()
|
||||
for (const record of evidence) {
|
||||
const item = record.payload, source = sources.get(item.source_chunk_id), chunk = chunks.get(source?.native_chunk_id)
|
||||
check(`evidence_grounded:${record.id}`, chunk && source.mapping_state === 'native_extraction'
|
||||
&& source.native_knowledge_id === chunk.knowledge_id && item.native_knowledge_id === chunk.knowledge_id
|
||||
&& item.source_content_hash === hashes.get(chunk.id) && source.source_content_hash === hashes.get(chunk.id)
|
||||
&& source.native_content_sha256 === hashes.get(chunk.id)
|
||||
&& typeof item.excerpt === 'string' && item.excerpt.trim().length > 0 && chunk.content.includes(item.excerpt))
|
||||
if (item.subject_type === 'relation') {
|
||||
if (!relationEvidence.has(item.subject_id)) relationEvidence.set(item.subject_id, [])
|
||||
relationEvidence.get(item.subject_id).push({ evidence_id: record.id, chunk_id: chunk?.id, quote: item.excerpt })
|
||||
}
|
||||
}
|
||||
report.relations = relations.map(record => {
|
||||
const item = record.payload, quotations = relationEvidence.get(record.id) || []
|
||||
const from = entities.get(item.from_entity_id), to = entities.get(item.to_entity_id)
|
||||
check(`relation_has_grounded_quote:${record.id}`, from && to && labels.has(item.relation_type)
|
||||
&& typeof item.evidence === 'string' && item.evidence.trim().length > 0
|
||||
&& quotations.length > 0 && quotations.some(quotation => quotation.quote === item.evidence))
|
||||
return { id: record.id, from_id: item.from_entity_id, to_id: item.to_entity_id,
|
||||
from: from?.payload.canonical_name, relation: item.relation_type,
|
||||
to: to?.payload.canonical_name, evidence: quotations }
|
||||
})
|
||||
const relationChunks = new Set(report.relations.flatMap(relation => relation.evidence.map(item => item.chunk_id)))
|
||||
report.counts.chunks_with_relations = relationChunks.size
|
||||
report.counts.checkpointed_without_relations = states.filter(state => !relationChunks.has(state.payload.native_chunk_id)).length
|
||||
report.counts.unresolved_chunks = [...chunks.keys()].filter(id => !relationChunks.has(id)).length
|
||||
for (const state of states) {
|
||||
check(`completed_checkpoint_has_relations:${state.payload.native_chunk_id}`, relationChunks.has(state.payload.native_chunk_id))
|
||||
}
|
||||
const problemRelations = report.relations.filter(relation => relation.evidence.some(item => item.chunk_id === problemChunk))
|
||||
const expectedRestrictions = [
|
||||
['过敏者', ['慎用于']],
|
||||
['粘膜破损处', ['禁用于', '不适用于', '不能用于']],
|
||||
['性生活中对性病的预防', ['禁用于', '不适用于', '不能用于']],
|
||||
]
|
||||
for (const [target, allowed] of expectedRestrictions) {
|
||||
const candidates = problemRelations.filter(relation => relation.to === target || relation.from === target)
|
||||
check(`known_restriction_preserved:${target}`, candidates.length > 0 && candidates.every(relation =>
|
||||
relation.from === '本品' && relation.to === target && allowed.includes(relation.relation)))
|
||||
}
|
||||
const compact = value => value.replace(/\s+/g, '')
|
||||
const quantities = value => new Set((compact(value).toLowerCase().match(/[0-9]+(?:[..·][0-9]+)*(?:ppm|ppb|kg|mg|ug|ml|cm|mm|min|g|l|%|%|毫克|千克|克|毫升|升|分钟|小时|天|颗|粒|瓶|支|片)?/g) || [])
|
||||
.map(value => value.replace(/[.·]/g, '.').replace(/%/g, '%')))
|
||||
const quantityGrounded = (endpoint, source) => [...quantities(endpoint)].every(value => quantities(source).has(value))
|
||||
const prohibition = /禁|不得|不能|不可|不宜|不应|不建议|不适用|不适合|不用于|不具有|不具备|避开|请勿|勿用|停用|停止使用/
|
||||
const affirmative = new Set(['适用于', '适合', '用于', '可用于', '条件适用于'])
|
||||
const genericEntities = new Map()
|
||||
for (const relation of report.relations) {
|
||||
const endpoints = compact(relation.from + relation.to)
|
||||
for (const [id, name] of [[relation.from_id, relation.from], [relation.to_id, relation.to]]) {
|
||||
const localName = compact(name).toLowerCase()
|
||||
if (!['本品', '该产品', '本产品', '产品', '本商品', '此产品', '本公司', '该公司', '本文', '本方法', '该方法', 'thisproduct', 'theproduct'].includes(localName)
|
||||
&& !localName.endsWith('未名产品') && !localName.startsWith('本品(')) continue
|
||||
if (!genericEntities.has(id)) genericEntities.set(id, new Set())
|
||||
for (const item of relation.evidence) genericEntities.get(id).add(item.chunk_id)
|
||||
}
|
||||
for (const item of relation.evidence) {
|
||||
const quote = compact(item.quote)
|
||||
const source = chunks.get(item.chunk_id)?.content || ''
|
||||
check(`endpoint_quantities_grounded:${relation.id}:${item.chunk_id}`,
|
||||
quantityGrounded(relation.from, source) && quantityGrounded(relation.to, item.quote))
|
||||
if (relation.relation === '抑制') {
|
||||
const target = compact(relation.to)
|
||||
check(`suppression_has_target_predicate:${relation.id}:${item.chunk_id}`, quote.includes(target)
|
||||
&& /抑制|杀灭|杀菌|抑菌|抗菌/.test(quote.split(target).join('')))
|
||||
}
|
||||
if (affirmative.has(relation.relation)) {
|
||||
check(`suitability_has_use_claim:${relation.id}:${item.chunk_id}`, /适用|适合|用于|可用|使用|涂于|涂抹|推入|推进|擦拭|清洁|饮用|服用|外用|内服|口服/.test(quote))
|
||||
check(`storage_not_suitability:${relation.id}:${item.chunk_id}`, !/贮存|贮藏|储存|储藏|保存|存放/.test(relation.to)
|
||||
&& !(/阴凉|干燥处|冷藏|常温|避光|遮光|密封/.test(relation.to) && /贮存|贮藏|储存|储藏|保存|存放/.test(quote)))
|
||||
}
|
||||
if (['禁用于', '不适用于', '不能用于'].includes(relation.relation)) {
|
||||
check(`storage_not_usage_prohibition:${relation.id}:${item.chunk_id}`,
|
||||
!(/阳光|直晒|阴凉|干燥|冷藏|常温|避光|遮光|密封/.test(relation.to)
|
||||
&& /贮存|贮藏|储存|储藏|保存|存放/.test(quote)))
|
||||
}
|
||||
if (relation.relation === '约束') {
|
||||
if (/避免.{0,16}(?:同用|合用)|不[宜可得能].{0,16}(?:同用|合用)/.test(quote) && /同用|合用|拮抗/.test(endpoints)) {
|
||||
check(`constraint_preserves_combination_prohibition:${relation.id}:${item.chunk_id}`, /避免|禁止|不得|不能|不可|不宜|不要|勿/.test(endpoints))
|
||||
}
|
||||
if (/停止使用|立即停用/.test(quote) && /不适|使用|停用/.test(endpoints)) {
|
||||
check(`constraint_preserves_stop_action:${relation.id}:${item.chunk_id}`, /停止使用|立即停用/.test(endpoints))
|
||||
}
|
||||
if (/儿童不[易宜能]触及|儿童触及不到/.test(quote) && /儿童/.test(endpoints)) {
|
||||
check(`constraint_preserves_child_storage_action:${relation.id}:${item.chunk_id}`, /不[易宜能]触及|触及不到|远离|避免/.test(endpoints))
|
||||
}
|
||||
}
|
||||
const nameAt = source.indexOf(relation.from), quoteAt = source.indexOf(item.quote)
|
||||
if (nameAt >= 0 && quoteAt >= 0 && quoteAt + item.quote.length < nameAt && !source.slice(nameAt).includes(item.quote)) {
|
||||
const boundary = source.lastIndexOf('\n\n', nameAt)
|
||||
check(`no_later_product_attribution:${relation.id}:${item.chunk_id}`, boundary < quoteAt + item.quote.length
|
||||
|| !/产品名称|品名/.test(source.slice(boundary)))
|
||||
}
|
||||
if (item.chunk_id === '3626de4e-88a2-41e4-8e09-05c51f23848a' && /避开女性崆期|产期/.test(quote)) {
|
||||
check(`avoidance_not_permission:${relation.id}`, !affirmative.has(relation.relation)
|
||||
&& relation.relation !== '慎用于')
|
||||
if (['禁用于', '不适用于', '不能用于', '约束'].includes(relation.relation)) {
|
||||
check(`vaginal_restriction_scope_preserved:${relation.id}`, endpoints.includes('阴道'))
|
||||
}
|
||||
}
|
||||
if (quote.includes('哺乳期请勿在乳头使用') && /哺乳期|乳头/.test(endpoints)
|
||||
&& ['禁用于', '不适用于', '不能用于', '约束'].includes(relation.relation)) {
|
||||
check(`lactation_restriction_scope_preserved:${relation.id}`, endpoints.includes('哺乳期') && endpoints.includes('乳头'))
|
||||
}
|
||||
if (/抑制|杀灭|抑菌|杀菌|抗菌/.test(quote) && /菌$|病毒$|微生物$/.test(compact(relation.to))
|
||||
&& !/抑制|杀灭|抑菌|杀菌|抗菌/.test(compact(relation.to))) {
|
||||
check(`suppression_not_support:${relation.id}`, !['支持', '促进', '增强'].includes(relation.relation))
|
||||
}
|
||||
if (/告别冷淡|改善粗糙与干燥/.test(quote) && !prohibition.test(quote)) {
|
||||
check(`benefit_not_prohibition:${relation.id}`, !['禁用于', '不适用于', '不能用于', '慎用于', '不支持', '不导致'].includes(relation.relation))
|
||||
}
|
||||
if (quote.includes('停止使用') && relation.to.includes('不适')) {
|
||||
check(`stop_use_not_caution:${relation.id}`, !affirmative.has(relation.relation) && relation.relation !== '慎用于')
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const [id, chunkIDs] of genericEntities) {
|
||||
check(`generic_product_is_chunk_scoped:${id}`, chunkIDs.size <= 1)
|
||||
}
|
||||
report.problem_chunk_relations = problemRelations
|
||||
report.chunks = input.chunks.map(chunk => ({ ...chunk, sha256: hashes.get(chunk.id),
|
||||
relations: report.relations.filter(relation => relation.evidence.some(item => item.chunk_id === chunk.id)) }))
|
||||
report.semantic_review_note = 'Automated checks cover exact-source evidence, original contraindications, numeric endpoints, product attribution and generic-entity scope, explicit suitability versus storage, and demonstrated polarity reversals. Full semantic correctness still requires reviewing each listed relation with its source chunk.'
|
||||
} catch (error) {
|
||||
report.failures.push(error.message)
|
||||
}
|
||||
report.passed = report.failures.length === 0
|
||||
fs.writeFileSync(outputPath, JSON.stringify(report, null, 2) + '\n')
|
||||
console.log(JSON.stringify({ checked_at: report.checked_at, production_write: report.production_write,
|
||||
passed: report.passed, counts: report.counts, failures: report.failures, report_path: outputPath }))
|
||||
if (!report.passed) process.exitCode = 1
|
||||
@@ -0,0 +1,66 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const assert = require('node:assert/strict')
|
||||
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
|
||||
const directory = path.resolve(__dirname, '../../acceptance')
|
||||
const origin = 'https://agent.pducv.cn'
|
||||
const checks = [], failures = [], snapshots = []
|
||||
let auth, browser
|
||||
function check(name, value) { checks.push({name, passed: !!value}); assert.ok(value, name) }
|
||||
async function api(route, method = 'GET', body) {
|
||||
const response = await fetch(origin + '/api/v1' + route, {
|
||||
method, headers: {'Content-Type': 'application/json', ...(auth ? {Authorization: `Bearer ${auth.token}`, 'X-Tenant-ID': '10059'} : {})},
|
||||
body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(45000),
|
||||
})
|
||||
assert.equal(response.status, 200, route)
|
||||
return response.json()
|
||||
}
|
||||
async function main() {
|
||||
const credentials = JSON.parse(fs.readFileSync(path.join(directory, 'super-admin-login.json'), 'utf8'))
|
||||
auth = await api('/auth/login', 'POST', {username: credentials.username, password: credentials.password})
|
||||
check('admin_login_opens_business_workspace', Number(auth.active_tenant.id) === 10059)
|
||||
const baseline = JSON.parse(fs.readFileSync(path.join(directory, 'business-wiki-backfill.json'), 'utf8'))
|
||||
for (const kb of baseline.before) {
|
||||
const current = (await api(`/knowledge-bases/${kb.id}`)).data
|
||||
check(`${kb.name}:rag_and_wiki_enabled`, current.indexing_strategy.wiki_enabled && current.indexing_strategy.vector_enabled && current.indexing_strategy.keyword_enabled)
|
||||
const stats = await api(`/knowledgebase/${kb.id}/wiki/stats`)
|
||||
const pages = await api(`/knowledgebase/${kb.id}/wiki/pages?page=1&page_size=5`)
|
||||
check(`${kb.name}:wiki_readable`, typeof stats.total_pages === 'number' && Array.isArray(pages.pages))
|
||||
snapshots.push({id: kb.id, name: kb.name, pages: stats.total_pages, pending_tasks: stats.pending_tasks, active: stats.is_active})
|
||||
}
|
||||
check('wiki_pages_actually_generated', snapshots.some(s => s.pages > 0))
|
||||
const product = baseline.before.find(k => k.name === '\u4ea7\u54c1\u4fe1\u606f')
|
||||
const start = Date.now()
|
||||
const hits = (await api(`/knowledge-bases/${product.id}/hybrid-search`, 'POST', {
|
||||
query_text: '\u4ea7\u54c1\u4fe1\u606f', match_count: 5, vector_threshold: 0, keyword_threshold: 0,
|
||||
})).data
|
||||
check('existing_rag_still_returns_source_chunks', Array.isArray(hits) && hits.some(hit => hit.knowledge_id))
|
||||
snapshots.push({rag_query_ms: Date.now() - start, rag_hits: hits.length})
|
||||
browser = await chromium.launch({headless: true, channel: 'chrome'})
|
||||
const page = await browser.newPage({viewport: {width: 1440, height: 1000}})
|
||||
await page.addInitScript(a => {
|
||||
const values = {weknora_user: JSON.stringify(a.user), weknora_tenant: JSON.stringify(a.active_tenant), weknora_token: a.token, weknora_refresh_token: a.refresh_token, weknora_memberships: JSON.stringify(a.memberships)}
|
||||
Object.entries(values).forEach(([k,v]) => localStorage.setItem(k,v))
|
||||
localStorage.setItem('weknora:new-user-guide-done:v1', '1')
|
||||
}, auth)
|
||||
const errors = []
|
||||
page.on('pageerror', e => errors.push(e.message))
|
||||
page.on('response', response => {if(response.status() >= 400 && /\/(api|assets)\//.test(new URL(response.url()).pathname)) errors.push(`${response.status()} ${new URL(response.url()).pathname}`)})
|
||||
await page.goto(`${origin}/platform/knowledge-bases/${product.id}?tab=wiki`)
|
||||
await page.locator('.wiki-browser').waitFor({timeout: 30000})
|
||||
await page.locator('.wiki-tab').first().waitFor({timeout: 30000})
|
||||
await page.locator('.wiki-directory-item, .wiki-page-item').first().waitFor({timeout: 30000})
|
||||
check('generated_wiki_entries_visible', await page.locator('.wiki-tab-count').first().innerText() !== '0')
|
||||
await page.screenshot({path: path.join(directory, 'business-wiki-desktop.png'), animations: 'disabled'})
|
||||
check('public_wiki_browser_visible', await page.locator('.wiki-browser').isVisible())
|
||||
await page.setViewportSize({width: 390, height: 844})
|
||||
await page.screenshot({path: path.join(directory, 'business-wiki-mobile.png'), animations: 'disabled'})
|
||||
check('wiki_no_horizontal_overflow', await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 2))
|
||||
check('wiki_no_failed_requests_or_page_errors', errors.length === 0)
|
||||
}
|
||||
main().catch(e => {failures.push(e.message); process.exitCode = 1}).finally(async () => {
|
||||
if (browser) await browser.close()
|
||||
const report = {checked_at: new Date().toISOString(), origin, checks, snapshots, failures, passed: failures.length === 0}
|
||||
fs.writeFileSync(path.join(directory, 'business-wiki-verification.json'), JSON.stringify(report, null, 2))
|
||||
console.log(JSON.stringify(report))
|
||||
})
|
||||
@@ -0,0 +1,319 @@
|
||||
// Creates disposable fixtures only. No generation actions or paid model calls.
|
||||
// VERIFY_API_ORIGIN / VERIFY_FRONTEND_ORIGIN default to the public deployment.
|
||||
// CANVAS_BROWSER=0 runs API checks only. PostgreSQL must back the selected API.
|
||||
const assert = require('node:assert/strict')
|
||||
const crypto = require('node:crypto')
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
|
||||
const root = path.resolve(__dirname, '../..')
|
||||
const directory = path.join(root, 'acceptance')
|
||||
const origin = (process.env.VERIFY_API_ORIGIN || 'https://agent.pducv.cn').replace(/\/$/, '')
|
||||
const frontend = (process.env.VERIFY_FRONTEND_ORIGIN || origin).replace(/\/$/, '')
|
||||
const reportName = process.env.VERIFY_REPORT_NAME || 'canvas-configuration-live'
|
||||
const tenant = 10059
|
||||
const database = 'yawan_native_acceptance_20260915'
|
||||
const prefix = `canvas-check-${Date.now()}-${crypto.randomBytes(3).toString('hex')}`
|
||||
const checks = [], failures = [], resources = []
|
||||
const knowledgeKeys = ['knowledge_base_ids', 'knowledge_ids', 'knowledge_folders', 'knowledge_tags']
|
||||
const browserEnabled = process.env.CANVAS_BROWSER !== '0'
|
||||
let admin, member, role, user, agent, kb, browser, foreignTenant, databaseVerified = false
|
||||
assert.match(reportName, /^[a-zA-Z0-9_-]+$/)
|
||||
const literal = value => `'${String(value).replaceAll("'", "''")}'`
|
||||
const sql = query => execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres',
|
||||
'psql', '-X', '-q', '-U', 'postgres', '-d', database, '-At', '-v', 'ON_ERROR_STOP=1'],
|
||||
{ input: query, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim()
|
||||
function check(name, value) {
|
||||
checks.push({ name, passed: !!value })
|
||||
assert.ok(value, name)
|
||||
console.log(`PASS ${name}`)
|
||||
}
|
||||
async function api(route, method = 'GET', body, identity = admin, expected = 200) {
|
||||
const response = await fetch(origin + '/api/v1' + route, { method, headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...(identity ? { Authorization: `Bearer ${identity.token}`, 'X-Tenant-ID': String(tenant) } : {}),
|
||||
}, body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(30000) })
|
||||
const data = await response.json()
|
||||
assert.ok((Array.isArray(expected) ? expected : [expected]).includes(response.status),
|
||||
`${method} ${route}: HTTP ${response.status}; ${data.message || data.error?.message || ''}`)
|
||||
return data
|
||||
}
|
||||
function resourcePath(resource) {
|
||||
assert.ok(resources.some(row => row.id === resource.id && row.kind === resource.kind), 'Only recorded fixtures may be changed')
|
||||
return `/workbench/resources/${resource.kind}/${resource.id}`
|
||||
}
|
||||
async function create(kind, label, data, identity = member) {
|
||||
const resource = (await api(`/workbench/resources/${kind}`, 'POST', { name: `${prefix}-${label}`, data }, identity, 201)).data
|
||||
resources.push({ ...resource, identity })
|
||||
assert.equal(resource.owner_id, identity.user.id)
|
||||
return resource
|
||||
}
|
||||
function fixtureWhere(resource) {
|
||||
resourcePath(resource)
|
||||
assert.ok(resource.name.startsWith(prefix + '-'))
|
||||
return `id=${literal(resource.id)} AND tenant_id=${Number(resource.tenant_id)} AND kind=${literal(resource.kind)} AND name LIKE ${literal(prefix + '-%')}`
|
||||
}
|
||||
function seed(resource, data, ownerID = resource.owner_id, status = resource.status) {
|
||||
assert.ok(databaseVerified, 'API and database identity must match before seeding')
|
||||
const id = sql(`UPDATE workbench_resources SET data=${literal(JSON.stringify(data))}::jsonb,
|
||||
owner_id=${literal(ownerID)},status=${literal(status)} WHERE ${fixtureWhere(resource)} RETURNING id;`)
|
||||
assert.equal(id, resource.id, 'Exactly the recorded fixture must be seeded')
|
||||
}
|
||||
async function read(resource, identity = admin) {
|
||||
return (await api(resourcePath(resource), 'GET', undefined, identity)).data
|
||||
}
|
||||
async function absent(resource, identity = admin) {
|
||||
await api(resourcePath(resource), 'GET', undefined, identity, 404)
|
||||
const listed = await api(`/workbench/resources/${resource.kind}?q=${encodeURIComponent(prefix)}&page_size=100`, 'GET', undefined, identity)
|
||||
assert.ok(!listed.data.some(row => row.id === resource.id), 'Deleted fixture must not reappear in a fresh list request')
|
||||
}
|
||||
function scopeIsEmpty(data, keys = knowledgeKeys) {
|
||||
return keys.every(key => data[key] == null || Object.keys(data[key]).length === 0)
|
||||
}
|
||||
function manuscript(task) {
|
||||
return { ...task.data, segments: [
|
||||
{ id: 'accepted', title: 'Accepted fixture', agent_id: agent.id, status: 'accepted', content: 'Keep this accepted fixture prose.' },
|
||||
{ id: 'skipped', title: 'Skipped fixture', agent_id: agent.id, status: 'skipped', content: 'Keep this skipped fixture prose.' },
|
||||
], review: 'Keep the fixture review.', composed: 'Keep the fixture final manuscript.',
|
||||
legacy_history: { fixture: prefix, opaque: { retained: true } }, custom_fixture_field: ['preserve', 123] }
|
||||
}
|
||||
async function checkDetached(before, actor) {
|
||||
const after = await read(before)
|
||||
assert.deepEqual(after.data, { ...before.data, flow_id: '' }, 'Detachment must preserve all other fields, including prose and accepted states')
|
||||
assert.equal(after.owner_id, before.owner_id)
|
||||
assert.equal(after.version, before.version + 1)
|
||||
assert.equal(after.status, 'needs_configuration')
|
||||
const revisions = JSON.parse(sql(`SELECT coalesce(json_agg(r ORDER BY version),'[]') FROM workbench_revisions r
|
||||
WHERE tenant_id=${tenant} AND resource_id=${literal(before.id)} AND version IN (${before.version},${after.version});`))
|
||||
const next = revisions.find(row => row.version === after.version)
|
||||
assert.ok(next && next.actor_id === actor.user.id, 'Detachment revision must record the actual actor')
|
||||
assert.deepEqual(next.data, after.data)
|
||||
return after
|
||||
}
|
||||
async function browserPage(identity, resource) {
|
||||
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } })
|
||||
await context.addInitScript(({ auth, tenantID }) => {
|
||||
for (const [key, value] of Object.entries({
|
||||
weknora_user: JSON.stringify({ ...auth.user, tenant_id: String(auth.user.tenant_id) }),
|
||||
weknora_tenant: JSON.stringify({ ...auth.active_tenant, id: String(tenantID) }),
|
||||
weknora_token: auth.token, weknora_refresh_token: auth.refresh_token,
|
||||
weknora_memberships: JSON.stringify(auth.memberships), 'weknora:new-user-guide-done:v1': '1',
|
||||
})) localStorage.setItem(key, value)
|
||||
}, { auth: identity, tenantID: tenant })
|
||||
const page = await context.newPage()
|
||||
page.setDefaultTimeout(20000)
|
||||
page.on('pageerror', error => failures.push(`browser: ${error.message}`))
|
||||
await page.route('**/api/v1/workbench/resources/**', async route => {
|
||||
const request = route.request()
|
||||
if (!['GET', 'HEAD'].includes(request.method())) {
|
||||
const pathname = new URL(request.url()).pathname
|
||||
const allowed = resources.some(row => pathname === '/api/v1' + resourcePath(row))
|
||||
if (!allowed) {
|
||||
failures.push(`blocked browser mutation outside recorded fixtures: ${pathname}`)
|
||||
return route.abort()
|
||||
}
|
||||
}
|
||||
return route.continue()
|
||||
})
|
||||
await page.goto(frontend + '/platform/workbench/flows')
|
||||
const row = page.locator('.workbench-resource').filter({ hasText: resource.name })
|
||||
await row.waitFor()
|
||||
await row.click()
|
||||
await page.getByRole('button', { name: '\u5220\u9664\u753b\u5e03', exact: true }).waitFor()
|
||||
return { page, context }
|
||||
}
|
||||
async function deleteInBrowser(page, resource) {
|
||||
const [response] = await Promise.all([
|
||||
page.waitForResponse(value => value.request().method() === 'DELETE' &&
|
||||
new URL(value.url()).pathname === '/api/v1' + resourcePath(resource)),
|
||||
(async () => {
|
||||
await page.getByRole('button', { name: '\u5220\u9664\u753b\u5e03', exact: true }).click()
|
||||
await page.locator('.t-popconfirm:visible').getByRole('button', { name: /^(\u786e\u5b9a|\u786e\u8ba4)$/ }).click()
|
||||
})(),
|
||||
])
|
||||
assert.equal(response.status(), 200)
|
||||
const row = page.locator('.workbench-resource').filter({ hasText: resource.name })
|
||||
await row.waitFor({ state: 'hidden' })
|
||||
const [reloaded] = await Promise.all([
|
||||
page.waitForResponse(value => value.request().method() === 'GET' &&
|
||||
new URL(value.url()).pathname === '/api/v1/workbench/resources/flows'),
|
||||
page.reload(),
|
||||
])
|
||||
assert.equal(reloaded.status(), 200)
|
||||
await page.getByRole('heading', { name: '\u753b\u5e03\u7f16\u6392', exact: true }).waitFor()
|
||||
await page.waitForLoadState('networkidle', { timeout: 30000 })
|
||||
assert.equal(await row.count(), 0)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const credentials = JSON.parse(fs.readFileSync(path.join(directory, 'super-admin-login.json'), 'utf8'))
|
||||
admin = await api('/auth/login', 'POST', { email: credentials.email, password: credentials.password }, null)
|
||||
check('same_tenant_administrator_available', (await api('/access/me')).data.administrator === true)
|
||||
role = (await api('/access/roles', 'POST', { name: prefix, permissions: [
|
||||
'flows.read', 'flows.write', 'content.read', 'content.write', 'agents.read', 'agents.write', 'knowledge.read', 'knowledge.write',
|
||||
] })).data
|
||||
check('api_uses_expected_fixture_database', sql(`SELECT count(*) FROM workbench_access_roles
|
||||
WHERE tenant_id=${tenant} AND id=${literal(role.id)} AND name=${literal(prefix)}`) === '1')
|
||||
databaseVerified = true
|
||||
const password = crypto.randomBytes(18).toString('base64url')
|
||||
const email = `${prefix}@example.test`
|
||||
user = (await api('/access/users', 'POST', { username: prefix, email, password, role: 'contributor', status: 'active', custom_role_id: role.id })).data
|
||||
member = await api('/auth/login', 'POST', { email, password }, null)
|
||||
check('fixture_member_is_not_an_administrator', (await api('/access/me', 'GET', undefined, member)).data.administrator === false)
|
||||
const source = (await api('/agents')).data.find(row => row.config?.model_id)
|
||||
assert.ok(source, 'An existing configured model ID is needed for the Agent selector; it will not be invoked')
|
||||
kb = (await api('/knowledge-bases', 'POST', { name: prefix + '-knowledge', type: 'document',
|
||||
indexing_strategy: { vector_enabled: false, keyword_enabled: false, wiki_enabled: false, graph_enabled: false },
|
||||
}, member, [200, 201])).data
|
||||
agent = (await api('/agents', 'POST', { name: prefix + '-agent', config: {
|
||||
role: 'production', agent_mode: 'quick-answer', model_id: source.config.model_id,
|
||||
kb_selection_mode: 'selected', knowledge_bases: [kb.id], system_prompt: 'Disposable canvas fixture. No generation requested.',
|
||||
fusion_agent_ids: [], instruction_skill_ids: [],
|
||||
} }, member, 201)).data
|
||||
const staleKB = crypto.randomUUID()
|
||||
const stale = { knowledge_base_ids: [staleKB], knowledge_ids: [crypto.randomUUID()],
|
||||
knowledge_folders: { [staleKB]: ['removed-folder'] }, knowledge_tags: { [staleKB]: [crypto.randomUUID()] } }
|
||||
const flowData = { nodes: [{ id: 'writer', label: 'Fixture writer', agent_id: agent.id }], edges: [], ...stale }
|
||||
const own = await create('flows', 'own', flowData)
|
||||
check('creating_canvas_removes_all_four_legacy_knowledge_fields', knowledgeKeys.every(key => !Object.hasOwn(own.data, key)))
|
||||
seed(own, { ...own.data, ...stale })
|
||||
const historicalScope = (await read(own, member)).data
|
||||
assert.deepEqual(Object.fromEntries(knowledgeKeys.map(key => [key, historicalScope[key]])), stale)
|
||||
let ownTask = await create('content-tasks', 'own-task', { source_text: 'Disposable outline', flow_id: own.id, ...stale })
|
||||
assert.deepEqual(ownTask.data.knowledge_base_ids, [kb.id])
|
||||
assert.ok(scopeIsEmpty(ownTask.data, knowledgeKeys.slice(1)))
|
||||
assert.ok(ownTask.data.flow_snapshot && scopeIsEmpty(ownTask.data.flow_snapshot))
|
||||
assert.equal(ownTask.data.agent_id, agent.id)
|
||||
check('stale_canvas_and_task_scopes_are_replaced_by_node_agent_scope', true)
|
||||
seed(ownTask, manuscript(ownTask))
|
||||
ownTask = await read(ownTask, member)
|
||||
await api(resourcePath(own), 'DELETE', undefined, member, 409)
|
||||
if (browserEnabled) {
|
||||
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
|
||||
browser = await chromium.launch({ headless: true, channel: 'chrome' })
|
||||
const { page, context } = await browserPage(member, own)
|
||||
check('canvas_has_no_knowledge_selector', await page.locator('.knowledge-scope').count() === 0 &&
|
||||
!/\u753b\u5e03\u77e5\u8bc6\u8303\u56f4/.test(await page.locator('.workbench-detail').innerText()))
|
||||
const [saved] = await Promise.all([
|
||||
page.waitForResponse(value => value.request().method() === 'PUT' && new URL(value.url()).pathname === '/api/v1' + resourcePath(own)),
|
||||
page.getByRole('button', { name: '\u4fdd\u5b58', exact: true }).click(),
|
||||
])
|
||||
assert.equal(saved.status(), 200)
|
||||
const savedData = (await read(own, member)).data
|
||||
check('saving_canvas_removes_all_four_legacy_knowledge_fields', knowledgeKeys.every(key => !Object.hasOwn(savedData, key)))
|
||||
await page.setViewportSize({ width: 390, height: 844 })
|
||||
await page.screenshot({ path: path.join(directory, `${reportName}-mobile.png`), animations: 'disabled' })
|
||||
check('canvas_mobile_has_no_horizontal_overflow', await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 2))
|
||||
await page.setViewportSize({ width: 1440, height: 1000 })
|
||||
await deleteInBrowser(page, own)
|
||||
check('owner_delete_survives_browser_reload', true)
|
||||
await context.close()
|
||||
} else {
|
||||
await api(resourcePath(own) + '?detach=true', 'DELETE', undefined, member)
|
||||
}
|
||||
await absent(own, member)
|
||||
await checkDetached(ownTask, member)
|
||||
check('owner_delete_detaches_safely_and_does_not_reappear', true)
|
||||
|
||||
const old = await create('flows', 'historical', flowData)
|
||||
let oldTask = await create('content-tasks', 'historical-task', { source_text: 'Disposable historical outline', flow_id: old.id })
|
||||
const blocker = await create('content-tasks', 'other-owner-reference', { source_text: 'Disposable blocker', flow_id: old.id })
|
||||
seed(blocker, blocker.data, admin.user.id)
|
||||
const blockerBefore = await read(blocker)
|
||||
assert.equal(blockerBefore.owner_id, admin.user.id)
|
||||
const orphanOwner = crypto.randomUUID()
|
||||
assert.equal(sql(`SELECT count(*) FROM users WHERE id=${literal(orphanOwner)}`), '0')
|
||||
seed(old, { ...old.data, legacy_import: { read_only: true, fixture: prefix } }, orphanOwner, 'needs_configuration')
|
||||
seed(oldTask, manuscript(oldTask), orphanOwner, 'completed')
|
||||
const historical = await read(old)
|
||||
oldTask = await read(oldTask)
|
||||
await api(resourcePath(old), 'PUT', { name: old.name + '-forbidden', data: historical.data, version: historical.version }, admin, [403, 404])
|
||||
await api(resourcePath(old), 'DELETE', undefined, member, [403, 404])
|
||||
await api(resourcePath(oldTask), 'DELETE', undefined, admin, [403, 404])
|
||||
assert.deepEqual(await read(old), historical)
|
||||
check('administrator_delete_permission_does_not_allow_editing_or_task_deletion', true)
|
||||
|
||||
await api(resourcePath(old) + '?detach=true', 'DELETE', undefined, admin, 409)
|
||||
assert.deepEqual(await read(oldTask), oldTask)
|
||||
assert.deepEqual(await read(blocker), blockerBefore)
|
||||
check('another_owners_reference_blocks_deletion_without_changing_tasks', true)
|
||||
await api(resourcePath(blocker), 'DELETE')
|
||||
await api(resourcePath(old), 'DELETE', undefined, admin, 409)
|
||||
if (browserEnabled) {
|
||||
const { page, context } = await browserPage(admin, old)
|
||||
check('administrator_sees_delete_but_not_save_for_historical_canvas',
|
||||
await page.getByRole('button', { name: '\u5220\u9664\u753b\u5e03', exact: true }).isEnabled() &&
|
||||
await page.getByRole('button', { name: '\u4fdd\u5b58', exact: true }).count() === 0)
|
||||
await page.screenshot({ path: path.join(directory, `${reportName}-admin.png`), animations: 'disabled' })
|
||||
await deleteInBrowser(page, old)
|
||||
await context.close()
|
||||
} else {
|
||||
await api(resourcePath(old) + '?detach=true', 'DELETE')
|
||||
}
|
||||
await absent(old)
|
||||
await checkDetached(oldTask, admin)
|
||||
assert.equal(sql(`SELECT count(*) FROM workbench_events WHERE tenant_id=${tenant} AND resource_id=${literal(old.id)}
|
||||
AND action='delete' AND owner_id=${literal(admin.user.id)}`), '1')
|
||||
check('administrator_deletes_orphaned_canvas_preserving_manuscript_and_audit_actor', true)
|
||||
|
||||
foreignTenant = Number(sql(`INSERT INTO tenants(name,business,status)
|
||||
VALUES (${literal(prefix)},'canvas-fixture','active') RETURNING id;`))
|
||||
assert.ok(Number.isSafeInteger(foreignTenant) && foreignTenant > 0 && foreignTenant !== tenant)
|
||||
const foreign = { id: crypto.randomUUID(), tenant_id: foreignTenant, owner_id: admin.user.id, kind: 'flows', name: prefix + '-foreign' }
|
||||
resources.push(foreign)
|
||||
sql(`INSERT INTO workbench_resources(id,tenant_id,owner_id,owner_name,kind,name,status,version,data,created_at,updated_at)
|
||||
VALUES(${literal(foreign.id)},${foreignTenant},${literal(admin.user.id)},${literal(prefix)},'flows',${literal(foreign.name)},'draft',1,'{}',now(),now());`)
|
||||
await api(resourcePath(foreign), 'GET', undefined, admin, 404)
|
||||
await api(resourcePath(foreign) + '?detach=true', 'DELETE', undefined, admin, 404)
|
||||
check('administrator_cannot_delete_same_owner_canvas_in_another_tenant',
|
||||
sql(`SELECT count(*) FROM workbench_resources WHERE ${fixtureWhere(foreign)}`) === '1')
|
||||
check('no_generation_jobs_or_model_usage', sql(`SELECT
|
||||
(SELECT count(*) FROM workbench_jobs WHERE resource_id IN (${resources.map(row => literal(row.id)).join(',')})) +
|
||||
(SELECT count(*) FROM workbench_usage WHERE tenant_id=${tenant} AND agent_id=${literal(agent.id)})`) === '0')
|
||||
}
|
||||
|
||||
async function cleanup() {
|
||||
async function attempt(label, action) {
|
||||
try { await action() } catch (error) { failures.push(`cleanup ${label}: ${error.message}`) }
|
||||
}
|
||||
await attempt('browser', async () => { await browser?.close() })
|
||||
if (databaseVerified) {
|
||||
for (const resource of [...resources].reverse()) await attempt(resource.id, async () => {
|
||||
assert.equal(sql(`SELECT count(*) FROM workbench_jobs WHERE resource_id=${literal(resource.id)} AND status IN ('queued','running')`), '0')
|
||||
sql(`BEGIN;
|
||||
DELETE FROM workbench_events WHERE tenant_id=${Number(resource.tenant_id)} AND resource_id=${literal(resource.id)};
|
||||
DELETE FROM workbench_revisions WHERE tenant_id=${Number(resource.tenant_id)} AND resource_id=${literal(resource.id)};
|
||||
DELETE FROM workbench_jobs WHERE tenant_id=${Number(resource.tenant_id)} AND resource_id=${literal(resource.id)};
|
||||
DELETE FROM workbench_resources WHERE ${fixtureWhere(resource)};
|
||||
COMMIT;`)
|
||||
})
|
||||
if (foreignTenant) await attempt('foreign tenant', async () => {
|
||||
assert.equal(sql(`SELECT count(*) FROM workbench_resources WHERE tenant_id=${foreignTenant}`), '0')
|
||||
sql(`DELETE FROM tenants WHERE id=${foreignTenant} AND name=${literal(prefix)} AND business='canvas-fixture';`)
|
||||
})
|
||||
}
|
||||
if (agent) await attempt('agent', () => api(`/agents/${agent.id}?detach=true`, 'DELETE', undefined, member, [200, 404]))
|
||||
if (kb) await attempt('knowledge', () => api(`/knowledge-bases/${kb.id}`, 'DELETE', undefined, member, [200, 202, 404]))
|
||||
if (databaseVerified) await attempt('fixture publications', async () => {
|
||||
const ids = [...resources.map(row => row.id), agent?.id, kb?.id].filter(Boolean)
|
||||
if (ids.length) for (const table of ['workbench_access_grants', 'workbench_access_publications']) {
|
||||
sql(`DELETE FROM ${table} WHERE tenant_id=${tenant} AND resource_id IN (${ids.map(literal).join(',')});`)
|
||||
}
|
||||
})
|
||||
if (user) await attempt('user', () => api(`/access/users/${user.id}`, 'DELETE', undefined, admin, [200, 404]))
|
||||
if (role) await attempt('role', () => api(`/access/roles/${role.id}`, 'DELETE', undefined, admin, [200, 404]))
|
||||
if (databaseVerified && resources.length) await attempt('verify resources removed', async () => {
|
||||
check('all_recorded_canvas_and_task_fixtures_removed', sql(`SELECT count(*) FROM workbench_resources
|
||||
WHERE id IN (${resources.map(row => literal(row.id)).join(',')})`) === '0')
|
||||
})
|
||||
}
|
||||
main().catch(error => { failures.push(error.message) }).finally(async () => {
|
||||
await cleanup()
|
||||
const report = { checked_at: new Date().toISOString(), origin, frontend, prefix, browser_enabled: browserEnabled,
|
||||
checks, failures, passed: failures.length === 0, fixtures: { role_id: role?.id, user_id: user?.id, agent_id: agent?.id,
|
||||
knowledge_base_id: kb?.id, foreign_tenant: foreignTenant, resources: resources.map(({ id, kind, tenant_id }) => ({ id, kind, tenant_id })) } }
|
||||
fs.writeFileSync(path.join(directory, `${reportName}.json`), JSON.stringify(report, null, 2), { mode: 0o600 })
|
||||
console.log(JSON.stringify(report, null, 2))
|
||||
if (failures.length) process.exitCode = 1
|
||||
})
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify category migration through authenticated, read-only acceptance APIs."""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from weknora_category_etl import TENANT, save_json
|
||||
|
||||
|
||||
BASE = "http://127.0.0.1:18080"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
parser.add_argument("--credentials", type=Path, required=True)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
manifest = json.loads(args.manifest.read_text())
|
||||
credentials = json.loads(args.credentials.read_text())
|
||||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
def request(path, token=None, payload=None):
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if token:
|
||||
headers["Authorization"] = "Bearer " + token
|
||||
req = urllib.request.Request(BASE + path, data=json.dumps(payload).encode() if payload else None, headers=headers)
|
||||
try:
|
||||
with opener.open(req, timeout=20) as response:
|
||||
return response.status, json.load(response)
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, None
|
||||
|
||||
status, auth = request("/api/v1/auth/login", payload={k: credentials[k] for k in ("email", "password")})
|
||||
if status != 200 or auth.get("active_tenant", {}).get("id") != TENANT:
|
||||
raise RuntimeError("Verification login failed or outside migration tenant")
|
||||
token = auth["token"]
|
||||
report = {"checked_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "tenant_id": TENANT, "checks": []}
|
||||
|
||||
def check(name, passed, **counts):
|
||||
report["checks"].append({"check": name, "passed": bool(passed), **counts})
|
||||
|
||||
for kb in manifest["knowledge_bases"]:
|
||||
files = [f for f in manifest["files"] if f["knowledge_base_id"] == kb]
|
||||
tags = [t for t in manifest["tags"] if t["knowledge_base_id"] == kb]
|
||||
status, response = request(f"/api/v1/knowledge-bases/{kb}/knowledge/folders", token)
|
||||
tree = response.get("data", {}) if response else {}
|
||||
nodes = {}
|
||||
|
||||
def walk(items):
|
||||
for node in items:
|
||||
nodes[node["path"]] = node
|
||||
walk(node.get("children") or [])
|
||||
|
||||
walk(tree.get("folders") or [])
|
||||
counts_match = all(t["folder_path"] in nodes and nodes[t["folder_path"]]["total_count"] == sum(
|
||||
f["folder_path"] == t["folder_path"] or f["folder_path"].startswith(t["folder_path"] + "/") for f in files) for t in tags)
|
||||
check("folder_tree", status == 200 and tree.get("total_document_count") == len(files) and counts_match,
|
||||
knowledge_base_id=kb, folders=len(nodes), documents=tree.get("total_document_count"))
|
||||
status, response = request(f"/api/v1/knowledge-bases/{kb}/tags?page=1&page_size=100", token)
|
||||
data = response.get("data", {}) if response else {}
|
||||
items = data.get("data", [])
|
||||
check("tag_list", status == 200 and {t["id"] for t in items} == {t["id"] for t in tags},
|
||||
knowledge_base_id=kb, tags=len(items))
|
||||
for tag in tags:
|
||||
params = urllib.parse.urlencode({"page": 1, "page_size": 100, "tag_ids": tag["id"]})
|
||||
status, response = request(f"/api/v1/knowledge-bases/{kb}/knowledge?{params}", token)
|
||||
data = response.get("data", []) if response else []
|
||||
if isinstance(data, dict):
|
||||
data = data.get("items", [])
|
||||
expected = {r["knowledge_id"] for r in manifest["relations"] if r["tag_id"] == tag["id"]}
|
||||
actual = {f["id"] for f in data}
|
||||
check("tag_document_scope", status == 200 and actual == expected,
|
||||
legacy_category_id=tag["legacy_id"], expected=len(expected), returned=len(actual))
|
||||
status, _ = request(f"/api/v1/knowledge-bases/{kb}/knowledge/folders")
|
||||
check("anonymous_denied", status in (401, 403), knowledge_base_id=kb)
|
||||
report["passed"] = all(c["passed"] for c in report["checks"])
|
||||
save_json(args.out, report)
|
||||
print(json.dumps({"passed": report["passed"], "checks": len(report["checks"]),
|
||||
"failed": [c for c in report["checks"] if not c["passed"]], "out": str(args.out)}))
|
||||
return 0 if report["passed"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,103 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const { chromium } = require('/Users/macclaw/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules/playwright')
|
||||
const directory = '/Volumes/Extreme SSD/yawan-storage/weknora/acceptance'
|
||||
const origin = process.env.VERIFY_FRONTEND_ORIGIN || 'http://127.0.0.1:15173'
|
||||
const apiOrigin = process.env.VERIFY_API_ORIGIN || 'http://127.0.0.1:18080'
|
||||
|
||||
async function main() {
|
||||
const credentials = JSON.parse(fs.readFileSync(`${directory}/super-admin-login.json`, 'utf8'))
|
||||
const response = await fetch(`${apiOrigin}/api/v1/auth/login`, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ email: credentials.email, password: credentials.password }),
|
||||
})
|
||||
assert.equal(response.status, 200)
|
||||
const auth = await response.json()
|
||||
const browser = await chromium.launch({ headless: true, channel: 'chrome' })
|
||||
const checks = [], errors = [], blockedWrites = [], writes = []
|
||||
const check = (name, passed) => { checks.push({ name, passed: !!passed }); assert.ok(passed, name) }
|
||||
try {
|
||||
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } })
|
||||
await context.addInitScript(a => {
|
||||
const entries = {
|
||||
weknora_user: JSON.stringify({ ...a.user, tenant_id: String(a.user.tenant_id) }),
|
||||
weknora_tenant: JSON.stringify({ ...a.active_tenant, id: '10059' }),
|
||||
weknora_token: a.token, weknora_refresh_token: a.refresh_token,
|
||||
weknora_memberships: JSON.stringify(a.memberships), 'weknora:new-user-guide-done:v1': '1',
|
||||
}
|
||||
for (const [key, value] of Object.entries(entries)) localStorage.setItem(key, value)
|
||||
}, auth)
|
||||
const fixture = (id, kind, name, data) => ({ id, kind, name, data, tenant_id: 10059, owner_id: auth.user.id, owner_name: 'UI verification', status: kind === 'training-maps' ? 'confirmed' : 'draft', version: 1, created_at: new Date().toISOString(), updated_at: new Date().toISOString() })
|
||||
const map = fixture('duration-map', 'training-maps', '时长验证导图', { days: [{ day: 1, title: '基础与实践' }], agent_id: 'duration-agent' })
|
||||
const flow = fixture('duration-flow', 'flows', '时长验证画布', { nodes: [{ id: 'node', label: '内容作者', agent_id: 'duration-agent' }], edges: [] })
|
||||
let task = null
|
||||
// All business writes below are in-memory fixtures; no task or AI job reaches the API.
|
||||
await context.route('**/api/v1/**', async route => {
|
||||
const request = route.request(), path = new URL(request.url()).pathname, method = request.method()
|
||||
const ok = body => route.fulfill({ status: 200, json: { success: true, ...body } })
|
||||
if (path === '/api/v1/access/me') return ok({ data: { administrator: false, restricted: true, permissions: ['content.read', 'content.write', 'flows.read'] } })
|
||||
const match = path.match(/^\/api\/v1\/workbench\/resources\/(content-tasks|training-maps|flows)(?:\/([^/]+))?(?:\/(jobs|actions\/.*))?$/)
|
||||
if (match) {
|
||||
const [, kind, id, action] = match
|
||||
if (method === 'GET' && action === 'jobs') return ok({ data: [], total: 0 })
|
||||
if (method === 'GET') {
|
||||
const items = kind === 'training-maps' ? [map] : kind === 'flows' ? [flow] : task ? [task] : []
|
||||
return ok(id ? { data: items.find(item => item.id === id) } : { data: items, total: items.length })
|
||||
}
|
||||
if (kind === 'content-tasks' && !action && ['POST', 'PUT'].includes(method)) {
|
||||
const body = request.postDataJSON()
|
||||
writes.push({ method, data: body.data })
|
||||
task = { ...fixture('duration-task', kind, body.name, body.data), version: (task?.version || 0) + 1 }
|
||||
return ok({ data: task })
|
||||
}
|
||||
}
|
||||
if (!['GET', 'HEAD', 'OPTIONS'].includes(method)) {
|
||||
blockedWrites.push({ path, method })
|
||||
return route.fulfill({ status: 409, json: { error: 'Verification prevents live writes' } })
|
||||
}
|
||||
return route.continue()
|
||||
})
|
||||
const page = await context.newPage()
|
||||
page.on('pageerror', error => errors.push(error.message))
|
||||
await page.goto(`${origin}/platform/workbench/content?tab=tasks`)
|
||||
await page.getByRole('button', { name: '创建内容任务', exact: true }).click()
|
||||
const drawer = page.locator('.t-drawer--open')
|
||||
const duration = drawer.locator('[aria-label="直播时长(分钟)"] input')
|
||||
await duration.waitFor()
|
||||
check('new_task_defaults_to_60_minutes', await duration.inputValue() === '60')
|
||||
await drawer.locator('.t-form__item').filter({ has: page.locator('.t-form__label').filter({ hasText: /^名称$/ }) }).locator('input').fill('直播时长验证任务')
|
||||
await duration.fill('135')
|
||||
await duration.press('Tab')
|
||||
await drawer.locator('.t-form__item').filter({ hasText: '已确认的训练营导图' }).locator('.t-select').click()
|
||||
await page.getByText('时长验证导图', { exact: true }).last().click()
|
||||
await drawer.locator('.t-form__item').filter({ hasText: '画布模板' }).locator('.t-select').click()
|
||||
await page.getByText('时长验证画布', { exact: true }).last().click()
|
||||
await page.screenshot({ path: `${directory}/content-duration-desktop.png`, animations: 'disabled' })
|
||||
await drawer.getByRole('button', { name: '保存草稿', exact: true }).click()
|
||||
await page.getByText('直播时长:135 分钟', { exact: true }).waitFor()
|
||||
check('draft_payload_and_detail_keep_duration', writes[0]?.data.duration_minutes === 135)
|
||||
await page.getByRole('button', { name: '编辑配置', exact: true }).click()
|
||||
await duration.waitFor()
|
||||
check('editing_reopens_saved_duration', await duration.inputValue() === '135')
|
||||
await page.setViewportSize({ width: 390, height: 844 })
|
||||
await duration.scrollIntoViewIfNeeded()
|
||||
await page.screenshot({ path: `${directory}/content-duration-mobile.png`, animations: 'disabled' })
|
||||
check('duration_control_fits_mobile', await duration.evaluate(element => { const r = element.getBoundingClientRect(); return r.left >= 0 && r.right <= innerWidth }))
|
||||
check('drawer_fits_mobile', await drawer.locator('.t-drawer__content-wrapper').evaluate(element => { const r = element.getBoundingClientRect(); return r.left >= -1 && r.right <= innerWidth + 1 }))
|
||||
await duration.fill('225')
|
||||
await duration.press('Tab')
|
||||
await drawer.getByRole('button', { name: '保存配置', exact: true }).click()
|
||||
await page.getByText('直播时长:225 分钟', { exact: true }).waitFor()
|
||||
check('edited_duration_saved', writes.at(-1)?.data.duration_minutes === 225)
|
||||
check('no_live_business_or_ai_writes', blockedWrites.length === 0 && writes.length === 2)
|
||||
check('no_runtime_errors', errors.length === 0)
|
||||
fs.writeFileSync(`${directory}/content-duration-ui.json`, JSON.stringify({ origin, checks, errors, businessWrites: 'mocked in browser only' }, null, 2))
|
||||
console.log(JSON.stringify({ checks, errors }, null, 2))
|
||||
} catch (error) {
|
||||
console.error(error.stack)
|
||||
process.exitCode = 1
|
||||
} finally {
|
||||
await browser.close()
|
||||
}
|
||||
}
|
||||
main().catch(error => { console.error(error.stack); process.exitCode = 1 })
|
||||
@@ -0,0 +1,62 @@
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const crypto = require('node:crypto')
|
||||
const { execFileSync } = require('node:child_process')
|
||||
|
||||
const schema = `content_permissions_${crypto.randomBytes(6).toString('hex')}`
|
||||
const migration = fs.readFileSync(path.join(__dirname, '../migrations/versioned/000207_content_action_permissions.up.sql'), 'utf8')
|
||||
const checks = []
|
||||
const literal = value => `'${String(value).replaceAll("'", "''")}'`
|
||||
function sql(query) {
|
||||
return execFileSync('docker', ['--context', 'colima-yawan', 'exec', '-i', 'WeKnora-postgres',
|
||||
'psql', '-X', '-q', '-U', 'postgres', '-d', 'yawan_native_acceptance_20260915', '-At', '-v', 'ON_ERROR_STOP=1'],
|
||||
{ input: `SET search_path TO ${schema}, public;\n${query}`, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim()
|
||||
}
|
||||
function permissions(id) { return JSON.parse(sql(`SELECT permissions FROM workbench_access_roles WHERE id=${literal(id)}`)) }
|
||||
function check(name, actual, expected) { assert.deepEqual(actual, expected, name); checks.push(name) }
|
||||
function legacyRole(id, sourcePermissions, nativePermissions = [], version = 1) {
|
||||
const spec = { id, source_role: id, name: id, source_permissions: sourcePermissions, permissions: nativePermissions, review_notes: [] }
|
||||
return `INSERT INTO workbench_access_roles(id,tenant_id,name,permissions,version) VALUES(${literal(id)},10059,${literal(id)},${literal(JSON.stringify(nativePermissions))},${version});
|
||||
INSERT INTO yawan_legacy_access_mappings(source_system,target_tenant_id,entity_key,source_spec,plan_sha256)
|
||||
VALUES('yawan-workbench',10059,${literal('role/' + id)},${literal(JSON.stringify(spec))},'test');`
|
||||
}
|
||||
try {
|
||||
sql(`CREATE SCHEMA ${schema};
|
||||
CREATE TABLE ${schema}.workbench_access_roles (LIKE public.workbench_access_roles INCLUDING ALL);
|
||||
CREATE TABLE ${schema}.yawan_legacy_access_mappings (LIKE public.yawan_legacy_access_mappings INCLUDING ALL);
|
||||
CREATE TABLE ${schema}.workbench_content_permission_migrations(version integer PRIMARY KEY, changes jsonb NOT NULL, created_at timestamptz NOT NULL DEFAULT now());
|
||||
INSERT INTO workbench_access_roles(id,tenant_id,name,permissions) VALUES
|
||||
('producer',10059,'producer','["content.read","content.write"]'),
|
||||
('reader',10059,'reader','["content.read"]'),
|
||||
('other-module',10059,'other-module','["agents.read"]');
|
||||
${legacyRole('reviewer', ['audit:read', 'read', 'review'])}
|
||||
${legacyRole('viewer', ['read'])}
|
||||
${legacyRole('edited-reviewer', ['review'], [], 2)}
|
||||
${legacyRole('editor', ['tasks:review', 'tasks:write', 'tasks:deliver'], ['content.read', 'content.write'])}
|
||||
${legacyRole('changed-reviewer', ['review'])}
|
||||
UPDATE workbench_access_roles SET permissions='["agents.read"]' WHERE id='changed-reviewer';`)
|
||||
const identityBefore = sql(`SELECT md5(coalesce(jsonb_agg(to_jsonb(u) ORDER BY id)::text,'')) FROM public.users u`)
|
||||
const membershipBefore = sql(`SELECT md5(coalesce(jsonb_agg(to_jsonb(m) ORDER BY id)::text,'')) FROM public.tenant_members m`)
|
||||
sql(`BEGIN; ${migration} COMMIT;`)
|
||||
check('native_producer_preserves_review_and_delivery', permissions('producer'), ['content.deliver', 'content.read', 'content.review', 'content.write'])
|
||||
check('native_reader_preserves_delivery_without_review', permissions('reader'), ['content.deliver', 'content.read'])
|
||||
check('legacy_reviewer_restored_without_generation_or_delivery', permissions('reviewer'), ['content.read', 'content.review'])
|
||||
check('generic_legacy_read_does_not_invent_task_access', permissions('viewer'), [])
|
||||
check('edited_import_role_is_preserved', permissions('edited-reviewer'), [])
|
||||
check('changed_import_permissions_are_preserved', permissions('changed-reviewer'), ['agents.read'])
|
||||
check('legacy_editor_preserves_all_existing_content_actions', permissions('editor'), ['content.deliver', 'content.read', 'content.review', 'content.write'])
|
||||
check('unrelated_module_unchanged', permissions('other-module'), ['agents.read'])
|
||||
check('original_legacy_ledger_is_not_rewritten', JSON.parse(sql(`SELECT source_spec->'permissions' FROM yawan_legacy_access_mappings WHERE entity_key='role/reviewer'`)), [])
|
||||
check('backfill_provenance_records_new_mapping', JSON.parse(sql(`SELECT c->'mapped_permissions' FROM workbench_content_permission_migrations,jsonb_array_elements(changes)c WHERE c->>'role_id'='reviewer'`)), ['content.read', 'content.review'])
|
||||
sql(`UPDATE workbench_access_roles SET permissions='[]',version=version+1 WHERE id='reviewer';
|
||||
INSERT INTO workbench_access_roles(id,tenant_id,name,permissions) VALUES('new-reader',10059,'new-reader','["content.read"]');
|
||||
BEGIN; ${migration} COMMIT;`)
|
||||
check('rerun_preserves_explicit_revocation', permissions('reviewer'), [])
|
||||
check('rerun_does_not_expand_post_migration_roles', permissions('new-reader'), ['content.read'])
|
||||
check('users_unchanged', sql(`SELECT md5(coalesce(jsonb_agg(to_jsonb(u) ORDER BY id)::text,'')) FROM public.users u`), identityBefore)
|
||||
check('memberships_unchanged', sql(`SELECT md5(coalesce(jsonb_agg(to_jsonb(m) ORDER BY id)::text,'')) FROM public.tenant_members m`), membershipBefore)
|
||||
console.log(JSON.stringify({ ok: true, checks }, null, 2))
|
||||
} finally {
|
||||
sql(`DROP SCHEMA IF EXISTS ${schema} CASCADE;`)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user