WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

scripts/weknora_resource_version_etl.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""Preserve terminal legacy resource reviews in the local acceptance database.
+
+No active fork is guessed or restored. The stopped SQLite source is required;
+live WAL sources must use a separately reviewed snapshot. Default is rehearsal.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sqlite3
+import subprocess
+import uuid
+from pathlib import Path
+
+from apply_weknora_identities import sql_run
+from weknora_category_etl import save_json
+from weknora_identity_etl import TARGET_DATABASE, TARGET_TENANT
+
+NAMESPACE = uuid.UUID("b81a9a18-0c49-4381-9754-6bd2f28c0107")
+TABLE = "workbench_resource_version_history"
+
+
+def canonical(value):
+    return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
+
+
+def digest(value):
+    return hashlib.sha256(canonical(value).encode()).hexdigest()
+
+
+def file_digest(path):
+    result = hashlib.sha256()
+    with path.open("rb") as stream:
+        for block in iter(lambda: stream.read(1024 * 1024), b""):
+            result.update(block)
+    return result.hexdigest()
+
+
+def stopped_source(path):
+    path = path.resolve()
+    if Path(str(path) + "-wal").exists() or Path(str(path) + "-shm").exists():
+        raise ValueError("Source has WAL/SHM; use a consistent stopped snapshot")
+    opened = subprocess.run(["lsof", str(path)], capture_output=True, text=True)
+    if opened.returncode != 1:
+        raise ValueError("Source is open or its state cannot be verified")
+    before = file_digest(path)
+    with sqlite3.connect(path.as_uri() + "?mode=ro&immutable=1", uri=True) as db:
+        db.row_factory = sqlite3.Row
+        db.execute("PRAGMA query_only=ON")
+        if db.execute("PRAGMA integrity_check").fetchone()[0] != "ok":
+            raise ValueError("Source integrity check failed")
+        forks = [dict(row) for row in db.execute("SELECT * FROM resourcefork ORDER BY id")]
+        reviews = [dict(row) for row in db.execute("SELECT * FROM resourcerevision ORDER BY id")]
+    if before != file_digest(path) or Path(str(path) + "-wal").exists():
+        raise ValueError("Source changed during read")
+    return {"forks": forks, "reviews": reviews, "source_sha256": before}
+
+
+def target_mappings():
+    return json.loads(sql_run(f"""BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY;
+SELECT json_build_object('database',current_database(),
+ 'identities',(SELECT json_object_agg(source_user_id::text,target_user_id) FROM yawan_legacy_identity_mappings WHERE target_tenant_id={TARGET_TENANT}),
+ 'agents',(SELECT json_object_agg(source_id::text,target_id) FROM yawan_legacy_asset_mappings WHERE target_tenant_id={TARGET_TENANT} AND source_kind='agent'));
+COMMIT;"""))
+
+
+def make_plan(source, target):
+    if target["database"] != TARGET_DATABASE:
+        raise ValueError("Unexpected database")
+    if source["forks"]:
+        raise ValueError("Active historical forks need explicit native cloning; refusing a ledger-only migration")
+    rows = []
+    for row in source["reviews"]:
+        if row["resource_type"] != "agent" or row["organization_id"] != 1 or row["status"] not in ("rejected", "withdrawn", "approved"):
+            raise ValueError("Unreviewed resource type, scope or pending submission")
+        if str(row["resource_id"]) not in target["agents"] or str(row["owner_user_id"]) not in target["identities"]:
+            raise ValueError("A durable resource or owner mapping is missing")
+        payload = json.loads(row["payload"]) if isinstance(row["payload"], str) else row["payload"]
+        if not isinstance(payload, dict) or not isinstance(payload.get("name"), str):
+            raise ValueError("Malformed historical submission")
+        rows.append({"id": str(uuid.uuid5(NAMESPACE, f"resourcerevision:{row['id']}")),
+                     "legacy_id": row["id"], "owner_id": target["identities"][str(row["owner_user_id"])],
+                     "source_id": target["agents"][str(row["resource_id"])], "name": payload["name"],
+                     "status": row["status"], "proposed_snapshot": payload, "review_note": row["review_note"],
+                     "created_at": row["created_at"], "reviewed_at": row["reviewed_at"] or None,
+                     "source_snapshot": row, "source_sha256": digest(row)})
+    return {"database": TARGET_DATABASE, "tenant_id": TARGET_TENANT,
+            "source_sha256": source["source_sha256"], "active_forks": 0, "rows": rows}
+
+
+def literal(value):
+    return "'" + value.replace("'", "''") + "'"
+
+
+def transaction(plan, apply=False):
+    if plan["database"] != TARGET_DATABASE or plan["tenant_id"] != TARGET_TENANT:
+        raise ValueError("Unexpected target")
+    migration = literal(canonical(plan))
+    return f"""BEGIN ISOLATION LEVEL SERIALIZABLE;
+SET LOCAL statement_timeout='30s'; SET LOCAL lock_timeout='5s';
+CREATE TEMP TABLE _version_users AS SELECT * FROM users;
+CREATE TEMP TABLE _version_members AS SELECT * FROM tenant_members;
+DO $migration$
+DECLARE p jsonb := {migration}::jsonb; item jsonb; old {TABLE}%ROWTYPE;
+BEGIN
+ IF current_database()<>'{TARGET_DATABASE}' THEN RAISE EXCEPTION 'Wrong database'; END IF;
+ FOR item IN SELECT value FROM jsonb_array_elements(p->'rows') LOOP
+  IF NOT EXISTS(SELECT 1 FROM yawan_legacy_identity_mappings WHERE target_tenant_id={TARGET_TENANT}
+      AND source_user_id=(item->'source_snapshot'->>'owner_user_id')::bigint AND target_user_id=item->>'owner_id')
+   OR NOT EXISTS(SELECT 1 FROM yawan_legacy_asset_mappings WHERE target_tenant_id={TARGET_TENANT} AND source_kind='agent'
+      AND source_id=(item->'source_snapshot'->>'resource_id')::bigint AND target_id=item->>'source_id')
+  THEN RAISE EXCEPTION 'Durable mapping mismatch'; END IF;
+  SELECT * INTO old FROM {TABLE} WHERE tenant_id={TARGET_TENANT} AND source_system='yawan-workbench'
+    AND source_table='resourcerevision' AND legacy_id=(item->>'legacy_id')::bigint;
+  IF FOUND THEN
+   IF old.id<>item->>'id' OR old.source_sha256<>item->>'source_sha256' OR old.source_snapshot<>item->'source_snapshot'
+    OR old.owner_id<>item->>'owner_id' OR old.source_id<>item->>'source_id' OR old.status<>item->>'status'
+    OR old.proposed_snapshot<>item->'proposed_snapshot'
+   THEN RAISE EXCEPTION 'History ledger conflict'; END IF;
+  ELSE
+   INSERT INTO {TABLE}(id,tenant_id,owner_id,kind,source_id,name,status,proposed_snapshot,review_note,created_at,reviewed_at,
+     source_system,source_table,legacy_id,source_snapshot,source_sha256)
+   VALUES(item->>'id',{TARGET_TENANT},item->>'owner_id','agents',item->>'source_id',item->>'name',item->>'status',
+     item->'proposed_snapshot',item->>'review_note',(item->>'created_at')::timestamptz,(item->>'reviewed_at')::timestamptz,
+     'yawan-workbench','resourcerevision',(item->>'legacy_id')::bigint,item->'source_snapshot',item->>'source_sha256');
+  END IF;
+ END LOOP;
+ IF EXISTS(SELECT 1 FROM _version_users b FULL JOIN users u USING(id) WHERE to_jsonb(b) IS DISTINCT FROM to_jsonb(u))
+ OR EXISTS(SELECT 1 FROM _version_members b FULL JOIN tenant_members m USING(id) WHERE to_jsonb(b) IS DISTINCT FROM to_jsonb(m))
+ THEN RAISE EXCEPTION 'Identity changed'; END IF;
+END $migration$;
+SELECT json_build_object('rows',(SELECT count(*) FROM {TABLE} WHERE tenant_id={TARGET_TENANT}),
+ 'identities_unchanged',true,'personal_resources_restored',false,'committed',{'true' if apply else 'false'});
+{'COMMIT' if apply else 'ROLLBACK'};
+"""
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--sqlite", type=Path, required=True)
+    parser.add_argument("--out", type=Path, required=True)
+    parser.add_argument("--apply", action="store_true")
+    args = parser.parse_args()
+    plan = make_plan(stopped_source(args.sqlite), target_mappings())
+    save_json(args.out.with_suffix(".plan.json"), plan)
+    result = json.loads(sql_run(transaction(plan, args.apply)))
+    save_json(args.out, result)
+    print(json.dumps(result))
+
+
+if __name__ == "__main__":
+    main()