xinmi/scripts/weknora_resource_version_etl.py

#!/usr/bin/env python3
"""Preserve terminal legacy resource reviews in the local acceptance database.

No active fork is guessed or restored. The stopped SQLite source is required;
live WAL sources must use a separately reviewed snapshot. Default is rehearsal.
"""
from __future__ import annotations

import argparse
import hashlib
import json
import sqlite3
import subprocess
import uuid
from pathlib import Path

from apply_weknora_identities import sql_run
from weknora_category_etl import save_json
from weknora_identity_etl import TARGET_DATABASE, TARGET_TENANT

NAMESPACE = uuid.UUID("b81a9a18-0c49-4381-9754-6bd2f28c0107")
TABLE = "workbench_resource_version_history"


def canonical(value):
    return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)


def digest(value):
    return hashlib.sha256(canonical(value).encode()).hexdigest()


def file_digest(path):
    result = hashlib.sha256()
    with path.open("rb") as stream:
        for block in iter(lambda: stream.read(1024 * 1024), b""):
            result.update(block)
    return result.hexdigest()


def stopped_source(path):
    path = path.resolve()
    if Path(str(path) + "-wal").exists() or Path(str(path) + "-shm").exists():
        raise ValueError("Source has WAL/SHM; use a consistent stopped snapshot")
    opened = subprocess.run(["lsof", str(path)], capture_output=True, text=True)
    if opened.returncode != 1:
        raise ValueError("Source is open or its state cannot be verified")
    before = file_digest(path)
    with sqlite3.connect(path.as_uri() + "?mode=ro&immutable=1", uri=True) as db:
        db.row_factory = sqlite3.Row
        db.execute("PRAGMA query_only=ON")
        if db.execute("PRAGMA integrity_check").fetchone()[0] != "ok":
            raise ValueError("Source integrity check failed")
        forks = [dict(row) for row in db.execute("SELECT * FROM resourcefork ORDER BY id")]
        reviews = [dict(row) for row in db.execute("SELECT * FROM resourcerevision ORDER BY id")]
    if before != file_digest(path) or Path(str(path) + "-wal").exists():
        raise ValueError("Source changed during read")
    return {"forks": forks, "reviews": reviews, "source_sha256": before}


def target_mappings():
    return json.loads(sql_run(f"""BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY;
SELECT json_build_object('database',current_database(),
 'identities',(SELECT json_object_agg(source_user_id::text,target_user_id) FROM yawan_legacy_identity_mappings WHERE target_tenant_id={TARGET_TENANT}),
 'agents',(SELECT json_object_agg(source_id::text,target_id) FROM yawan_legacy_asset_mappings WHERE target_tenant_id={TARGET_TENANT} AND source_kind='agent'));
COMMIT;"""))


def make_plan(source, target):
    if target["database"] != TARGET_DATABASE:
        raise ValueError("Unexpected database")
    if source["forks"]:
        raise ValueError("Active historical forks need explicit native cloning; refusing a ledger-only migration")
    rows = []
    for row in source["reviews"]:
        if row["resource_type"] != "agent" or row["organization_id"] != 1 or row["status"] not in ("rejected", "withdrawn", "approved"):
            raise ValueError("Unreviewed resource type, scope or pending submission")
        if str(row["resource_id"]) not in target["agents"] or str(row["owner_user_id"]) not in target["identities"]:
            raise ValueError("A durable resource or owner mapping is missing")
        payload = json.loads(row["payload"]) if isinstance(row["payload"], str) else row["payload"]
        if not isinstance(payload, dict) or not isinstance(payload.get("name"), str):
            raise ValueError("Malformed historical submission")
        rows.append({"id": str(uuid.uuid5(NAMESPACE, f"resourcerevision:{row['id']}")),
                     "legacy_id": row["id"], "owner_id": target["identities"][str(row["owner_user_id"])],
                     "source_id": target["agents"][str(row["resource_id"])], "name": payload["name"],
                     "status": row["status"], "proposed_snapshot": payload, "review_note": row["review_note"],
                     "created_at": row["created_at"], "reviewed_at": row["reviewed_at"] or None,
                     "source_snapshot": row, "source_sha256": digest(row)})
    return {"database": TARGET_DATABASE, "tenant_id": TARGET_TENANT,
            "source_sha256": source["source_sha256"], "active_forks": 0, "rows": rows}


def literal(value):
    return "'" + value.replace("'", "''") + "'"


def transaction(plan, apply=False):
    if plan["database"] != TARGET_DATABASE or plan["tenant_id"] != TARGET_TENANT:
        raise ValueError("Unexpected target")
    migration = literal(canonical(plan))
    return f"""BEGIN ISOLATION LEVEL SERIALIZABLE;
SET LOCAL statement_timeout='30s'; SET LOCAL lock_timeout='5s';
CREATE TEMP TABLE _version_users AS SELECT * FROM users;
CREATE TEMP TABLE _version_members AS SELECT * FROM tenant_members;
DO $migration$
DECLARE p jsonb := {migration}::jsonb; item jsonb; old {TABLE}%ROWTYPE;
BEGIN
 IF current_database()<>'{TARGET_DATABASE}' THEN RAISE EXCEPTION 'Wrong database'; END IF;
 FOR item IN SELECT value FROM jsonb_array_elements(p->'rows') LOOP
  IF NOT EXISTS(SELECT 1 FROM yawan_legacy_identity_mappings WHERE target_tenant_id={TARGET_TENANT}
      AND source_user_id=(item->'source_snapshot'->>'owner_user_id')::bigint AND target_user_id=item->>'owner_id')
   OR NOT EXISTS(SELECT 1 FROM yawan_legacy_asset_mappings WHERE target_tenant_id={TARGET_TENANT} AND source_kind='agent'
      AND source_id=(item->'source_snapshot'->>'resource_id')::bigint AND target_id=item->>'source_id')
  THEN RAISE EXCEPTION 'Durable mapping mismatch'; END IF;
  SELECT * INTO old FROM {TABLE} WHERE tenant_id={TARGET_TENANT} AND source_system='yawan-workbench'
    AND source_table='resourcerevision' AND legacy_id=(item->>'legacy_id')::bigint;
  IF FOUND THEN
   IF old.id<>item->>'id' OR old.source_sha256<>item->>'source_sha256' OR old.source_snapshot<>item->'source_snapshot'
    OR old.owner_id<>item->>'owner_id' OR old.source_id<>item->>'source_id' OR old.status<>item->>'status'
    OR old.proposed_snapshot<>item->'proposed_snapshot'
   THEN RAISE EXCEPTION 'History ledger conflict'; END IF;
  ELSE
   INSERT INTO {TABLE}(id,tenant_id,owner_id,kind,source_id,name,status,proposed_snapshot,review_note,created_at,reviewed_at,
     source_system,source_table,legacy_id,source_snapshot,source_sha256)
   VALUES(item->>'id',{TARGET_TENANT},item->>'owner_id','agents',item->>'source_id',item->>'name',item->>'status',
     item->'proposed_snapshot',item->>'review_note',(item->>'created_at')::timestamptz,(item->>'reviewed_at')::timestamptz,
     'yawan-workbench','resourcerevision',(item->>'legacy_id')::bigint,item->'source_snapshot',item->>'source_sha256');
  END IF;
 END LOOP;
 IF EXISTS(SELECT 1 FROM _version_users b FULL JOIN users u USING(id) WHERE to_jsonb(b) IS DISTINCT FROM to_jsonb(u))
 OR EXISTS(SELECT 1 FROM _version_members b FULL JOIN tenant_members m USING(id) WHERE to_jsonb(b) IS DISTINCT FROM to_jsonb(m))
 THEN RAISE EXCEPTION 'Identity changed'; END IF;
END $migration$;
SELECT json_build_object('rows',(SELECT count(*) FROM {TABLE} WHERE tenant_id={TARGET_TENANT}),
 'identities_unchanged',true,'personal_resources_restored',false,'committed',{'true' if apply else 'false'});
{'COMMIT' if apply else 'ROLLBACK'};
"""


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--sqlite", type=Path, required=True)
    parser.add_argument("--out", type=Path, required=True)
    parser.add_argument("--apply", action="store_true")
    args = parser.parse_args()
    plan = make_plan(stopped_source(args.sqlite), target_mappings())
    save_json(args.out.with_suffix(".plan.json"), plan)
    result = json.loads(sql_run(transaction(plan, args.apply)))
    save_json(args.out, result)
    print(json.dumps(result))


if __name__ == "__main__":
    main()