"""Insert-only operational history import for the pinned acceptance tenant. Authoritative balances and actual transactions are retained independently. Discrepancies remain explicit; no invented credit or shared-user broadcasts. """ from __future__ import annotations import datetime import hashlib import json import re import sqlite3 import uuid from collections import Counter, defaultdict from weknora_asset_etl import LEDGER as ASSET_LEDGER, _mapping from weknora_business_etl import archive_text, _canonical from weknora_identity_etl import ADMIN_TARGET_ID, IDENTITY_LEDGER, TARGET_DATABASE, TARGET_TENANT SOURCE = "yawan-workbench" LEDGER = "yawan_legacy_operations_mappings" NAMESPACE = uuid.UUID("3fce9870-2fb6-51c6-b727-37ab10674612") KINDS = ("pointaccount", "pointtransaction", "usageevent", "knowledgeusage", "notification", "pendingmemory", "auditlog", "systemlog") TABLES = ("workbench_balances", "workbench_transactions", "workbench_usage", "workbench_consumption", "workbench_events", "workbench_pending_memories", "workbench_operation_logs") NATIVE_TABLE = dict(zip(KINDS, (*TABLES[:6], TABLES[-1], TABLES[-1]))) POLICY = "insert_only_authoritative_balances_owned_history_no_inferred_credit" SENSITIVE_KEY = re.compile(r"(?i)(api.?key|authorization|password|passwd|secret|access.?token|refresh.?token|cookie)") SECRET_PATTERNS = ( re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._~+/-]{8,}"), re.compile(r"\bsk-[A-Za-z0-9_-]{12,}"), re.compile(r'''(?i)((?:api[_-]?key|authorization|password|passwd|secret|access[_-]?token|refresh[_-]?token)\s*[=:]\s*["']?)[^\s,;"'&}\]]+'''), re.compile(r"(?i)(https?://)[^/\s:@]+:[^/\s@]+@"), ) def operation_id(kind, source_id): if kind not in (*KINDS, "history") or type(source_id) is not int or source_id <= 0: raise ValueError("Invalid operational source identity") return str(uuid.uuid5(NAMESPACE, f"{SOURCE}:tenant:{TARGET_TENANT}:organization:1:{kind}:{source_id}")) def history_id(kind, source_id): return str(uuid.uuid5(NAMESPACE, f"history:{kind}:{source_id}")) def redact(value): if isinstance(value, dict): return {k: "[REDACTED]" if SENSITIVE_KEY.search(k) else redact(v) for k, v in value.items()} if isinstance(value, list): return [redact(v) for v in value] if not isinstance(value, str): return value value = value.replace("\x00", "[NUL]") value = "".join(c if not 0xd800 <= ord(c) <= 0xdfff else "[INVALID-UNICODE]" for c in value) for pattern in SECRET_PATTERNS: value = pattern.sub(lambda m: (m.group(1) if m.lastindex else "") + "[REDACTED]", value) return value def _read(db, table): if table not in (*KINDS, "campaign", "segment"): raise ValueError("Unreviewed operations source table") result = db.execute(f'SELECT * FROM "{table}" ORDER BY id') fields = [c[0] for c in result.description] rows = [dict(zip(fields, row)) for row in result] for row in rows: if type(row["id"]) is not int or row["id"] <= 0: raise ValueError("Invalid operations source ID") if row.get("organization_id", 1) != 1: raise ValueError("Operations source organization differs") if table in ("pointtransaction", "usageevent"): row["metadata"] = json.loads(row["metadata"] or "{}") return rows def _time(value): if not value: return "1970-01-01T00:00:00+00:00" date = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) if date.tzinfo is None: date = date.replace(tzinfo=datetime.timezone.utc) return date.isoformat() def reconcile(accounts, transactions): totals = defaultdict(int) for tx in transactions: totals[tx["user_id"]] += tx["delta_points"] by_user = {} for account in accounts: if account["user_id"] in by_user: raise ValueError("Duplicate legacy point account") by_user[account["user_id"]] = account result = [] for user in sorted(set(totals) | set(by_user)): account = by_user.get(user) balance = account["balance_points"] if account else None matched = account is not None and balance == totals[user] result.append({"source_user_id": user, "source_balance": balance, "transaction_sum": totals[user], "difference": balance - totals[user] if account else None, "reconciled": matched, "native_import": account is not None, "reason": "" if matched else "balance_mismatch" if account else "missing_source_account"}) return result def _relation_owner(row, campaigns, segments, identities): campaign = row.get("campaign_id") segment = row.get("segment_id") if segment: sc = segments.get(segment) if not sc or (campaign and sc != campaign): return "", "segment_campaign_missing_or_mismatch" campaign = sc owner = campaigns.get(campaign) if not owner or str(owner) not in identities: return "", "unresolved_campaign_owner" return identities[str(owner)], "" def notification_owner(row, campaigns, segments, identities): ref, kind = row["ref_id"], row["type"] if kind == "pending_review": return _relation_owner({"segment_id": ref}, campaigns, segments, identities) if kind == "info": return _relation_owner({"campaign_id": ref}, campaigns, segments, identities) if kind == "failure": candidates = {campaigns[c] for c in (ref, segments.get(ref)) if c in campaigns} if len(candidates) == 1 and str(next(iter(candidates))) in identities: return identities[str(next(iter(candidates)))], "" return "", "ambiguous_or_missing_notification_reference" return "", "notification_has_no_reliable_recipient" def operations_plan(db, identities, agents, kbs, business, models): if db.execute("PRAGMA query_only").fetchone()[0] != 1: raise ValueError("Operations import requires a read-only consistent snapshot") identities, agents, kbs = (_mapping(m) for m in (identities, agents, kbs)) if identities.get("1") != ADMIN_TARGET_ID: raise ValueError("Pinned administrator identity differs") sources = {table: _read(db, table) for table in KINDS} campaigns = {r["id"]: r["owner_user_id"] for r in _read(db, "campaign")} segments = {r["id"]: r["campaign_id"] for r in _read(db, "segment")} balances = reconcile(sources["pointaccount"], sources["pointtransaction"]) reconciliation = {r["source_user_id"]: r for r in balances} charges = defaultdict(list) for tx in sources["pointtransaction"]: event = tx["metadata"].get("usage_event_id") if event: charges[event].append(tx) plan = {"schema_version": 1, "source": SOURCE, "target_database": TARGET_DATABASE, "target_tenant_id": TARGET_TENANT, "policy": POLICY, "identity_mapping": identities, "agent_mapping": agents, "kb_mapping": kbs, "business_mapping": business, "model_mapping": models, "reconciliation": balances, "items": []} for kind, rows in sources.items(): for source in rows: sid, review = source["id"], [] owner = identities.get(str(source.get("user_id", source.get("owner_user_id", source.get("actor_user_id")))), "") timestamp = _time(source.get("created_at", source.get("used_at", source.get("updated_at")))) target = {"id": operation_id(kind, sid), "tenant_id": TARGET_TENANT, "created_at": timestamp} table = "" metadata = {"legacy_import": {"source_kind": kind, "source_id": sid, "read_only": True}, "legacy_metadata": redact(source.get("metadata", {}))} if kind in ("pointaccount", "pointtransaction"): reconciliation_row = reconciliation[source["user_id"]] if not reconciliation_row["reconciled"]: review.append("balance_reconciliation_requires_admin_review") metadata["balance_reconciliation"] = reconciliation_row if not owner: review.append("missing_identity") elif kind == "pointaccount": table = "workbench_balances" target = {"tenant_id": TARGET_TENANT, "user_id": owner, "balance": source["balance_points"], "reserved": 0, "updated_at": _time(source["updated_at"])} else: table = "workbench_transactions" target.update(user_id=owner, delta=source["delta_points"], kind=source["kind"], reason=redact(source["reason"]), actor_id=identities.get(str(source["operator_user_id"]), ""), idempotency_key=f"legacy:pointtransaction:{sid}", metadata=metadata) elif kind == "usageevent": if not owner: review.append("missing_usage_owner") else: table = "workbench_usage" linked = charges[sid] if any(tx["user_id"] != source["user_id"] or tx["delta_points"] >= 0 for tx in linked): raise ValueError("Usage charge has conflicting owner or sign") points = -sum(tx["delta_points"] for tx in linked) metadata.update(legacy_unit=source["unit"], legacy_quantity=source["quantity"], legacy_currency=source["currency"], legacy_amount_minor=source["amount_minor"], legacy_transaction_ids=[tx["id"] for tx in linked], points_basis="exact_linked_debits" if linked else "no_linked_debit", legacy_call_id=source["llm_call_id"], legacy_campaign_id=source["campaign_id"], legacy_run_id=source["run_id"], legacy_model=redact(source["model"])) token = source["unit"] == "token" target.update(owner_id=owner, job_id="", call_id=f"legacy:usageevent:{sid}", agent_id=agents.get(str(source["agent_id"]), ""), model_id=models.get(f"{source['provider_id']}:{source['model']}", ""), action=redact(source["metadata"].get("operation") or source["service_type"]), input_tokens=source["input_quantity"] if token else 0, output_tokens=source["output_quantity"] if token else 0, points=points, status=source["status"], metadata=metadata) elif kind == "knowledgeusage": owner, reason = _relation_owner(source, campaigns, segments, identities) if reason: review.append(reason) if not agents.get(str(source["agent_id"])) or not kbs.get(str(source["kb_id"])): review.append("missing_consumption_asset_mapping") if not review: table = "workbench_consumption" # Reindexing changes chunks. Namespacing preserves historical usage # without falsely marking an unrelated native chunk consumed. chunk = "legacy:" + str(source["chunk_id"]) if len(chunk) > 200: chunk = "legacy:sha256:" + hashlib.sha256(chunk.encode()).hexdigest() target.update(owner_id=owner, agent_id=agents[str(source["agent_id"])], kb_id=kbs[str(source["kb_id"])], chunk_id=chunk, knowledge_id="", file_name=redact(source["file_name"]), category=redact(source["heading"]), excerpt=redact(source["excerpt"]), resource_id=business.get(f"campaign:{source['campaign_id']}", ""), segment_id=business.get(f"segment:{source['segment_id']}", ""), job_id="", status=source["status"], banned=bool(source["banned"]), lease_until=None, updated_at=_time(source["released_at"] or source["used_at"])) elif kind == "notification": owner, reason = notification_owner(source, campaigns, segments, identities) if reason: review.append(reason) else: table = "workbench_events" target.update(owner_id=owner, resource_id="", action=source["type"], message=redact(source["message"]), level="error" if source["type"] == "failure" else "info", read=source["status"] == "read") elif kind == "pendingmemory": if not owner or not agents.get(str(source["target_agent_id"])): review.append("missing_memory_owner_or_agent") else: table = "workbench_pending_memories" target.update(owner_id=owner, agent_id=agents[str(source["target_agent_id"])], content=redact(source["content"]), source_ref=redact(source["source_ref"]), status={"approved": "accepted"}.get(source["status"], source["status"]), reviewed_by="", updated_at=timestamp) else: table = "workbench_operation_logs" system = kind == "systemlog" level = str(source["level"]).lower() if system else ("info" if source["result"] in ("ok", "success") else "error") status = source.get("status", 0) if system else (200 if level == "info" else 400) status = int(status) if str(status).isdigit() else 0 target.update(owner_id=owner, owner_name=redact(source.get("actor_display_name", "")), kind="system" if system else "audit", level=level, action=redact(source["source"] if system else source["action"]), message=redact(source["message"] if system else source["result"]), resource_id=redact(source.get("resource", "")), trace_id=redact(source.get("trace_id", ""))[:100], status=status, metadata=metadata) plan["items"].append({"source_kind": kind, "source_id": sid, "source_snapshot": archive_text(source), "history_id": history_id(kind, sid), "owner_id": owner, "review": review, "target_table": table, "target_key": target.get("user_id") if table == "workbench_balances" else target["id"], "native_snapshot": target if table else {}, "created_at": timestamp, "history_payload": {"source": archive_text(redact(source)), "review": review, "read_only": True, "original_in_private_export": True}}) if kind in ("pointaccount", "pointtransaction"): plan["items"][-1]["history_payload"]["balance_reconciliation"] = reconciliation[source["user_id"]] return plan def validate_plan(plan): if (plan.get("schema_version") != 1 or plan.get("source") != SOURCE or plan.get("target_database") != TARGET_DATABASE or plan.get("target_tenant_id") != TARGET_TENANT or plan.get("policy") != POLICY): raise ValueError("Operations migration scope differs") if _mapping(plan["identity_mapping"]).get("1") != ADMIN_TARGET_ID: raise ValueError("Operations administrator mapping differs") seen = set() for item in plan["items"]: key = item["source_kind"], item["source_id"] if key in seen or key[0] not in KINDS or item["source_snapshot"]["id"] != key[1] or item["history_id"] != history_id(*key): raise ValueError("Operations source identity conflict") seen.add(key) native, table = item["native_snapshot"], item["target_table"] if table and table != NATIVE_TABLE[key[0]]: raise ValueError("Operations native table differs") if table and native.get("tenant_id") != TARGET_TENANT: raise ValueError("Operations native tenant differs") if table and table != "workbench_balances" and native["id"] != operation_id(*key): raise ValueError("Operations native identity differs") if table and table != "workbench_operation_logs" and not item["owner_id"]: raise ValueError("Owned operations cannot be broadcast") if table: owner_column = "user_id" if table in ("workbench_balances", "workbench_transactions") else "owner_id" native_key = native["user_id"] if table == "workbench_balances" else native["id"] if native[owner_column] != item["owner_id"] or item["target_key"] != native_key: raise ValueError("Operations native owner or key differs") if item["owner_id"] and item["owner_id"] not in plan["identity_mapping"].values(): raise ValueError("Operations owner is not mapped") accounts = [i["source_snapshot"] for i in plan["items"] if i["source_kind"] == "pointaccount"] transactions = [i["source_snapshot"] for i in plan["items"] if i["source_kind"] == "pointtransaction"] if plan["reconciliation"] != reconcile(accounts, transactions): raise ValueError("Balance reconciliation was changed") for item in plan["items"]: source, native = item["source_snapshot"], item["native_snapshot"] if item["target_table"] == "workbench_balances" and native["balance"] != source["balance_points"]: raise ValueError("Authoritative current balance must remain exact") if item["target_table"] == "workbench_transactions" and native["delta"] != source["delta_points"]: raise ValueError("Historical transactions must remain exact") def safe_operations_summary(plan): validate_plan(plan) return {"database": TARGET_DATABASE, "tenant_id": TARGET_TENANT, "source_counts": dict(Counter(i["source_kind"] for i in plan["items"])), "native_counts": dict(Counter(i["target_table"] for i in plan["items"] if i["target_table"])), "archive_only_counts": dict(Counter(i["source_kind"] for i in plan["items"] if not i["target_table"])), "review_counts": dict(Counter(r for i in plan["items"] for r in i["review"])), "reconciliation": plan["reconciliation"], "source_snapshot_sha256": plan.get("source_snapshot_sha256"), "history_owner_policy": "admin_only_archive", "full_original_export_required": True, "consumption_chunk_policy": "legacy_namespace_no_inferred_native_chunk", "jobs_created": 0} def operations_sql(plan): validate_plan(plan) fingerprint = plan.get("source_snapshot_sha256", "") if not re.fullmatch("[a-f0-9]{64}", fingerprint): raise ValueError("Operations source snapshot fingerprint required") literal = "'" + _canonical(plan).replace("'", "''") + "'::jsonb" sql = f"""SET LOCAL standard_conforming_strings=on; SET LOCAL log_statement='none'; SET LOCAL log_min_error_statement='panic'; CREATE TEMP TABLE _ops_plan(payload jsonb) ON COMMIT DROP; INSERT INTO _ops_plan VALUES ({literal}); DO $scope$ BEGIN IF current_database()<>'{TARGET_DATABASE}' THEN RAISE EXCEPTION 'Operations database mismatch'; END IF; PERFORM pg_advisory_xact_lock({TARGET_TENANT},11907); PERFORM 1 FROM tenants WHERE id={TARGET_TENANT} AND status='active' AND deleted_at IS NULL FOR SHARE; IF NOT FOUND THEN RAISE EXCEPTION 'Operations target tenant inactive'; END IF; IF EXISTS (SELECT 1 FROM _ops_plan p,jsonb_each_text(p.payload->'identity_mapping') i LEFT JOIN {IDENTITY_LEDGER} m ON m.source_system='{SOURCE}' AND m.source_organization_id=1 AND m.source_user_id=i.key::bigint AND m.target_tenant_id={TARGET_TENANT} AND m.target_user_id=i.value LEFT JOIN users u ON u.id=i.value AND u.deleted_at IS NULL LEFT JOIN tenant_members tm ON tm.user_id=i.value AND tm.tenant_id={TARGET_TENANT} AND tm.deleted_at IS NULL WHERE m.target_user_id IS NULL OR u.id IS NULL OR tm.id IS NULL) THEN RAISE EXCEPTION 'Operations durable owner mapping conflict'; END IF; IF EXISTS (SELECT 1 FROM _ops_plan p,jsonb_each_text(p.payload->'agent_mapping') i LEFT JOIN {ASSET_LEDGER} m ON m.source_system='{SOURCE}' AND m.source_organization_id=1 AND m.source_kind='agent' AND m.source_id=i.key::bigint AND m.target_tenant_id={TARGET_TENANT} AND m.target_id=i.value LEFT JOIN custom_agents a ON a.id=i.value AND a.tenant_id={TARGET_TENANT} WHERE m.target_id IS NULL OR a.id IS NULL) THEN RAISE EXCEPTION 'Operations durable Agent mapping conflict'; END IF; IF EXISTS (SELECT 1 FROM _ops_plan p,jsonb_each_text(p.payload->'kb_mapping') i LEFT JOIN knowledge_bases k ON k.id=i.value AND k.tenant_id={TARGET_TENANT} AND k.deleted_at IS NULL WHERE k.id IS NULL) THEN RAISE EXCEPTION 'Operations KB scope conflict'; END IF; IF EXISTS (SELECT 1 FROM _ops_plan p,jsonb_each_text(p.payload->'business_mapping') i LEFT JOIN yawan_legacy_business_mappings b ON b.source_system='{SOURCE}' AND b.source_organization_id=1 AND b.source_kind||':'||b.source_id=i.key AND b.target_tenant_id={TARGET_TENANT} AND b.target_id=i.value WHERE b.target_id IS NULL) THEN RAISE EXCEPTION 'Operations durable business mapping conflict'; END IF; IF EXISTS (SELECT 1 FROM _ops_plan p,jsonb_each_text(p.payload->'model_mapping') i LEFT JOIN yawan_legacy_model_mappings m ON m.source_system='{SOURCE}' AND m.source_organization_id=1 AND m.target_tenant_id={TARGET_TENANT} AND m.target_model_id=i.value WHERE m.target_model_id IS NULL) THEN RAISE EXCEPTION 'Operations durable model mapping conflict'; END IF; END $scope$; LOCK TABLE {','.join(TABLES)},workbench_history IN SHARE ROW EXCLUSIVE MODE; CREATE TABLE IF NOT EXISTS {LEDGER} ( source_system text NOT NULL, source_organization_id bigint NOT NULL, source_kind text NOT NULL, source_id bigint NOT NULL, target_tenant_id bigint NOT NULL, history_id varchar(36) NOT NULL, target_table text NOT NULL, target_key text NOT NULL, source_snapshot jsonb NOT NULL, native_snapshot jsonb NOT NULL, review jsonb NOT NULL, source_snapshot_sha256 varchar(64) NOT NULL CHECK(source_snapshot_sha256 ~ '^[a-f0-9]{{64}}$'), created_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY(source_system,source_organization_id,source_kind,source_id), UNIQUE(target_tenant_id,history_id) ); LOCK TABLE {LEDGER} IN SHARE ROW EXCLUSIVE MODE; CREATE TEMP TABLE _ops_items ON COMMIT DROP AS SELECT item.* FROM _ops_plan p, jsonb_to_recordset(p.payload->'items') AS item(source_kind text,source_id bigint,source_snapshot jsonb, history_id text,owner_id text,review jsonb,target_table text,target_key text,native_snapshot jsonb,created_at timestamptz,history_payload jsonb); CREATE INDEX ON _ops_items(source_kind,source_id); DO $provenance$ BEGIN IF EXISTS (SELECT 1 FROM _ops_items i JOIN {LEDGER} l USING(source_kind,source_id) WHERE l.source_system='{SOURCE}' AND l.source_organization_id=1 AND (l.target_tenant_id IS DISTINCT FROM {TARGET_TENANT} OR l.history_id IS DISTINCT FROM i.history_id OR l.target_table IS DISTINCT FROM i.target_table OR l.target_key IS DISTINCT FROM i.target_key OR l.source_snapshot IS DISTINCT FROM i.source_snapshot OR l.native_snapshot IS DISTINCT FROM i.native_snapshot OR l.review IS DISTINCT FROM i.review)) THEN RAISE EXCEPTION 'Operations provenance conflict'; END IF; IF EXISTS (SELECT 1 FROM _ops_items i JOIN workbench_history h ON h.id=i.history_id WHERE NOT EXISTS (SELECT 1 FROM {LEDGER} l WHERE l.source_system='{SOURCE}' AND l.source_organization_id=1 AND l.source_kind=i.source_kind AND l.source_id=i.source_id)) THEN RAISE EXCEPTION 'Operations history collision'; END IF; IF EXISTS (SELECT 1 FROM _ops_items i JOIN {LEDGER} l USING(source_kind,source_id) LEFT JOIN workbench_history h ON h.id=i.history_id AND h.tenant_id={TARGET_TENANT} AND h.owner_id='' WHERE l.source_system='{SOURCE}' AND l.source_organization_id=1 AND h.id IS NULL) THEN RAISE EXCEPTION 'Operations history removed or scope changed'; END IF; END $provenance$; CREATE TEMP TABLE _ops_new ON COMMIT DROP AS SELECT i.* FROM _ops_items i WHERE NOT EXISTS (SELECT 1 FROM {LEDGER} l WHERE l.source_system='{SOURCE}' AND l.source_organization_id=1 AND l.source_kind=i.source_kind AND l.source_id=i.source_id); """ for table in TABLES: key = "user_id" if table == "workbench_balances" else "id" owner = "user_id" if table in ("workbench_balances", "workbench_transactions") else "owner_id" sql += f"""DO $native$ BEGIN IF EXISTS (SELECT 1 FROM _ops_new i JOIN {table} n ON n.{key}=i.target_key WHERE i.target_table='{table}' {'AND n.tenant_id=' + str(TARGET_TENANT) if key == 'user_id' else ''}) THEN RAISE EXCEPTION 'Operations native collision in {table}'; END IF; IF EXISTS (SELECT 1 FROM _ops_items i JOIN {LEDGER} l USING(source_kind,source_id) LEFT JOIN {table} n ON n.{key}=i.target_key AND n.tenant_id={TARGET_TENANT} AND n.{owner}=i.owner_id WHERE l.source_system='{SOURCE}' AND l.source_organization_id=1 AND i.target_table='{table}' AND n.{key} IS NULL) THEN RAISE EXCEPTION 'Operations native row removed or ownership changed in {table}'; END IF; END $native$; INSERT INTO {table} SELECT (jsonb_populate_record(NULL::{table},i.native_snapshot)).* FROM _ops_new i WHERE i.target_table='{table}'; """ sql += f"""INSERT INTO workbench_history(id,tenant_id,owner_id,source_kind,source_id,payload,created_at) SELECT history_id,{TARGET_TENANT},'',source_kind,source_id::text,history_payload,created_at FROM _ops_new; INSERT INTO {LEDGER}(source_system,source_organization_id,source_kind,source_id,target_tenant_id,history_id,target_table,target_key, source_snapshot,native_snapshot,review,source_snapshot_sha256) SELECT '{SOURCE}',1,source_kind,source_id,{TARGET_TENANT},history_id,target_table,target_key,source_snapshot,native_snapshot,review,'{fingerprint}' FROM _ops_new; SELECT json_build_object('inserted_history',(SELECT count(*) FROM _ops_new), 'inserted_native',(SELECT count(*) FROM _ops_new WHERE target_table<>''), 'ledger_total',(SELECT count(*) FROM {LEDGER} WHERE target_tenant_id={TARGET_TENANT})); """ return sql