xinmi/scripts/test_weknora_asset_etl.py

"""Read-only asset plans, lossless text and native SQL migration contracts."""
import copy
import json
import sqlite3
import sys
import unittest
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent))
from weknora_asset_etl import (AGENT_COLUMNS, SKILL_COLUMNS, TEMPLATE_COLUMNS, ADMIN_TARGET_ID,
                               LEDGER, BINDING_LEDGER, asset_id, asset_plan, asset_sql, safe_asset_summary)
from weknora_model_etl import model_plan, model_id

USER_ID = "ab043a7d-d10e-58fe-8417-3fd2cbab5f77"
KB_ID = "7e6ac485-f766-4d58-ae38-2f0456a022ef"


class AssetTests(unittest.TestCase):
    def setUp(self):
        self.db = sqlite3.connect(":memory:")
        self.addCleanup(self.db.close)
        integer_columns = {"id", "model_provider_id", "model_catalog_id", "prompt_template_id", "rules_version",
                           "builtin", "organization_id", "owner_user_id", "digital_human_id", "enabled", "version"}
        for table, columns in (("agent", AGENT_COLUMNS), ("agentskill", SKILL_COLUMNS), ("agentprompttemplate", TEMPLATE_COLUMNS)):
            self.db.execute("CREATE TABLE " + table + " (" + ",".join(
                column + (" INTEGER" if column in integer_columns else " TEXT") for column in columns) + ")")
        self.db.executescript("""
            CREATE TABLE agentskillbinding (id INTEGER,agent_id INTEGER,skill_id INTEGER);
            CREATE TABLE agentgrant (id INTEGER,agent_id INTEGER,user_id INTEGER,permission TEXT,enabled INTEGER,created_at TEXT);
            CREATE TABLE agentskillgrant (id INTEGER,skill_id INTEGER,user_id INTEGER,permission TEXT,enabled INTEGER,created_at TEXT);
            CREATE TABLE modelprovider (id INTEGER,name TEXT,base_url TEXT,api_key TEXT,models TEXT,
                default_model TEXT,builtin INTEGER,enabled INTEGER,organization_id INTEGER);
            CREATE TABLE modelcatalog (id INTEGER,provider_id INTEGER,model_name TEXT,display_name TEXT,
                capabilities TEXT,capability_status TEXT,enabled INTEGER,builtin INTEGER,rate_version TEXT,
                input_rate REAL,output_rate REAL);
            CREATE TABLE appsetting (key TEXT,value TEXT);
            INSERT INTO modelprovider VALUES (3,'deepseek','https://api.deepseek.com/v1','DO-NOT-READ-SECRET',
                '["deepseek-flash","deepseek-v4-pro"]','deepseek-flash',0,1,1);
            INSERT INTO modelcatalog VALUES (6,3,'deepseek-v4-pro','Pro','[]','untested',1,0,'r1',1,2);
            INSERT INTO modelcatalog VALUES (7,3,'deepseek-flash','Flash','[]','untested',1,0,'r1',1,2);
            INSERT INTO appsetting VALUES ('default_model_provider_id','3');
        """)
        self.agent(1, owner_user_id=0, builtin=1, visibility="global", kb_ids="[3]")
        self.agent(19, owner_user_id=3, status="disabled", visibility="private")
        self.agent(22, owner_user_id=3, visibility="private")
        self.skill(1, owner_user_id=0, builtin=1, visibility="global")
        self.skill(5, owner_user_id=3, visibility="private")
        self.skill(9, owner_user_id=1, visibility="private")
        self.db.executemany("INSERT INTO agentskillbinding VALUES (?,?,?)",
                            [(101, 1, 1), (102, 1, 5), (103, 1, 9), (104, 19, 1), (105, 22, 1), (106, 22, 5)])
        template = {key: "" for key in TEMPLATE_COLUMNS}
        template.update(id=1, name="Source template", desc="Preserve template", version=1, enabled=1, builtin=1,
                        organization_id=1, owner_user_id=0, visibility="global", constraints='["constraint"]',
                        examples='[]', output_schema='{}', role_prompt="original-template-body",
                        task_template="{{runtime_input}}", created_at="2026-09-15", updated_at="2026-09-15")
        self.insert("agentprompttemplate", TEMPLATE_COLUMNS, template)
        self.identities, self.kbs = {1: ADMIN_TARGET_ID, 3: USER_ID}, {3: KB_ID}

    def insert(self, table, columns, values):
        self.db.execute("INSERT INTO " + table + " (" + ",".join(columns) + ") VALUES (" +
                        ",".join("?" for _ in columns) + ")", [values[k] for k in columns])

    def agent(self, source_id, **overrides):
        values = {key: None for key in AGENT_COLUMNS}
        values.update(id=source_id, name=f"Agent {source_id}", role="production", desc="original description",
                      content_tags='["source-tag"]', kb_ids='[]', fusion_agent_ids='[]', model_provider_id=None,
                      model_name='deepseek-chat', model_catalog_id=None, prompt_template_id=1,
                      allowed_tools='["knowledge_search","read_agent_memory"]', rules="  Original rules\n\n",
                      rules_version=2, status="active", builtin=0, organization_id=1, owner_user_id=1,
                      visibility="private", created_at="2026-09-15 00:00:00",
                      runtime_snapshot='{"primary":{"rules":"frozen-original-rules"}}', persona_json='{"voice":"plain"}',
                      digital_human_id=None, kb_category_ids='{}')
        values.update(overrides)
        self.insert("agent", AGENT_COLUMNS, values)

    def skill(self, source_id, **overrides):
        values = dict(id=source_id, name=f"Skill {source_id}", desc="source description",
                      instructions="  Original skill body\n\n", enabled=1, builtin=0, version=2,
                      organization_id=1, owner_user_id=1, visibility="private", created_at="2026-09-15 00:00:00")
        values.update(overrides)
        self.insert("agentskill", SKILL_COLUMNS, values)

    def plan(self):
        plan = asset_plan(self.db, self.identities, self.kbs, model_plan(self.db))
        plan["source_snapshot_sha256"] = "a" * 64
        return plan

    def test_preserves_original_text_and_unsupported_configuration(self):
        plan = self.plan()
        self.assertEqual(plan["agents"][0]["target"]["config"]["system_prompt"], "  Original rules\n\n")
        self.assertEqual(plan["skills"][0]["target"]["instructions"], "  Original skill body\n\n")
        self.assertEqual(plan["agents"][0]["source"]["runtime_snapshot"]["primary"]["rules"], "frozen-original-rules")
        self.assertEqual(plan["prompt_templates"][0]["role_prompt"], "original-template-body")
        self.assertEqual(plan["agents"][0]["source"]["content_tags"], ["source-tag"])
        self.assertEqual(plan["agents"][0]["target"]["config"]["content_tags"], ["source-tag"])
        self.assertEqual(plan["agents"][0]["target"]["config"]["role"], "production")
        self.assertNotIn("role", plan["unsupported_fields"])
        self.assertNotIn("content_tags", plan["unsupported_fields"])
        self.assertEqual(plan["agents"][0]["target"]["config"]["fusion_agent_ids"], [])
        self.assertNotIn("fusion_agent_ids", plan["unsupported_fields"])

    def test_rejects_invalid_fusion_configuration(self):
        for value in ('[1]', '[2,2]', '[true]', '["2"]', '[3]', '[0]'):
            with self.subTest(value=value):
                self.db.execute("UPDATE agent SET fusion_agent_ids=? WHERE id=1", (value,))
                with self.assertRaises(ValueError):
                    self.plan()
        self.db.execute("UPDATE agent SET fusion_agent_ids='[]' WHERE id=1")

    def test_maps_direct_fusions_without_adding_their_knowledge_or_recursion(self):
        self.db.execute("UPDATE agent SET fusion_agent_ids='[22,19]' WHERE id=1")
        self.db.execute("UPDATE agent SET fusion_agent_ids='[1]' WHERE id=22")
        plan = self.plan()
        config = plan["agents"][0]["target"]["config"]
        self.assertEqual(config["fusion_agent_ids"], [asset_id("agent", 22), asset_id("agent", 19)])
        self.assertEqual(config["knowledge_bases"], [KB_ID])
        self.db.execute("UPDATE agent SET role='qc' WHERE id=19")
        with self.assertRaises(ValueError):
            self.plan()

    def test_rejects_invalid_native_duties_and_tags(self):
        for field, value in (("role", "unknown"), ("content_tags", '["same","same"]'),
                             ("content_tags", '[" "]'), ("content_tags", '[1]')):
            with self.subTest(field=field, value=value):
                self.db.execute(f"UPDATE agent SET {field}=? WHERE id=1", (value,))
                with self.assertRaises(ValueError):
                    self.plan()
                self.db.execute("UPDATE agent SET role='production',content_tags='[\"source-tag\"]' WHERE id=1")

    def test_replay_accepts_only_exact_pre_duty_snapshot(self):
        sql = asset_sql(self.plan())
        self.assertIn("NOT (mapping.native_snapshot->'config' ? 'role')", sql)
        self.assertIn("NOT (mapping.native_snapshot->'config' ? 'content_tags')", sql)
        self.assertIn("target #- '{config,role}' #- '{config,content_tags}'", sql)

    def test_owner_zero_resolves_pinned_admin_and_private_owner_stays_separate(self):
        plan = self.plan()
        self.assertEqual(plan["agents"][0]["target"]["created_by"], ADMIN_TARGET_ID)
        self.assertEqual(plan["skills"][0]["target"]["created_by"], ADMIN_TARGET_ID)
        self.assertEqual(plan["agents"][1]["target"]["created_by"], USER_ID)
        self.assertEqual(plan["agents"][1]["target"]["visibility"], "private")
        self.assertFalse(plan["agents"][1]["target"]["enabled"])
        self.assertFalse(plan["agents"][0]["target"]["is_builtin"])

    def test_owner_checks_include_template_owners_but_exclude_unrelated_identities(self):
        unrelated = "d574dd78-7ca7-41e1-ae19-e2874665a183"
        template_owner = "a986bcf9-3325-4ff0-aac7-cbc1d93f5b63"
        self.identities.update({2: unrelated, 8: template_owner})
        self.db.execute("UPDATE agentprompttemplate SET owner_user_id=8 WHERE id=1")

        plan = self.plan()

        self.assertEqual(plan["identity_mapping"], {"1": ADMIN_TARGET_ID, "3": USER_ID, "8": template_owner})
        self.assertEqual(self.identities[2], unrelated)
        self.assertNotIn(unrelated, asset_sql(plan))

    def test_private_cross_owner_binding_is_preserved_but_not_effective(self):
        plan = self.plan()
        self.assertEqual(len(plan["bindings"]), 6)
        blocked = next(b for b in plan["bindings"] if b["source"]["id"] == 102)
        self.assertFalse(blocked["effective"])
        self.assertIn("blocked_private_owner_mismatch", blocked["review"])
        self.assertNotIn(asset_id("skill", 5), plan["agents"][0]["target"]["config"]["instruction_skill_ids"])
        self.assertIn(asset_id("skill", 9), plan["agents"][0]["target"]["config"]["instruction_skill_ids"])
        self.assertIn(asset_id("skill", 5), plan["agents"][2]["target"]["config"]["instruction_skill_ids"])

    def test_global_skill_remains_bound_to_private_other_owner_agent(self):
        plan = self.plan()
        selected = plan["agents"][2]["target"]["config"]["instruction_skill_ids"]
        self.assertIn(asset_id("skill", 1), selected)
        binding = next(b for b in plan["bindings"] if b["source"]["id"] == 105)
        self.assertTrue(binding["effective"])
        self.assertEqual(binding["review"], [])

    def test_native_skill_is_non_executable_and_disabled_flag_is_preserved(self):
        self.db.execute("UPDATE agentskill SET enabled=0 WHERE id=1")
        plan = self.plan()
        self.assertFalse(plan["skills"][0]["target"]["enabled"])
        self.assertTrue(plan["skills"][0]["target"]["instruction_only"])
        self.assertEqual(plan["skills"][0]["target"]["bundle_ref"], "")
        for agent in plan["agents"]:
            self.assertEqual(agent["target"]["config"]["allowed_tools"], [])
            self.assertEqual(agent["target"]["config"]["skills_selection_mode"], "none")

    def test_native_model_uses_reviewed_legacy_fallback(self):
        plan = self.plan()
        for item in plan["agents"]:
            self.assertEqual(item["target"]["config"]["model_id"], model_id(3, "deepseek-flash"))
            self.assertTrue(item["model_resolution"]["model_fallback"])

    def test_selected_kb_is_mapped_and_no_kb_becomes_none(self):
        plan = self.plan()
        self.assertEqual(plan["agents"][0]["target"]["config"]["knowledge_bases"], [KB_ID])
        self.assertEqual(plan["agents"][0]["target"]["config"]["kb_selection_mode"], "selected")
        self.assertEqual(plan["agents"][1]["target"]["config"]["kb_selection_mode"], "none")

    def test_readonly_and_no_credentials_read(self):
        def authorize(action, table, column, database, trigger):
            if action == sqlite3.SQLITE_READ and table == "modelprovider" and column == "api_key":
                return sqlite3.SQLITE_DENY
            return sqlite3.SQLITE_OK
        self.db.set_authorizer(authorize)
        self.db.execute("PRAGMA query_only=ON")
        plan = self.plan()
        self.assertNotIn("DO-NOT-READ-SECRET", json.dumps(plan))

    def test_bad_identity_or_organization_rejected(self):
        for identity in ({3: USER_ID}, {1: USER_ID, 3: ADMIN_TARGET_ID}, {1: ADMIN_TARGET_ID}):
            with self.subTest(identity=identity), self.assertRaises(ValueError):
                asset_plan(self.db, identity, self.kbs, model_plan(self.db))
        self.db.execute("UPDATE agent SET organization_id=2 WHERE id=19")
        with self.assertRaisesRegex(ValueError, "organization"):
            self.plan()

    def test_unknown_visibility_rejected(self):
        self.db.execute("UPDATE agentskill SET visibility='shared' WHERE id=5")
        with self.assertRaisesRegex(ValueError, "visibility"):
            self.plan()

    def test_explicit_grants_require_review(self):
        self.db.execute("INSERT INTO agentskillgrant VALUES (1,5,1,'use',1,'2026-09-15')")
        with self.assertRaisesRegex(ValueError, "explicit grants"):
            self.plan()

    def test_missing_and_duplicate_binding_rejected(self):
        self.db.execute("INSERT INTO agentskillbinding VALUES (200,1,1)")
        with self.assertRaisesRegex(ValueError, "duplicated"):
            self.plan()
        self.db.execute("DELETE FROM agentskillbinding WHERE id=200")
        self.db.execute("INSERT INTO agentskillbinding VALUES (200,777,1)")
        with self.assertRaisesRegex(ValueError, "missing asset"):
            self.plan()

    def test_unmapped_kb_or_category_scope_never_widens(self):
        self.db.execute("UPDATE agent SET kb_ids='[7]' WHERE id=1")
        with self.assertRaisesRegex(ValueError, "unmapped"):
            self.plan()
        self.db.execute("UPDATE agent SET kb_ids='[3]',kb_category_ids='{\"3\":[4]}' WHERE id=1")
        with self.assertRaisesRegex(ValueError, "category restrictions"):
            self.plan()

    def test_missing_legacy_template_rejected(self):
        self.db.execute("DELETE FROM agentprompttemplate")
        with self.assertRaisesRegex(ValueError, "missing prompt template"):
            self.plan()

    def test_plan_is_deterministic_and_source_ids_are_namespaced(self):
        self.assertEqual(self.plan(), self.plan())
        self.assertNotEqual(asset_id("agent", 1), asset_id("skill", 1))
        self.assertNotEqual(asset_id("agent", 1), asset_id("agent", 19))

    def test_sql_requires_fingerprint(self):
        for digest in (None, "", "A" * 64, "a" * 63):
            plan = self.plan()
            plan["source_snapshot_sha256"] = digest
            with self.subTest(digest=digest), self.assertRaises(ValueError):
                asset_sql(plan)

    def test_tampered_targets_or_access_fail_before_sql(self):
        mutations = [
            lambda p: p.update(target_tenant_id=42),
            lambda p: p.update(target_database="production"),
            lambda p: p["agents"][1]["target"].update(enabled=True),
            lambda p: p["agents"][1]["target"].update(visibility="global"),
            lambda p: p["skills"][1]["target"].update(instruction_only=False),
            lambda p: p["agents"][0]["target"]["config"].update(allowed_tools=["execute"]),
            lambda p: p["agents"][0]["target"]["config"].update(system_prompt="changed"),
            lambda p: p["bindings"][1].update(effective=True),
            lambda p: p["bindings"][1].update(review=[]),
        ]
        for mutate in mutations:
            plan = self.plan()
            mutate(plan)
            with self.subTest(mutate=mutate), self.assertRaises(ValueError):
                asset_sql(plan)

    def test_safe_summary_contains_no_prompt_bodies(self):
        summary = safe_asset_summary(self.plan())
        self.assertEqual(summary["counts"]["bindings"], 6)
        self.assertEqual(summary["effective_bindings"], 5)
        self.assertEqual(summary["blocked_bindings"], 1)
        self.assertEqual(summary["disabled_agent_ids"], [19])
        self.assertNotIn("Original", json.dumps(summary))
        self.assertNotIn("frozen", json.dumps(summary))
        self.assertNotIn("template-body", json.dumps(summary))

    def test_sql_is_guarded_insert_only_and_keeps_tombstones_and_user_edits(self):
        sql = asset_sql(self.plan())
        self.assertIn("current_database()", sql)
        self.assertIn("pg_advisory_xact_lock", sql)
        self.assertIn("Asset owner durable identity or membership mismatch", sql)
        self.assertIn("Asset durable mapping conflict", sql)
        self.assertIn("refusing recreation", sql)
        self.assertIn("Native skill ID or owner/name collision", sql)
        self.assertIn("Native Agent ID or owner/name collision", sql)
        self.assertIn("effective boolean NOT NULL", sql)
        self.assertIn("SET LOCAL log_min_error_statement = 'panic'", sql)
        self.assertIn(f"CREATE TABLE IF NOT EXISTS {LEDGER}", sql)
        self.assertIn(f"CREATE TABLE IF NOT EXISTS {BINDING_LEDGER}", sql)
        self.assertNotIn("ON CONFLICT", sql.upper())
        self.assertNotIn("UPDATE CUSTOM_AGENTS", sql.upper())
        self.assertNotIn("UPDATE TENANT_SKILL_CATALOG", sql.upper())
        self.assertNotIn("COMMIT", sql.upper())
        self.assertNotIn("BEGIN TRANSACTION", sql.upper())
        self.assertNotIn("SELECT value FROM jsonb_each_text", sql)

    def test_text_is_sql_escaped_without_semantic_transformation(self):
        value = "  O'Brien\\line\n$assets$; SELECT 'opaque data';  "
        self.db.execute("UPDATE agent SET rules=? WHERE id=1", (value,))
        plan = self.plan()
        self.assertEqual(plan["agents"][0]["target"]["config"]["system_prompt"], value)
        sql = asset_sql(plan)
        self.assertIn("O''Brien", sql)
        self.assertIn("standard_conforming_strings = on", sql)


if __name__ == "__main__":
    unittest.main()