feat(server): add workbench API, DSH planner worker and RAG pipeline

2026-09-16 19:06:20 +08:00
parent 72fc5b47c7
commit d4b5b3f43a
122 changed files with 31906 additions and 0 deletions
@@ -0,0 +1,26 @@
+"""清理未完成的 SRT 导入：重置 kb_3 collection + 更新库状态为空"""
+import sys
+sys.path.insert(0, r'C:\Users\xinmiclaw\Desktop\雅文内容创造工作台\server')
+from sqlmodel import Session, select
+from app.db import engine
+from app.models import KnowledgeBase
+from app.rag import store
+
+# 重置向量 collection（防半成品索引损坏）
+try:
+    store.reset_kb(3)
+    print("kb_3 collection 已重置")
+except Exception as e:
+    print("重置异常:", e)
+
+# 更新库状态
+with Session(engine) as s:
+    kb = s.exec(select(KnowledgeBase).where(KnowledgeBase.name == "直播原文库")).first()
+    if kb:
+        kb.index_status = "empty"
+        kb.file_count = 0
+        kb.chunk_count = 0
+        s.add(kb); s.commit()
+        print("直播原文库状态 → empty（以后想导入随时可导）")
+    else:
+        print("未找到直播原文库记录")
@@ -0,0 +1,37 @@
+"""修复：按种子定义给 Agent 设置 kb_ids（库名字 → id）"""
+import sys
+sys.path.insert(0, r'C:\Users\xinmiclaw\Desktop\雅文内容创造工作台\server')
+from sqlmodel import Session, select
+from app.db import engine
+from app.models import Agent, KnowledgeBase
+
+# 与 seed.py AGENTS 定义一致
+AGENT_KBS = {
+    "主控 Agent": [],
+    "开场段 Agent": ["风格档案库"],
+    "心法段 Agent": ["风格档案库", "训练营框架库"],
+    "技法段 Agent": ["风格档案库", "直播原文库"],
+    "产品销售 Agent": ["产品知识库", "风格档案库"],
+    "收尾段 Agent": ["风格档案库"],
+    "钩子段 Agent": ["训练营框架库"],
+    "结构质检": ["质检规则库"],
+    "内容质检": ["质检规则库", "产品知识库"],
+    "风格尺度质检": ["质检规则库"],
+    "跨稿质检": ["备稿库", "训练营框架库"],
+    "组合 Agent": [],
+    "风格顾问": ["风格档案库"],
+    "产品顾问": ["产品知识库"],
+}
+
+with Session(engine) as s:
+    kbs = {k.name: k.id for k in s.exec(select(KnowledgeBase)).all()}
+    print("现有库:", kbs)
+    for a in s.exec(select(Agent)).all():
+        names = AGENT_KBS.get(a.name, [])
+        ids = [kbs[n] for n in names if n in kbs]
+        missing = [n for n in names if n not in kbs]
+        a.kb_ids = ids
+        s.add(a)
+        print(f"{a.name}: {ids}{'（缺库:' + str(missing) + '）' if missing else ''}")
+    s.commit()
+    print("完成")
@@ -0,0 +1,15 @@
+"""补挂：心法/钩子/跨稿质检 挂接训练营框架库(6)"""
+import sys
+sys.path.insert(0, r'C:\Users\xinmiclaw\Desktop\雅文内容创造工作台\server')
+from sqlmodel import Session, select
+from app.db import engine
+from app.models import Agent
+
+with Session(engine) as s:
+    for a in s.exec(select(Agent)).all():
+        if a.name in ("心法段 Agent", "钩子段 Agent", "跨稿质检") and 6 not in (a.kb_ids or []):
+            a.kb_ids = (a.kb_ids or []) + [6]
+            s.add(a)
+            print(f"{a.name} kb_ids → {a.kb_ids}")
+    s.commit()
+    print("完成")
@@ -0,0 +1,30 @@
+"""M2: 导入 73 场 SRT 到直播原文库（后台任务）"""
+import sys
+from pathlib import Path
+sys.path.insert(0, r'C:\Users\xinmiclaw\Desktop\雅文内容创造工作台\server')
+
+from sqlmodel import Session, select
+from app.db import engine, init_db
+from app.models import KnowledgeBase
+from app.rag import pipeline
+
+init_db()
+srt_files = []
+for p in Path(r'E:\03_直播素材\直播合集').rglob('*.srt'):
+    srt_files.append(str(p))
+print(f"找到 {len(srt_files)} 个 SRT")
+
+with Session(engine) as s:
+    kb = s.exec(select(KnowledgeBase).where(KnowledgeBase.name == "直播原文库")).first()
+    if not kb:
+        kb = KnowledgeBase(name="直播原文库", desc="73场雅文直播SRT转录（573万字）")
+        s.add(kb); s.commit(); s.refresh(kb)
+    kid = kb.id
+    print(f"库 id={kid}（复用），开始导入（分块+向量化，预计 30 分钟）...")
+    done = [0]
+    def cb(name, n):
+        done[0] += 1
+        if done[0] % 10 == 0:
+            print(f"  进度: {done[0]}/{len(srt_files)} 文件，累计 {n} 块", flush=True)
+    res = pipeline.import_kb_files(s, kid, srt_files, progress_cb=cb)
+    print("完成:", res)
@@ -0,0 +1,68 @@
+"""独立重建 KB3，避免交互命令超时中止向量索引的持久化。"""
+from __future__ import annotations
+
+import gc
+import json
+import sys
+import time
+from pathlib import Path
+
+SERVER_DIR = Path(__file__).resolve().parents[1]
+if str(SERVER_DIR) not in sys.path:
+    sys.path.insert(0, str(SERVER_DIR))
+
+from sqlmodel import Session
+
+from app.db import engine, init_db
+from app.models import KnowledgeBase, now
+from app.rag import store
+from app.rag.pipeline import rebuild_kb
+
+
+LOG_PATH = SERVER_DIR / "data" / "kb3-rebuild.log"
+
+
+def log(payload: dict) -> None:
+    payload["at"] = now()
+    with LOG_PATH.open("a", encoding="utf-8") as stream:
+        stream.write(json.dumps(payload, ensure_ascii=False) + "\n")
+
+
+def main() -> None:
+    init_db()
+    with Session(engine) as session:
+        kb = session.get(KnowledgeBase, 3)
+        if not kb:
+            raise RuntimeError("KB3 不存在")
+        kb.index_status = "needs_reindex"
+        kb.chunk_count = 0
+        session.add(kb); session.commit()
+        log({"status": "started", "files": kb.file_count})
+        completed = 0
+        def progress(file_name: str, chunks: int) -> None:
+            nonlocal completed
+            completed += 1
+            log({"status": "progress", "completed_files": completed, "file": file_name, "chunks": chunks})
+        result = rebuild_kb(session, 3, progress_cb=progress)
+        # 给持久化 HNSW writer 一个明确的落盘窗口，再以同一独立进程验证计数。
+        gc.collect()
+        time.sleep(5)
+        vector_count = store.count(3)
+        kb = session.get(KnowledgeBase, 3)
+        if vector_count <= 0 or vector_count != kb.chunk_count:
+            kb.index_status = "needs_reindex"
+            session.add(kb); session.commit()
+            log({"status": "verification_failed", "result": result,
+                 "database_chunks": kb.chunk_count, "vector_chunks": vector_count})
+            raise RuntimeError("KB3 向量计数与数据库不一致")
+        kb.index_status = "ready"
+        session.add(kb); session.commit()
+        log({"status": "completed", "result": result, "vector_chunks": vector_count})
+        # Chroma 1.5 的 Rust compactor 在后台写 HNSW；显式停止系统确保索引文件
+        # 在进程退出前完整落盘，避免新进程读取到只有 index_metadata 的半成品。
+        store.get_client()._system.stop()
+        log({"status": "persisted", "vector_chunks": vector_count})
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,16 @@
+"""刷新全部 Agent rules（种子模板更新后重写）"""
+import sys
+sys.path.insert(0, r'C:\Users\xinmiclaw\Desktop\雅文内容创造工作台\server')
+from sqlmodel import Session, select
+from app.db import engine
+from app.models import Agent
+from app.seed import RULES, AGENTS
+
+with Session(engine) as s:
+    for name, role, tags, kb_names, rules_key in AGENTS:
+        a = s.exec(select(Agent).where(Agent.name == name)).first()
+        if a:
+            a.rules = RULES[rules_key].format(name=name) if "{name}" in RULES[rules_key] else RULES[rules_key]
+            s.add(a)
+    s.commit()
+    print("全部 Agent rules 已刷新（含颜色标记规范）")
@@ -0,0 +1,12 @@
+"""M2: 删除损坏的 kb_3 collection（服务器已停，无并发）"""
+import sys
+sys.path.insert(0, r'C:\Users\xinmiclaw\Desktop\雅文内容创造工作台\server')
+from app.rag import store
+
+print("删除前 count:", store.count(3))
+store.reset_kb(3)
+print("已删除 kb_3 collection")
+try:
+    print("删除后 count:", store.count(3))
+except Exception as e:
+    print("删除后 count 异常（应为重建空 collection）:", e)
@@ -0,0 +1,25 @@
+"""插入画布模板（无产品场/纯心法场）"""
+import sys
+sys.path.insert(0, r'C:\Users\xinmiclaw\Desktop\雅文内容创造工作台\server')
+from sqlmodel import Session, select
+from app.db import engine
+from app.models import Canvas
+from app.seed import default_canvas
+
+with Session(engine) as s:
+    exist = {c.name for c in s.exec(select(Canvas)).all()}
+    if "【模板】无产品场" not in exist:
+        no_product = [n for n in default_canvas()["nodes"] if n["node_id"] != "n4"]
+        edges = [e for e in default_canvas()["edges"] if e["from"] != "n4" and e["to"] != "n4"]
+        s.add(Canvas(name="【模板】无产品场", nodes=no_product, edges=edges))
+        print("已插入 【模板】无产品场")
+    if "【模板】纯心法场" not in exist:
+        s.add(Canvas(name="【模板】纯心法场", nodes=[
+            {"node_id": "n1", "agent_id": 2, "enabled": True},
+            {"node_id": "n2", "agent_id": 3, "enabled": True},
+            {"node_id": "n3", "agent_id": 3, "enabled": True},
+            {"node_id": "n5", "agent_id": 6, "enabled": True},
+        ], edges=[{"from": "n1", "to": "n2"}, {"from": "n2", "to": "n3"}, {"from": "n3", "to": "n5"}]))
+        print("已插入 【模板】纯心法场")
+    s.commit()
+    print("当前画布:", [c.name for c in s.exec(select(Canvas)).all()])