admin/yawan-workbench
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server): add workbench API, DSH planner worker and RAG pipeline

Browse Source
This commit is contained in:
yi
2026-09-16 19:06:20 +08:00
parent 72fc5b47c7
commit d4b5b3f43a
122 changed files with 31906 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/training_corpus.py
+676
View File
@@ -0,0 +1,676 @@
"""训练营全库分批阅读与磁盘来源树；业务数据库只接收有界的最终证据。
build_corpus(session,row,summarize,progress) -> (evidence,coverage)
summarize(evidence,max_chars) -> str；输入 JSON <=16000，最终证据 JSON <=40000。
小库原文直接交给 planner：queued_raw_chunks 在 planner 成功后才计入 read_chunks。
coverage 的 phase/stage、per_kb/kbs 为兼容别名；complete 由 planner 成功后确认。
"""
from __future__ import annotations
import copy
import hashlib
import json
import os
import re
import shutil
import sqlite3
import uuid
import threading
import time
from collections import deque
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeout
from pathlib import Path
from sqlmodel import Session, select
from .config import DATA_DIR
from .models import KnowledgeBase, KnowledgeFile
from .rag.store import iter_chunks
from . import training_knowledge as knowledge
CORPUS_DIR = DATA_DIR / "training-corpus"
BATCH_CHARS = 16000
FINAL_CHARS = 40000
_SUFFIXES = {".doc", ".docx", ".srt", ".md", ".txt", ".json", ".jsonl", ".pdf",
".mp4", ".mov", ".m4a", ".mp3", ".wav", ".flv", ".mkv", ".avi",
".webm", ".aac", ".ogg", ".flac"}
def _json(value):
# 与规划模型的默认 json.dumps 序列化保持相同预算，不依赖紧凑 separators。
return json.dumps(value, ensure_ascii=False)
def _positive(value, label):
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
raise ValueError(f"{label}必须是正整数")
return value
def _path(draft_id, version):
_positive(draft_id, "导图编号")
_positive(version, "版本")
directory = CORPUS_DIR / str(draft_id)
if directory.resolve().parent != CORPUS_DIR.resolve():
raise ValueError("资料来源路径无效")
path = directory / f"v{version}.sqlite"
if path.resolve().parent != directory.resolve():
raise ValueError("资料来源路径无效")
return path
def _connect(path, *, readonly=False):
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True) if readonly else sqlite3.connect(path)
connection.row_factory = sqlite3.Row
return connection
def _schema(connection):
connection.executescript("""
CREATE TABLE originals (id INTEGER PRIMARY KEY, kb_id INTEGER NOT NULL,
kb_name TEXT NOT NULL, file TEXT NOT NULL, chunk_id TEXT NOT NULL,
text TEXT NOT NULL, digest TEXT NOT NULL, UNIQUE(kb_id,chunk_id));
CREATE TABLE nodes (seq INTEGER PRIMARY KEY, id TEXT UNIQUE, kb_id INTEGER NOT NULL,
kb_name TEXT NOT NULL, file TEXT NOT NULL, chunk_id TEXT NOT NULL, text TEXT NOT NULL,
kind TEXT NOT NULL, source_count INTEGER NOT NULL, level INTEGER NOT NULL,
original_id INTEGER, consumed INTEGER DEFAULT 0, reused INTEGER DEFAULT 0,
source_seq INTEGER, heading TEXT DEFAULT '');
CREATE INDEX nodes_kb_level ON nodes(kb_id,level,seq);
CREATE INDEX nodes_original ON nodes(original_id,consumed);
CREATE INDEX nodes_file_chunk ON nodes(kb_id,level,file,chunk_id,seq);
CREATE TABLE edges (parent TEXT NOT NULL, child TEXT NOT NULL, PRIMARY KEY(parent,child));
CREATE INDEX edges_child ON edges(child);
CREATE TABLE files (kb_id INTEGER NOT NULL, name TEXT NOT NULL, registered INTEGER DEFAULT 0,
physical INTEGER DEFAULT 0, indexed INTEGER DEFAULT 0, digest TEXT DEFAULT '', stale INTEGER DEFAULT 0,
PRIMARY KEY(kb_id,name));
""")
def _evidence(row):
return {key: row[key] for key in ("id", "kb_id", "kb_name", "file", "chunk_id", "text", "kind", "source_count", "source_seq", "heading") if key in row.keys()}
def _insert_node(db, kb, text, *, kind, level, file="", chunk_id="", original_id=None, children=(), source_seq=None, heading=""):
cursor = db.execute("""INSERT INTO nodes(kb_id,kb_name,file,chunk_id,text,kind,source_count,level,original_id,source_seq,heading)
VALUES(?,?,?,?,?,?,?,?,?,?,?)""",
(kb.id, kb.name, file, chunk_id, text, kind, 1, level, original_id, source_seq, heading))
identifier = f"E{cursor.lastrowid}"
db.execute("UPDATE nodes SET id=? WHERE seq=?", (identifier, cursor.lastrowid))
if children:
db.executemany("INSERT INTO edges(parent,child) VALUES(?,?)", ((identifier, child) for child in children))
count = db.execute(_source_cte() + " SELECT COUNT(*) FROM raw_sources", (identifier,)).fetchone()[0]
db.execute("UPDATE nodes SET source_count=? WHERE id=?", (count, identifier))
return _evidence(db.execute("SELECT * FROM nodes WHERE id=?", (identifier,)).fetchone())
def _source_cte():
return """WITH RECURSIVE descendants(id) AS (
SELECT id FROM nodes WHERE id=? UNION SELECT e.child FROM edges e JOIN descendants d ON e.parent=d.id
), raw_sources AS (SELECT DISTINCT n.original_id FROM nodes n JOIN descendants d ON n.id=d.id
WHERE n.original_id IS NOT NULL)"""
def _split_text(kb, chunk_id, file, text, first_seq, *, reusable=False, source_seq=None,
heading="", max_chars=None):
"""按序列化后预算分片，保留全部字符；来源仍指向完整原块。"""
# Automatic pre-reading can use a slightly larger request envelope than
# interactive training-map assessment. Keep the historical 16k default
# for every existing caller and validate an override at the boundary so a
# provider cannot receive an accidentally unbounded prompt.
batch_chars = BATCH_CHARS if max_chars is None else max(BATCH_CHARS, min(int(max_chars), 32000))
offset, part = 0, 0
while offset < len(text):
sample = {"id": f"E{first_seq + part}", "kb_id": kb.id, "kb_name": kb.name,
"file": file, "chunk_id": chunk_id, "kind": "raw", "source_count": 1, "text": "", "source_seq": source_seq, "heading": heading}
if reusable:
sample = knowledge.canonical_entries([sample])[0]
if len(_json([sample])) >= batch_chars:
raise ValueError(f"知识库「{kb.name}」来源元数据过长，无法安全分批")
low, high = 1, min(len(text) - offset, batch_chars)
while low < high:
mid = (low + high + 1) // 2
sample["text"] = text[offset:offset + mid]
if len(_json([sample])) <= batch_chars:
low = mid
else:
high = mid - 1
sample["text"] = text[offset:offset + low]
if len(_json([sample])) > batch_chars:
raise ValueError("资料字符无法放入分批预算")
yield sample["text"]
offset += low
part += 1
def _inventory(session, db, kb, allowed_files=None, excluded_files=None):
excluded_files = excluded_files or set()
offset = 0
while True:
records = session.exec(select(KnowledgeFile).where(KnowledgeFile.kb_id == kb.id)
.order_by(KnowledgeFile.id).offset(offset).limit(256)).all()
if not records:
break
for row in records:
if row.file_name in excluded_files or (allowed_files is not None and row.file_name not in allowed_files):
continue
db.execute("INSERT INTO files(kb_id,name,registered,digest) VALUES(?,?,1,?) "
"ON CONFLICT(kb_id,name) DO UPDATE SET registered=1,digest=excluded.digest",
(kb.id, row.file_name, row.content_hash or ""))
offset += len(records)
if kb.source_dir:
directory = Path(kb.source_dir)
if directory.is_dir():
# 现有入库过程把源文件平铺在 KB source_dir，遍历不跟随目录或文件符号链接。
with os.scandir(directory) as entries:
for entry in entries:
if entry.is_file(follow_symlinks=False) and Path(entry.name).suffix.lower() in _SUFFIXES and entry.name not in excluded_files and (allowed_files is None or entry.name in allowed_files):
db.execute("INSERT INTO files(kb_id,name,physical) VALUES(?,?,1) "
"ON CONFLICT(kb_id,name) DO UPDATE SET physical=1", (kb.id, entry.name))
def _refresh_counts(db, coverage):
for kb in coverage["per_kb"]:
counts = db.execute("SELECT COUNT(*),SUM(indexed),SUM(CASE WHEN indexed=0 THEN 1 ELSE 0 END) "
"FROM files WHERE kb_id=?", (kb["kb_id"],)).fetchone()
kb["total_files"], kb["indexed_files"] = counts[0], counts[1] or 0
kb["unindexed_files"] = [row[0] for row in db.execute(
"SELECT name FROM files WHERE kb_id=? AND indexed=0 ORDER BY name", (kb["kb_id"],))]
kb["read_chunks"] = db.execute("""SELECT COUNT(*) FROM originals o WHERE o.kb_id=?
AND EXISTS(SELECT 1 FROM nodes n WHERE n.original_id=o.id)
AND NOT EXISTS(SELECT 1 FROM nodes n WHERE n.original_id=o.id AND n.consumed=0)""",
(kb["kb_id"],)).fetchone()[0]
kb["reused_read_chunks"] = db.execute("""SELECT COUNT(*) FROM originals o WHERE o.kb_id=?
AND EXISTS(SELECT 1 FROM nodes n WHERE n.original_id=o.id)
AND NOT EXISTS(SELECT 1 FROM nodes n WHERE n.original_id=o.id AND (n.consumed=0 OR n.reused=0))""",
(kb["kb_id"],)).fetchone()[0]
kb["new_read_chunks"] = kb["read_chunks"] - kb["reused_read_chunks"]
for field in ("total_files", "indexed_files", "scanned_chunks", "read_chunks", "new_read_chunks", "reused_read_chunks"):
coverage[field] = sum(row[field] for row in coverage["per_kb"])
coverage["unindexed_files"] = sum(len(row["unindexed_files"]) for row in coverage["per_kb"])
coverage["kbs"] = copy.deepcopy(coverage["per_kb"])
coverage["stage"] = coverage["phase"]
coverage["processed_batches"] = coverage["batches_done"]
def _source_seq(meta):
for key in ("seq", "source_seq"):
value = meta.get(key)
if isinstance(value, int) and not isinstance(value, bool) and value >= 0:
return value
return None
def _reading_entries(entries):
"""Remove only exact adjoining-window overlap; all source IDs remain present."""
output, previous = [], None
for entry in entries:
current = dict(entry)
sequence = entry.get("source_seq")
if (previous and sequence is not None and previous.get("source_seq") is not None
and sequence == previous["source_seq"] + 1
and entry["file"] == previous["file"]
and entry.get("heading", "") == previous.get("heading", "")
and entry.get("kb_id") == previous.get("kb_id")):
left, right = previous["text"], entry["text"]
maximum = min(len(left) // 3, len(right) // 3, 400)
for count in range(maximum, 31, -1):
if left[-count:] == right[:count]:
current["text"] = right[count:]
break
output.append(current)
previous = entry
return output
def _batches(db, kb_id, level, *, reusable=False, batch_chars=None, preserve_files=True):
# ``batch_chars`` is opt-in so assessment and its existing tests retain the
# 16k safety envelope. Auto-reading can raise it after checking the model
# context budget, reducing the number of expensive model calls.
budget = BATCH_CHARS if batch_chars is None else max(BATCH_CHARS, min(int(batch_chars), 32000))
current, previous_file = [], None
ordering = ("file,source_seq IS NULL,source_seq," + ("chunk_id,seq" if reusable else "seq")) if level == 0 else "seq"
def size(entries):
return len(_json(knowledge.canonical_entries(_reading_entries(entries)) if reusable and level == 0 else _reading_entries(entries) if level == 0 else entries))
for row in db.execute(f"SELECT * FROM nodes WHERE kb_id=? AND level=? ORDER BY {ordering}", (kb_id, level)):
evidence = _evidence(row)
if current and ((level == 0 and preserve_files and previous_file != evidence["file"]) or size(current + [evidence]) > budget):
yield _reading_entries(current) if level == 0 else current
current = []
if size([evidence]) > budget:
raise ValueError("单份摘要超出分批预算")
current.append(evidence)
previous_file = evidence["file"]
if current:
yield _reading_entries(current) if level == 0 else current
def build_corpus(session: Session, row, summarize, progress, *, parallelism=1, should_stop=None, cache_signature=""):
"""扫描所有所选 KB，逐批阅读后保留各库代表；不写业务库或知识消费状态。"""
path = _path(row.id, row.version)
parallelism = min(8, _positive(parallelism, "阅读并发数"))
reusable = (getattr(row, "coverage", {}) or {}).get("mode") == "reuse"
final_chars = min(FINAL_CHARS, 32000) if reusable else FINAL_CHARS
stopped = threading.Event()
# should_stop must be thread safe; the integration reads cancellation with an
# independent business session. No worker ever accesses `session`, `row` or `db`.
def stopping():
return stopped.is_set() or bool(should_stop and should_stop())
def check_stop():
knowledge.check_stop(stopping)
from .training_scope import normalize_scope, file_scope
mapping = normalize_scope(session, row.kb_ids, getattr(row, "kb_category_ids", {}) or {})
selected = list(dict.fromkeys(_positive(kid, "知识库编号") for kid in row.kb_ids))
if not selected or len(selected) > 20:
raise ValueError("请选择 1 至 20 个知识库")
path.parent.mkdir(parents=True, exist_ok=True)
staging = path.with_name(f"v{row.version}.building-{uuid.uuid4().hex}.sqlite")
coverage = {"mode": "reuse" if reusable else "full", "phase": "scanning", "stage": "scanning", "scanned_chunks": 0,
"read_chunks": 0, "duplicate_chunks": 0, "empty_chunks": 0, "total_files": 0,
"indexed_files": 0, "unindexed_files": 0, "batches_done": 0, "per_kb": [], "kbs": [],
"warnings": [], "complete": False, "source_complete": True,
"queued_raw_chunks": 0, "evidence_count": 0, "kb_category_ids": mapping,
"new_read_chunks": 0, "reused_read_chunks": 0, "new_read_batches": 0,
"cached_batches": 0, "parallelism": parallelism, "active_batches": 0, "total_read_batches": 0,
# Layered digest metrics: level-0 raw batches may be reused from
# the exact topic/model cache while generic pre-reading keeps its
# own manifest. Expose both counters so operators can distinguish
# a true incremental run from a fresh model pass.
"summary_cache_hits": 0, "summary_cache_misses": 0,
"digest_strategy": "layered_incremental",
"summary_cache_scope": "batch_digest_topic_model_scope",
"scope_label": "所选分类（含子分类）" if mapping else "所选知识库全部分类"}
db = _connect(staging)
def report():
_refresh_counts(db, coverage)
db.commit()
if progress:
progress(copy.deepcopy(coverage))
try:
_schema(db)
libraries, index_tokens = [], {}
for kid in selected:
check_stop()
kb = session.get(KnowledgeBase, kid)
if not kb:
raise ValueError(f"所选知识库 {kid} 已不存在")
row_org = getattr(row, "organization_id", None)
if row_org is not None and kb.organization_id not in (0, row_org):
raise ValueError("所选知识库不属于当前组织")
libraries.append(kb)
index_tokens[kid] = knowledge.index_revision(kid) if reusable else ""
info = {"kb_id": kb.id, "kb_name": kb.name, "index_status": kb.index_status,
"total_files": 0, "indexed_files": 0, "scanned_chunks": 0, "read_chunks": 0,
"duplicate_chunks": 0, "empty_chunks": 0, "queued_raw_chunks": 0,
"unindexed_files": [], "new_read_chunks": 0, "reused_read_chunks": 0,
"new_read_batches": 0, "cached_batches": 0}
coverage["per_kb"].append(info)
allowed_files = file_scope(session, kid, mapping)
from .rag.quarantine import file_exclusions
excluded_files = file_exclusions(session, kid)
# Whole-library inventory must retain not-yet-indexed sources, even
# when retrieval has a positive whitelist to exclude a test index.
_inventory(session, db, kb, allowed_files if str(kid) in mapping else None, excluded_files)
report()
if kb.index_status == "indexing":
raise ValueError(f"知识库「{kb.name}」正在建立索引，请完成后重新评估")
if kb.index_status != "ready":
coverage["source_complete"] = False
coverage["warnings"].append(f"知识库「{kb.name}」索引状态为 {kb.index_status}，当前资料范围可能不完整")
for chunk in iter_chunks(kid, **({"file_names": allowed_files} if allowed_files is not None else {})):
if info["scanned_chunks"] % 128 == 0:
check_stop()
info["scanned_chunks"] += 1
text, meta = chunk.get("text"), chunk.get("meta") or {}
identifier = str(chunk.get("id") or "")
if not identifier or not isinstance(text, str) or not text.strip() or not isinstance(meta, dict):
coverage["empty_chunks"] += 1
info["empty_chunks"] += 1
raise ValueError(f"知识库「{kb.name}」包含空正文或来源损坏的索引片段")
if meta.get("kb_id") is not None and str(meta["kb_id"]) != str(kid):
raise ValueError(f"知识库「{kb.name}」索引片段的归属不匹配")
file = str(meta.get("file") or "未记录源文件")
if allowed_files is not None and file not in allowed_files:
raise ValueError("索引返回了所选分类以外的文件")
digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
existing = db.execute("SELECT digest,file FROM originals WHERE kb_id=? AND chunk_id=?", (kid, identifier)).fetchone()
if existing:
if existing[0] != digest or existing[1] != file:
raise ValueError(f"知识库「{kb.name}」存在编号相同但正文冲突的片段")
coverage["duplicate_chunks"] += 1
info["duplicate_chunks"] += 1
continue
cursor = db.execute("INSERT INTO originals(kb_id,kb_name,file,chunk_id,text,digest) VALUES(?,?,?,?,?,?)",
(kid, kb.name, file, identifier, text, digest))
original_id = cursor.lastrowid
db.execute("INSERT INTO files(kb_id,name,indexed) VALUES(?,?,1) "
"ON CONFLICT(kb_id,name) DO UPDATE SET indexed=1", (kid, file))
if meta.get("file_hash"):
db.execute("UPDATE files SET stale=1 WHERE kb_id=? AND name=? AND digest<>'' AND digest<>?",
(kid, file, str(meta["file_hash"])))
first_seq = db.execute("SELECT COALESCE(MAX(seq),0)+1 FROM nodes").fetchone()[0]
for part in _split_text(kb, identifier, file, text, first_seq, reusable=reusable,
source_seq=_source_seq(meta), heading=str(meta.get("heading") or "")):
_insert_node(db, kb, part, kind="raw", level=0, file=file, chunk_id=identifier,
original_id=original_id, source_seq=_source_seq(meta),
heading=str(meta.get("heading") or ""))
if info["scanned_chunks"] % 128 == 0:
report()
if not db.execute("SELECT 1 FROM originals WHERE kb_id=? LIMIT 1", (kid,)).fetchone():
raise ValueError(f"知识库「{kb.name}」没有可读取的索引正文，请先完成索引")
report()
for kb in coverage["per_kb"]:
if kb["unindexed_files"]:
coverage["source_complete"] = False
coverage["warnings"].append(f"知识库「{kb['kb_name']}」有 {len(kb['unindexed_files'])} 个文件尚无索引正文，未参与 AI 阅读")
stale = db.execute("SELECT COUNT(*) FROM files WHERE kb_id=? AND stale=1", (kb["kb_id"],)).fetchone()[0]
if stale:
coverage["source_complete"] = False
coverage["warnings"].append(f"知识库「{kb['kb_name']}」有 {stale} 个文件的索引与源文件版本不一致")
source_dir = next(item.source_dir for item in libraries if item.id == kb["kb_id"])
missing = [item[0] for item in db.execute("SELECT name FROM files WHERE kb_id=? AND registered=1 AND physical=0",
(kb["kb_id"],))] if source_dir else []
kb["missing_source_files"] = missing
if missing:
coverage["source_complete"] = False
coverage["warnings"].append(f"知识库「{kb['kb_name']}」有 {len(missing)} 个登记源文件在目录中不存在；现有索引已保留，但源文件完整性无法确认")
raw_size = 2
small = []
for node in db.execute("SELECT * FROM nodes ORDER BY kb_id,file,source_seq IS NULL,source_seq,seq"):
evidence = _evidence(node)
raw_size += len(_json(evidence)) + (2 if small else 0)
if raw_size > BATCH_CHARS:
small = None
break
small.append(evidence)
if small is not None and not reusable:
result = small
coverage["queued_raw_chunks"] = db.execute("SELECT COUNT(*) FROM originals").fetchone()[0]
for kb in coverage["per_kb"]:
kb["queued_raw_chunks"] = db.execute("SELECT COUNT(*) FROM originals WHERE kb_id=?",
(kb["kb_id"],)).fetchone()[0]
else:
result = []
quota = (final_chars - 2) // len(libraries) - 2
coverage["total_read_batches"] = sum(1 for kb in libraries for _ in _batches(db, kb.id, 0, reusable=reusable))
report()
with ThreadPoolExecutor(max_workers=parallelism, thread_name_prefix="corpus-read") as executor:
for kb in libraries:
level = 0
info = next(info for info in coverage["per_kb"] if info["kb_id"] == kb.id)
organization = getattr(row, "organization_id", None)
organization = organization if organization is not None else kb.organization_id or 1
while True:
check_stop()
coverage["phase"] = "reading" if level == 0 else "reducing"
report() # Publish the phase before any worker can contact a model.
batch_count = 0
generic = reusable and level == 0
target = knowledge.GENERIC_TARGET if generic else min(2400, quota - 600 - len(kb.name))
if target < 100:
raise ValueError("知识库名称过长，无法保留各库摘要")
manifests = {}
# All SQLite iteration and inserts remain on the coordinator.
batches = iter(_batches(db, kb.id, level, reusable=reusable))
pending = deque()
def run(batch, *, generic=generic, target=target, organization=organization, kid=kb.id):
check_stop()
if generic:
entries = knowledge.canonical_entries(batch)
return knowledge.reusable_summary(organization, kid, entries, target, cache_signature,
lambda: summarize(entries, target), stopping)
# Topic scoped summaries are cached by the exact
# batch digest, topic, selected categories and model
# signature. This is the second layer of the digest
# pipeline: unchanged batches on a new assessment or
# retry skip the model call while changed batches are
# regenerated independently.
summary, reused = cached_summary(
row, batch, target, cache_signature,
lambda: summarize(batch, target))
check_stop()
knowledge._validate(summary, target)
return summary.strip(), reused, ""
def raise_failed():
# A later request can fail before the oldest finishes.
# Notice it promptly and never refill a known failed queue.
for _, active in pending:
if active.cancelled():
raise knowledge.ReadingCancelled("资料阅读已取消")
if active.done():
error = active.exception()
if error is not None:
raise error
def schedule():
check_stop()
raise_failed()
batch = next(batches, None)
if batch is None:
return False
pending.append((batch, executor.submit(run, batch)))
coverage["active_batches"] = len(pending)
return True
try:
while len(pending) < parallelism and schedule():
pass
report()
waiting_report_at = time.monotonic() + 2
while pending:
check_stop()
raise_failed()
batch, future = pending[0]
try:
summary, cached, key = future.result(timeout=0.25)
except FutureTimeout:
# Cancellation must not wait for the currently oldest request.
check_stop()
if time.monotonic() >= waiting_report_at:
report()
waiting_report_at = time.monotonic() + 2
continue
check_stop()
pending.popleft()
coverage["active_batches"] = len(pending)
_insert_node(db, kb, summary, kind="summary", level=level + 1,
file="通用知识阅读摘要" if generic else "全库分批阅读摘要",
chunk_id=f"summary-level-{level + 1}",
children=[item["id"] for item in batch])
if level == 0:
db.executemany("UPDATE nodes SET consumed=1,reused=? WHERE id=?",
((int(cached), item["id"]) for item in batch))
metric = "cached_batches" if cached else "new_read_batches"
coverage[metric] += 1
info[metric] += 1
coverage["summary_cache_hits" if cached else "summary_cache_misses"] += 1
if generic:
manifests.setdefault(batch[0]["file"], []).append(key)
coverage["batches_done"] += 1
batch_count += 1
schedule()
report()
waiting_report_at = time.monotonic() + 2
except BaseException:
stopped.set()
for _, future in pending:
future.cancel()
coverage["active_batches"] = 0
raise
if generic and index_tokens[kb.id] and index_tokens[kb.id] == knowledge.index_revision(kb.id):
# A manifest certifies a complete file, never a partial successful prefix.
for name, keys in manifests.items():
check_stop()
file = session.exec(select(KnowledgeFile).where(KnowledgeFile.kb_id == kb.id,
KnowledgeFile.file_name == name)).first()
scanned = db.execute("SELECT stale,digest FROM files WHERE kb_id=? AND name=?", (kb.id, name)).fetchone()
if file and not scanned[0] and file.content_hash == scanned[1]:
count = db.execute("SELECT COUNT(*) FROM originals WHERE kb_id=? AND file=?", (kb.id, name)).fetchone()[0]
knowledge.record_manifest(organization, kb.id, name, file.content_hash, file.revision,
count, cache_signature, index_tokens[kb.id], keys)
level += 1
final = []
total = 2
for node in db.execute("SELECT * FROM nodes WHERE kb_id=? AND level=? ORDER BY seq", (kb.id, level)):
item = _evidence(node)
total += len(_json(item)) + (2 if final else 0)
if total > quota:
final = None
break
final.append(item)
if final:
result.extend(final)
break
if batch_count == 0 or level > 24:
raise ValueError(f"知识库「{kb.name}」摘要无法在预算内收敛")
check_stop()
if len(_json(result)) > final_chars:
raise ValueError("全库证据超出规划预算")
coverage["phase"], coverage["evidence_count"] = "planning", len(result)
report()
db.close()
check_stop()
os.replace(staging, path)
return result, coverage
except Exception as exc:
stopped.set()
coverage["active_batches"] = 0
coverage["phase"], coverage["complete"], coverage["source_complete"] = "failed", False, False
coverage["warnings"].append(str(exc))
try:
report()
except Exception:
pass
raise
finally:
db.close()
staging.unlink(missing_ok=True)
def source_page(row, evidence_id, offset=0, limit=20):
"""来源权限由调用者按导图 owner 和所选 KB 核对；这里只读取本版本快照。"""
if not isinstance(evidence_id, str) or not re.fullmatch(r"E[1-9][0-9]*", evidence_id):
raise ValueError("证据编号无效")
if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
raise ValueError("来源分页起点无效")
if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 100:
raise ValueError("来源分页大小必须在 1 至 100 之间")
path = _path(row.id, row.version)
if not path.is_file():
raise ValueError("当前版本的原文快照不存在，请重新评估资料")
with _connect(path, readonly=True) as db:
node = db.execute("SELECT kb_id FROM nodes WHERE id=?", (evidence_id,)).fetchone()
if not node or node[0] not in row.kb_ids:
raise ValueError("证据不属于当前导图所选资料")
cte = _source_cte()
total = db.execute(cte + " SELECT COUNT(*) FROM raw_sources", (evidence_id,)).fetchone()[0]
items = [dict(item) for item in db.execute(cte + """ SELECT o.id,o.kb_id,o.kb_name,o.file,o.chunk_id,o.text
FROM originals o JOIN raw_sources s ON o.id=s.original_id ORDER BY o.id LIMIT ? OFFSET ?""",
(evidence_id, limit, offset))]
for item in items:
item["id"] = f"source-{item['id']}"
item["kind"], item["source_count"] = "raw", 1
return {"items": items, "total": total, "offset": offset, "limit": limit}
def day_evidence(row, evidence, max_chars=FINAL_CHARS):
"""保留传入的全部摘要，再在剩余预算内展开真实原文分片。"""
result = [dict(item) for item in evidence]
if len(_json(result)) > max_chars:
raise ValueError("当天证据摘要超出模型预算")
path = _path(row.id, row.version)
if not path.is_file():
return result # 兼容升级前只存业务库原文的导图。
seen = {item["id"] for item in result}
with _connect(path, readonly=True) as db:
for item in evidence:
if item.get("kind") != "summary" or item.get("kb_id") not in row.kb_ids:
continue
remaining = max_chars - len(_json(result))
if remaining < 240:
break
cte = _source_cte()
for node in db.execute(cte + """ SELECT n.* FROM nodes n JOIN raw_sources s
ON n.original_id=s.original_id WHERE n.kind='raw' AND length(n.text)<=?
ORDER BY n.seq LIMIT 128""", (item["id"], remaining - 200)):
candidate = _evidence(node)
if candidate["id"] not in seen and len(_json(result + [candidate])) <= max_chars:
result.append(candidate)
seen.add(candidate["id"])
if max_chars - len(_json(result)) < 240:
break
return result
def remove_corpus(draft_id):
directory = _path(draft_id, 1).parent
if directory.is_symlink():
raise ValueError("资料来源路径无效")
if directory.exists():
shutil.rmtree(directory)
def cached_summary(row, evidence, max_chars, signature, generate):
"""复用完全相同批次的摘要，跨草稿/版本增量命中。
The cache intentionally lives at the corpus root rather than under a draft
version. A new draft selecting the same material can therefore reuse the
expensive model result. The fingerprint still contains organization/user,
topic, selected KB/category scope, model signature and the complete batch
evidence, preventing cross-tenant or stale-content reuse.
"""
CORPUS_DIR.mkdir(parents=True, exist_ok=True)
path = CORPUS_DIR / 'summary-cache.sqlite'
# Keep this helper usable by legacy/unit-test draft objects that only carry
# id/version/topic. Persisted TrainingMap rows still include the complete
# identity fields, so cross-user or cross-organisation reuse remains isolated.
fingerprint = hashlib.sha256(_json({
'policy': 'training-corpus-summary-v1', 'signature': signature,
'user': getattr(row, 'owner_user_id', 0),
'organization': getattr(row, 'organization_id', 0),
# Keep topic and requested days in the key because the current summary
# prompt may include planning context. This still reuses unchanged
# batches across draft versions and retries for the same assessment.
'topic': getattr(row, 'topic', ''),
'requested_days': getattr(row, 'requested_days', None),
'kb_ids': sorted(set(getattr(row, 'kb_ids', []) or [])),
'kb_category_ids': getattr(row, 'kb_category_ids', {}) or {},
'evidence': evidence, 'max_chars': max_chars,
}).encode('utf-8')).hexdigest()
# A few assessment threads can finish at the same time. WAL + a bounded
# busy timeout keeps cache writes from surfacing as spurious failures while
# never holding the database lock across the model/network call below.
lock_path = CORPUS_DIR / 'summary-cache-locks' / f'{fingerprint[:32]}.lock'
lock_path.parent.mkdir(parents=True, exist_ok=True)
# Per-fingerprint single-flight: identical batches wait for the first model
# call and then reuse its result; unrelated batches remain fully parallel.
try:
import fcntl
except ImportError:
fcntl = None
with lock_path.open('a') as lock_handle:
if fcntl:
fcntl.flock(lock_handle, fcntl.LOCK_EX)
try:
with sqlite3.connect(path, timeout=30) as cache:
cache.execute('PRAGMA busy_timeout=30000')
cache.execute('PRAGMA journal_mode=WAL')
try:
os.chmod(path, 0o600)
except OSError:
pass
cache.execute('CREATE TABLE IF NOT EXISTS summaries (fingerprint TEXT PRIMARY KEY, text TEXT NOT NULL)')
saved = cache.execute('SELECT text FROM summaries WHERE fingerprint=?', (fingerprint,)).fetchone()
if saved and saved[0].strip() and len(saved[0]) <= max_chars:
return saved[0], True
# Keep the cache database transaction short. The per-fingerprint
# lock still prevents duplicate model calls, while unrelated cache
# keys and normal application writes are never blocked by network IO.
summary = generate()
if not isinstance(summary, str) or not summary.strip() or len(summary) > max_chars:
raise ValueError('摘要超出预算或为空，未写入复用记录')
with sqlite3.connect(path, timeout=30) as cache:
cache.execute('PRAGMA busy_timeout=30000')
cache.execute('PRAGMA journal_mode=WAL')
cache.execute('INSERT OR REPLACE INTO summaries VALUES (?,?)', (fingerprint, summary))
return summary, False
finally:
if fcntl:
fcntl.flock(lock_handle, fcntl.LOCK_UN)