677 lines
38 KiB
Python
677 lines
38 KiB
Python
|
"""训练营全库分批阅读与磁盘来源树；业务数据库只接收有界的最终证据。
|
|||
|
|
|
|||
|
|
build_corpus(session,row,summarize,progress) -> (evidence,coverage)
|
|||
|
|
summarize(evidence,max_chars) -> str；输入 JSON <=16000，最终证据 JSON <=40000。
|
|||
|
|
小库原文直接交给 planner：queued_raw_chunks 在 planner 成功后才计入 read_chunks。
|
|||
|
|
coverage 的 phase/stage、per_kb/kbs 为兼容别名；complete 由 planner 成功后确认。
|
|||
|
|
"""
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import copy
|
|||
|
|
import hashlib
|
|||
|
|
import json
|
|||
|
|
import os
|
|||
|
|
import re
|
|||
|
|
import shutil
|
|||
|
|
import sqlite3
|
|||
|
|
import uuid
|
|||
|
|
import threading
|
|||
|
|
import time
|
|||
|
|
from collections import deque
|
|||
|
|
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeout
|
|||
|
|
from pathlib import Path
|
|||
|
|
|
|||
|
|
from sqlmodel import Session, select
|
|||
|
|
|
|||
|
|
from .config import DATA_DIR
|
|||
|
|
from .models import KnowledgeBase, KnowledgeFile
|
|||
|
|
from .rag.store import iter_chunks
|
|||
|
|
from . import training_knowledge as knowledge
|
|||
|
|
|
|||
|
|
CORPUS_DIR = DATA_DIR / "training-corpus"
|
|||
|
|
BATCH_CHARS = 16000
|
|||
|
|
FINAL_CHARS = 40000
|
|||
|
|
_SUFFIXES = {".doc", ".docx", ".srt", ".md", ".txt", ".json", ".jsonl", ".pdf",
|
|||
|
|
".mp4", ".mov", ".m4a", ".mp3", ".wav", ".flv", ".mkv", ".avi",
|
|||
|
|
".webm", ".aac", ".ogg", ".flac"}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _json(value):
|
|||
|
|
# 与规划模型的默认 json.dumps 序列化保持相同预算，不依赖紧凑 separators。
|
|||
|
|
return json.dumps(value, ensure_ascii=False)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _positive(value, label):
|
|||
|
|
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
|||
|
|
raise ValueError(f"{label}必须是正整数")
|
|||
|
|
return value
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _path(draft_id, version):
|
|||
|
|
_positive(draft_id, "导图编号")
|
|||
|
|
_positive(version, "版本")
|
|||
|
|
directory = CORPUS_DIR / str(draft_id)
|
|||
|
|
if directory.resolve().parent != CORPUS_DIR.resolve():
|
|||
|
|
raise ValueError("资料来源路径无效")
|
|||
|
|
path = directory / f"v{version}.sqlite"
|
|||
|
|
if path.resolve().parent != directory.resolve():
|
|||
|
|
raise ValueError("资料来源路径无效")
|
|||
|
|
return path
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _connect(path, *, readonly=False):
|
|||
|
|
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True) if readonly else sqlite3.connect(path)
|
|||
|
|
connection.row_factory = sqlite3.Row
|
|||
|
|
return connection
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _schema(connection):
|
|||
|
|
connection.executescript("""
|
|||
|
|
CREATE TABLE originals (id INTEGER PRIMARY KEY, kb_id INTEGER NOT NULL,
|
|||
|
|
kb_name TEXT NOT NULL, file TEXT NOT NULL, chunk_id TEXT NOT NULL,
|
|||
|
|
text TEXT NOT NULL, digest TEXT NOT NULL, UNIQUE(kb_id,chunk_id));
|
|||
|
|
CREATE TABLE nodes (seq INTEGER PRIMARY KEY, id TEXT UNIQUE, kb_id INTEGER NOT NULL,
|
|||
|
|
kb_name TEXT NOT NULL, file TEXT NOT NULL, chunk_id TEXT NOT NULL, text TEXT NOT NULL,
|
|||
|
|
kind TEXT NOT NULL, source_count INTEGER NOT NULL, level INTEGER NOT NULL,
|
|||
|
|
original_id INTEGER, consumed INTEGER DEFAULT 0, reused INTEGER DEFAULT 0,
|
|||
|
|
source_seq INTEGER, heading TEXT DEFAULT '');
|
|||
|
|
CREATE INDEX nodes_kb_level ON nodes(kb_id,level,seq);
|
|||
|
|
CREATE INDEX nodes_original ON nodes(original_id,consumed);
|
|||
|
|
CREATE INDEX nodes_file_chunk ON nodes(kb_id,level,file,chunk_id,seq);
|
|||
|
|
CREATE TABLE edges (parent TEXT NOT NULL, child TEXT NOT NULL, PRIMARY KEY(parent,child));
|
|||
|
|
CREATE INDEX edges_child ON edges(child);
|
|||
|
|
CREATE TABLE files (kb_id INTEGER NOT NULL, name TEXT NOT NULL, registered INTEGER DEFAULT 0,
|
|||
|
|
physical INTEGER DEFAULT 0, indexed INTEGER DEFAULT 0, digest TEXT DEFAULT '', stale INTEGER DEFAULT 0,
|
|||
|
|
PRIMARY KEY(kb_id,name));
|
|||
|
|
""")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _evidence(row):
|
|||
|
|
return {key: row[key] for key in ("id", "kb_id", "kb_name", "file", "chunk_id", "text", "kind", "source_count", "source_seq", "heading") if key in row.keys()}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _insert_node(db, kb, text, *, kind, level, file="", chunk_id="", original_id=None, children=(), source_seq=None, heading=""):
|
|||
|
|
cursor = db.execute("""INSERT INTO nodes(kb_id,kb_name,file,chunk_id,text,kind,source_count,level,original_id,source_seq,heading)
|
|||
|
|
VALUES(?,?,?,?,?,?,?,?,?,?,?)""",
|
|||
|
|
(kb.id, kb.name, file, chunk_id, text, kind, 1, level, original_id, source_seq, heading))
|
|||
|
|
identifier = f"E{cursor.lastrowid}"
|
|||
|
|
db.execute("UPDATE nodes SET id=? WHERE seq=?", (identifier, cursor.lastrowid))
|
|||
|
|
if children:
|
|||
|
|
db.executemany("INSERT INTO edges(parent,child) VALUES(?,?)", ((identifier, child) for child in children))
|
|||
|
|
count = db.execute(_source_cte() + " SELECT COUNT(*) FROM raw_sources", (identifier,)).fetchone()[0]
|
|||
|
|
db.execute("UPDATE nodes SET source_count=? WHERE id=?", (count, identifier))
|
|||
|
|
return _evidence(db.execute("SELECT * FROM nodes WHERE id=?", (identifier,)).fetchone())
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _source_cte():
|
|||
|
|
return """WITH RECURSIVE descendants(id) AS (
|
|||
|
|
SELECT id FROM nodes WHERE id=? UNION SELECT e.child FROM edges e JOIN descendants d ON e.parent=d.id
|
|||
|
|
), raw_sources AS (SELECT DISTINCT n.original_id FROM nodes n JOIN descendants d ON n.id=d.id
|
|||
|
|
WHERE n.original_id IS NOT NULL)"""
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _split_text(kb, chunk_id, file, text, first_seq, *, reusable=False, source_seq=None,
|
|||
|
|
heading="", max_chars=None):
|
|||
|
|
"""按序列化后预算分片，保留全部字符；来源仍指向完整原块。"""
|
|||
|
|
# Automatic pre-reading can use a slightly larger request envelope than
|
|||
|
|
# interactive training-map assessment. Keep the historical 16k default
|
|||
|
|
# for every existing caller and validate an override at the boundary so a
|
|||
|
|
# provider cannot receive an accidentally unbounded prompt.
|
|||
|
|
batch_chars = BATCH_CHARS if max_chars is None else max(BATCH_CHARS, min(int(max_chars), 32000))
|
|||
|
|
offset, part = 0, 0
|
|||
|
|
while offset < len(text):
|
|||
|
|
sample = {"id": f"E{first_seq + part}", "kb_id": kb.id, "kb_name": kb.name,
|
|||
|
|
"file": file, "chunk_id": chunk_id, "kind": "raw", "source_count": 1, "text": "", "source_seq": source_seq, "heading": heading}
|
|||
|
|
if reusable:
|
|||
|
|
sample = knowledge.canonical_entries([sample])[0]
|
|||
|
|
if len(_json([sample])) >= batch_chars:
|
|||
|
|
raise ValueError(f"知识库「{kb.name}」来源元数据过长，无法安全分批")
|
|||
|
|
low, high = 1, min(len(text) - offset, batch_chars)
|
|||
|
|
while low < high:
|
|||
|
|
mid = (low + high + 1) // 2
|
|||
|
|
sample["text"] = text[offset:offset + mid]
|
|||
|
|
if len(_json([sample])) <= batch_chars:
|
|||
|
|
low = mid
|
|||
|
|
else:
|
|||
|
|
high = mid - 1
|
|||
|
|
sample["text"] = text[offset:offset + low]
|
|||
|
|
if len(_json([sample])) > batch_chars:
|
|||
|
|
raise ValueError("资料字符无法放入分批预算")
|
|||
|
|
yield sample["text"]
|
|||
|
|
offset += low
|
|||
|
|
part += 1
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _inventory(session, db, kb, allowed_files=None, excluded_files=None):
|
|||
|
|
excluded_files = excluded_files or set()
|
|||
|
|
offset = 0
|
|||
|
|
while True:
|
|||
|
|
records = session.exec(select(KnowledgeFile).where(KnowledgeFile.kb_id == kb.id)
|
|||
|
|
.order_by(KnowledgeFile.id).offset(offset).limit(256)).all()
|
|||
|
|
if not records:
|
|||
|
|
break
|
|||
|
|
for row in records:
|
|||
|
|
if row.file_name in excluded_files or (allowed_files is not None and row.file_name not in allowed_files):
|
|||
|
|
continue
|
|||
|
|
db.execute("INSERT INTO files(kb_id,name,registered,digest) VALUES(?,?,1,?) "
|
|||
|
|
"ON CONFLICT(kb_id,name) DO UPDATE SET registered=1,digest=excluded.digest",
|
|||
|
|
(kb.id, row.file_name, row.content_hash or ""))
|
|||
|
|
offset += len(records)
|
|||
|
|
if kb.source_dir:
|
|||
|
|
directory = Path(kb.source_dir)
|
|||
|
|
if directory.is_dir():
|
|||
|
|
# 现有入库过程把源文件平铺在 KB source_dir，遍历不跟随目录或文件符号链接。
|
|||
|
|
with os.scandir(directory) as entries:
|
|||
|
|
for entry in entries:
|
|||
|
|
if entry.is_file(follow_symlinks=False) and Path(entry.name).suffix.lower() in _SUFFIXES and entry.name not in excluded_files and (allowed_files is None or entry.name in allowed_files):
|
|||
|
|
db.execute("INSERT INTO files(kb_id,name,physical) VALUES(?,?,1) "
|
|||
|
|
"ON CONFLICT(kb_id,name) DO UPDATE SET physical=1", (kb.id, entry.name))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _refresh_counts(db, coverage):
|
|||
|
|
for kb in coverage["per_kb"]:
|
|||
|
|
counts = db.execute("SELECT COUNT(*),SUM(indexed),SUM(CASE WHEN indexed=0 THEN 1 ELSE 0 END) "
|
|||
|
|
"FROM files WHERE kb_id=?", (kb["kb_id"],)).fetchone()
|
|||
|
|
kb["total_files"], kb["indexed_files"] = counts[0], counts[1] or 0
|
|||
|
|
kb["unindexed_files"] = [row[0] for row in db.execute(
|
|||
|
|
"SELECT name FROM files WHERE kb_id=? AND indexed=0 ORDER BY name", (kb["kb_id"],))]
|
|||
|
|
kb["read_chunks"] = db.execute("""SELECT COUNT(*) FROM originals o WHERE o.kb_id=?
|
|||
|
|
AND EXISTS(SELECT 1 FROM nodes n WHERE n.original_id=o.id)
|
|||
|
|
AND NOT EXISTS(SELECT 1 FROM nodes n WHERE n.original_id=o.id AND n.consumed=0)""",
|
|||
|
|
(kb["kb_id"],)).fetchone()[0]
|
|||
|
|
kb["reused_read_chunks"] = db.execute("""SELECT COUNT(*) FROM originals o WHERE o.kb_id=?
|
|||
|
|
AND EXISTS(SELECT 1 FROM nodes n WHERE n.original_id=o.id)
|
|||
|
|
AND NOT EXISTS(SELECT 1 FROM nodes n WHERE n.original_id=o.id AND (n.consumed=0 OR n.reused=0))""",
|
|||
|
|
(kb["kb_id"],)).fetchone()[0]
|
|||
|
|
kb["new_read_chunks"] = kb["read_chunks"] - kb["reused_read_chunks"]
|
|||
|
|
for field in ("total_files", "indexed_files", "scanned_chunks", "read_chunks", "new_read_chunks", "reused_read_chunks"):
|
|||
|
|
coverage[field] = sum(row[field] for row in coverage["per_kb"])
|
|||
|
|
coverage["unindexed_files"] = sum(len(row["unindexed_files"]) for row in coverage["per_kb"])
|
|||
|
|
coverage["kbs"] = copy.deepcopy(coverage["per_kb"])
|
|||
|
|
coverage["stage"] = coverage["phase"]
|
|||
|
|
coverage["processed_batches"] = coverage["batches_done"]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _source_seq(meta):
|
|||
|
|
for key in ("seq", "source_seq"):
|
|||
|
|
value = meta.get(key)
|
|||
|
|
if isinstance(value, int) and not isinstance(value, bool) and value >= 0:
|
|||
|
|
return value
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _reading_entries(entries):
|
|||
|
|
"""Remove only exact adjoining-window overlap; all source IDs remain present."""
|
|||
|
|
output, previous = [], None
|
|||
|
|
for entry in entries:
|
|||
|
|
current = dict(entry)
|
|||
|
|
sequence = entry.get("source_seq")
|
|||
|
|
if (previous and sequence is not None and previous.get("source_seq") is not None
|
|||
|
|
and sequence == previous["source_seq"] + 1
|
|||
|
|
and entry["file"] == previous["file"]
|
|||
|
|
and entry.get("heading", "") == previous.get("heading", "")
|
|||
|
|
and entry.get("kb_id") == previous.get("kb_id")):
|
|||
|
|
left, right = previous["text"], entry["text"]
|
|||
|
|
maximum = min(len(left) // 3, len(right) // 3, 400)
|
|||
|
|
for count in range(maximum, 31, -1):
|
|||
|
|
if left[-count:] == right[:count]:
|
|||
|
|
current["text"] = right[count:]
|
|||
|
|
break
|
|||
|
|
output.append(current)
|
|||
|
|
previous = entry
|
|||
|
|
return output
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _batches(db, kb_id, level, *, reusable=False, batch_chars=None, preserve_files=True):
|
|||
|
|
# ``batch_chars`` is opt-in so assessment and its existing tests retain the
|
|||
|
|
# 16k safety envelope. Auto-reading can raise it after checking the model
|
|||
|
|
# context budget, reducing the number of expensive model calls.
|
|||
|
|
budget = BATCH_CHARS if batch_chars is None else max(BATCH_CHARS, min(int(batch_chars), 32000))
|
|||
|
|
current, previous_file = [], None
|
|||
|
|
ordering = ("file,source_seq IS NULL,source_seq," + ("chunk_id,seq" if reusable else "seq")) if level == 0 else "seq"
|
|||
|
|
def size(entries):
|
|||
|
|
return len(_json(knowledge.canonical_entries(_reading_entries(entries)) if reusable and level == 0 else _reading_entries(entries) if level == 0 else entries))
|
|||
|
|
for row in db.execute(f"SELECT * FROM nodes WHERE kb_id=? AND level=? ORDER BY {ordering}", (kb_id, level)):
|
|||
|
|
evidence = _evidence(row)
|
|||
|
|
if current and ((level == 0 and preserve_files and previous_file != evidence["file"]) or size(current + [evidence]) > budget):
|
|||
|
|
yield _reading_entries(current) if level == 0 else current
|
|||
|
|
current = []
|
|||
|
|
if size([evidence]) > budget:
|
|||
|
|
raise ValueError("单份摘要超出分批预算")
|
|||
|
|
current.append(evidence)
|
|||
|
|
previous_file = evidence["file"]
|
|||
|
|
if current:
|
|||
|
|
yield _reading_entries(current) if level == 0 else current
|
|||
|
|
|
|||
|
|
|
|||
|
|
def build_corpus(session: Session, row, summarize, progress, *, parallelism=1, should_stop=None, cache_signature=""):
|
|||
|
|
"""扫描所有所选 KB，逐批阅读后保留各库代表；不写业务库或知识消费状态。"""
|
|||
|
|
path = _path(row.id, row.version)
|
|||
|
|
parallelism = min(8, _positive(parallelism, "阅读并发数"))
|
|||
|
|
reusable = (getattr(row, "coverage", {}) or {}).get("mode") == "reuse"
|
|||
|
|
final_chars = min(FINAL_CHARS, 32000) if reusable else FINAL_CHARS
|
|||
|
|
stopped = threading.Event()
|
|||
|
|
# should_stop must be thread safe; the integration reads cancellation with an
|
|||
|
|
# independent business session. No worker ever accesses `session`, `row` or `db`.
|
|||
|
|
def stopping():
|
|||
|
|
return stopped.is_set() or bool(should_stop and should_stop())
|
|||
|
|
def check_stop():
|
|||
|
|
knowledge.check_stop(stopping)
|
|||
|
|
from .training_scope import normalize_scope, file_scope
|
|||
|
|
mapping = normalize_scope(session, row.kb_ids, getattr(row, "kb_category_ids", {}) or {})
|
|||
|
|
selected = list(dict.fromkeys(_positive(kid, "知识库编号") for kid in row.kb_ids))
|
|||
|
|
if not selected or len(selected) > 20:
|
|||
|
|
raise ValueError("请选择 1 至 20 个知识库")
|
|||
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|||
|
|
staging = path.with_name(f"v{row.version}.building-{uuid.uuid4().hex}.sqlite")
|
|||
|
|
coverage = {"mode": "reuse" if reusable else "full", "phase": "scanning", "stage": "scanning", "scanned_chunks": 0,
|
|||
|
|
"read_chunks": 0, "duplicate_chunks": 0, "empty_chunks": 0, "total_files": 0,
|
|||
|
|
"indexed_files": 0, "unindexed_files": 0, "batches_done": 0, "per_kb": [], "kbs": [],
|
|||
|
|
"warnings": [], "complete": False, "source_complete": True,
|
|||
|
|
"queued_raw_chunks": 0, "evidence_count": 0, "kb_category_ids": mapping,
|
|||
|
|
"new_read_chunks": 0, "reused_read_chunks": 0, "new_read_batches": 0,
|
|||
|
|
"cached_batches": 0, "parallelism": parallelism, "active_batches": 0, "total_read_batches": 0,
|
|||
|
|
# Layered digest metrics: level-0 raw batches may be reused from
|
|||
|
|
# the exact topic/model cache while generic pre-reading keeps its
|
|||
|
|
# own manifest. Expose both counters so operators can distinguish
|
|||
|
|
# a true incremental run from a fresh model pass.
|
|||
|
|
"summary_cache_hits": 0, "summary_cache_misses": 0,
|
|||
|
|
"digest_strategy": "layered_incremental",
|
|||
|
|
"summary_cache_scope": "batch_digest_topic_model_scope",
|
|||
|
|
"scope_label": "所选分类（含子分类）" if mapping else "所选知识库全部分类"}
|
|||
|
|
db = _connect(staging)
|
|||
|
|
def report():
|
|||
|
|
_refresh_counts(db, coverage)
|
|||
|
|
db.commit()
|
|||
|
|
if progress:
|
|||
|
|
progress(copy.deepcopy(coverage))
|
|||
|
|
try:
|
|||
|
|
_schema(db)
|
|||
|
|
libraries, index_tokens = [], {}
|
|||
|
|
for kid in selected:
|
|||
|
|
check_stop()
|
|||
|
|
kb = session.get(KnowledgeBase, kid)
|
|||
|
|
if not kb:
|
|||
|
|
raise ValueError(f"所选知识库 {kid} 已不存在")
|
|||
|
|
row_org = getattr(row, "organization_id", None)
|
|||
|
|
if row_org is not None and kb.organization_id not in (0, row_org):
|
|||
|
|
raise ValueError("所选知识库不属于当前组织")
|
|||
|
|
libraries.append(kb)
|
|||
|
|
index_tokens[kid] = knowledge.index_revision(kid) if reusable else ""
|
|||
|
|
info = {"kb_id": kb.id, "kb_name": kb.name, "index_status": kb.index_status,
|
|||
|
|
"total_files": 0, "indexed_files": 0, "scanned_chunks": 0, "read_chunks": 0,
|
|||
|
|
"duplicate_chunks": 0, "empty_chunks": 0, "queued_raw_chunks": 0,
|
|||
|
|
"unindexed_files": [], "new_read_chunks": 0, "reused_read_chunks": 0,
|
|||
|
|
"new_read_batches": 0, "cached_batches": 0}
|
|||
|
|
coverage["per_kb"].append(info)
|
|||
|
|
allowed_files = file_scope(session, kid, mapping)
|
|||
|
|
from .rag.quarantine import file_exclusions
|
|||
|
|
excluded_files = file_exclusions(session, kid)
|
|||
|
|
# Whole-library inventory must retain not-yet-indexed sources, even
|
|||
|
|
# when retrieval has a positive whitelist to exclude a test index.
|
|||
|
|
_inventory(session, db, kb, allowed_files if str(kid) in mapping else None, excluded_files)
|
|||
|
|
report()
|
|||
|
|
if kb.index_status == "indexing":
|
|||
|
|
raise ValueError(f"知识库「{kb.name}」正在建立索引，请完成后重新评估")
|
|||
|
|
if kb.index_status != "ready":
|
|||
|
|
coverage["source_complete"] = False
|
|||
|
|
coverage["warnings"].append(f"知识库「{kb.name}」索引状态为 {kb.index_status}，当前资料范围可能不完整")
|
|||
|
|
for chunk in iter_chunks(kid, **({"file_names": allowed_files} if allowed_files is not None else {})):
|
|||
|
|
if info["scanned_chunks"] % 128 == 0:
|
|||
|
|
check_stop()
|
|||
|
|
info["scanned_chunks"] += 1
|
|||
|
|
text, meta = chunk.get("text"), chunk.get("meta") or {}
|
|||
|
|
identifier = str(chunk.get("id") or "")
|
|||
|
|
if not identifier or not isinstance(text, str) or not text.strip() or not isinstance(meta, dict):
|
|||
|
|
coverage["empty_chunks"] += 1
|
|||
|
|
info["empty_chunks"] += 1
|
|||
|
|
raise ValueError(f"知识库「{kb.name}」包含空正文或来源损坏的索引片段")
|
|||
|
|
if meta.get("kb_id") is not None and str(meta["kb_id"]) != str(kid):
|
|||
|
|
raise ValueError(f"知识库「{kb.name}」索引片段的归属不匹配")
|
|||
|
|
file = str(meta.get("file") or "未记录源文件")
|
|||
|
|
if allowed_files is not None and file not in allowed_files:
|
|||
|
|
raise ValueError("索引返回了所选分类以外的文件")
|
|||
|
|
digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|||
|
|
existing = db.execute("SELECT digest,file FROM originals WHERE kb_id=? AND chunk_id=?", (kid, identifier)).fetchone()
|
|||
|
|
if existing:
|
|||
|
|
if existing[0] != digest or existing[1] != file:
|
|||
|
|
raise ValueError(f"知识库「{kb.name}」存在编号相同但正文冲突的片段")
|
|||
|
|
coverage["duplicate_chunks"] += 1
|
|||
|
|
info["duplicate_chunks"] += 1
|
|||
|
|
continue
|
|||
|
|
cursor = db.execute("INSERT INTO originals(kb_id,kb_name,file,chunk_id,text,digest) VALUES(?,?,?,?,?,?)",
|
|||
|
|
(kid, kb.name, file, identifier, text, digest))
|
|||
|
|
original_id = cursor.lastrowid
|
|||
|
|
db.execute("INSERT INTO files(kb_id,name,indexed) VALUES(?,?,1) "
|
|||
|
|
"ON CONFLICT(kb_id,name) DO UPDATE SET indexed=1", (kid, file))
|
|||
|
|
if meta.get("file_hash"):
|
|||
|
|
db.execute("UPDATE files SET stale=1 WHERE kb_id=? AND name=? AND digest<>'' AND digest<>?",
|
|||
|
|
(kid, file, str(meta["file_hash"])))
|
|||
|
|
first_seq = db.execute("SELECT COALESCE(MAX(seq),0)+1 FROM nodes").fetchone()[0]
|
|||
|
|
for part in _split_text(kb, identifier, file, text, first_seq, reusable=reusable,
|
|||
|
|
source_seq=_source_seq(meta), heading=str(meta.get("heading") or "")):
|
|||
|
|
_insert_node(db, kb, part, kind="raw", level=0, file=file, chunk_id=identifier,
|
|||
|
|
original_id=original_id, source_seq=_source_seq(meta),
|
|||
|
|
heading=str(meta.get("heading") or ""))
|
|||
|
|
if info["scanned_chunks"] % 128 == 0:
|
|||
|
|
report()
|
|||
|
|
if not db.execute("SELECT 1 FROM originals WHERE kb_id=? LIMIT 1", (kid,)).fetchone():
|
|||
|
|
raise ValueError(f"知识库「{kb.name}」没有可读取的索引正文，请先完成索引")
|
|||
|
|
report()
|
|||
|
|
for kb in coverage["per_kb"]:
|
|||
|
|
if kb["unindexed_files"]:
|
|||
|
|
coverage["source_complete"] = False
|
|||
|
|
coverage["warnings"].append(f"知识库「{kb['kb_name']}」有 {len(kb['unindexed_files'])} 个文件尚无索引正文，未参与 AI 阅读")
|
|||
|
|
stale = db.execute("SELECT COUNT(*) FROM files WHERE kb_id=? AND stale=1", (kb["kb_id"],)).fetchone()[0]
|
|||
|
|
if stale:
|
|||
|
|
coverage["source_complete"] = False
|
|||
|
|
coverage["warnings"].append(f"知识库「{kb['kb_name']}」有 {stale} 个文件的索引与源文件版本不一致")
|
|||
|
|
source_dir = next(item.source_dir for item in libraries if item.id == kb["kb_id"])
|
|||
|
|
missing = [item[0] for item in db.execute("SELECT name FROM files WHERE kb_id=? AND registered=1 AND physical=0",
|
|||
|
|
(kb["kb_id"],))] if source_dir else []
|
|||
|
|
kb["missing_source_files"] = missing
|
|||
|
|
if missing:
|
|||
|
|
coverage["source_complete"] = False
|
|||
|
|
coverage["warnings"].append(f"知识库「{kb['kb_name']}」有 {len(missing)} 个登记源文件在目录中不存在；现有索引已保留，但源文件完整性无法确认")
|
|||
|
|
raw_size = 2
|
|||
|
|
small = []
|
|||
|
|
for node in db.execute("SELECT * FROM nodes ORDER BY kb_id,file,source_seq IS NULL,source_seq,seq"):
|
|||
|
|
evidence = _evidence(node)
|
|||
|
|
raw_size += len(_json(evidence)) + (2 if small else 0)
|
|||
|
|
if raw_size > BATCH_CHARS:
|
|||
|
|
small = None
|
|||
|
|
break
|
|||
|
|
small.append(evidence)
|
|||
|
|
if small is not None and not reusable:
|
|||
|
|
result = small
|
|||
|
|
coverage["queued_raw_chunks"] = db.execute("SELECT COUNT(*) FROM originals").fetchone()[0]
|
|||
|
|
for kb in coverage["per_kb"]:
|
|||
|
|
kb["queued_raw_chunks"] = db.execute("SELECT COUNT(*) FROM originals WHERE kb_id=?",
|
|||
|
|
(kb["kb_id"],)).fetchone()[0]
|
|||
|
|
else:
|
|||
|
|
result = []
|
|||
|
|
quota = (final_chars - 2) // len(libraries) - 2
|
|||
|
|
coverage["total_read_batches"] = sum(1 for kb in libraries for _ in _batches(db, kb.id, 0, reusable=reusable))
|
|||
|
|
report()
|
|||
|
|
with ThreadPoolExecutor(max_workers=parallelism, thread_name_prefix="corpus-read") as executor:
|
|||
|
|
for kb in libraries:
|
|||
|
|
level = 0
|
|||
|
|
info = next(info for info in coverage["per_kb"] if info["kb_id"] == kb.id)
|
|||
|
|
organization = getattr(row, "organization_id", None)
|
|||
|
|
organization = organization if organization is not None else kb.organization_id or 1
|
|||
|
|
while True:
|
|||
|
|
check_stop()
|
|||
|
|
coverage["phase"] = "reading" if level == 0 else "reducing"
|
|||
|
|
report() # Publish the phase before any worker can contact a model.
|
|||
|
|
batch_count = 0
|
|||
|
|
generic = reusable and level == 0
|
|||
|
|
target = knowledge.GENERIC_TARGET if generic else min(2400, quota - 600 - len(kb.name))
|
|||
|
|
if target < 100:
|
|||
|
|
raise ValueError("知识库名称过长，无法保留各库摘要")
|
|||
|
|
manifests = {}
|
|||
|
|
# All SQLite iteration and inserts remain on the coordinator.
|
|||
|
|
batches = iter(_batches(db, kb.id, level, reusable=reusable))
|
|||
|
|
pending = deque()
|
|||
|
|
def run(batch, *, generic=generic, target=target, organization=organization, kid=kb.id):
|
|||
|
|
check_stop()
|
|||
|
|
if generic:
|
|||
|
|
entries = knowledge.canonical_entries(batch)
|
|||
|
|
return knowledge.reusable_summary(organization, kid, entries, target, cache_signature,
|
|||
|
|
lambda: summarize(entries, target), stopping)
|
|||
|
|
# Topic scoped summaries are cached by the exact
|
|||
|
|
# batch digest, topic, selected categories and model
|
|||
|
|
# signature. This is the second layer of the digest
|
|||
|
|
# pipeline: unchanged batches on a new assessment or
|
|||
|
|
# retry skip the model call while changed batches are
|
|||
|
|
# regenerated independently.
|
|||
|
|
summary, reused = cached_summary(
|
|||
|
|
row, batch, target, cache_signature,
|
|||
|
|
lambda: summarize(batch, target))
|
|||
|
|
check_stop()
|
|||
|
|
knowledge._validate(summary, target)
|
|||
|
|
return summary.strip(), reused, ""
|
|||
|
|
def raise_failed():
|
|||
|
|
# A later request can fail before the oldest finishes.
|
|||
|
|
# Notice it promptly and never refill a known failed queue.
|
|||
|
|
for _, active in pending:
|
|||
|
|
if active.cancelled():
|
|||
|
|
raise knowledge.ReadingCancelled("资料阅读已取消")
|
|||
|
|
if active.done():
|
|||
|
|
error = active.exception()
|
|||
|
|
if error is not None:
|
|||
|
|
raise error
|
|||
|
|
def schedule():
|
|||
|
|
check_stop()
|
|||
|
|
raise_failed()
|
|||
|
|
batch = next(batches, None)
|
|||
|
|
if batch is None:
|
|||
|
|
return False
|
|||
|
|
pending.append((batch, executor.submit(run, batch)))
|
|||
|
|
coverage["active_batches"] = len(pending)
|
|||
|
|
return True
|
|||
|
|
try:
|
|||
|
|
while len(pending) < parallelism and schedule():
|
|||
|
|
pass
|
|||
|
|
report()
|
|||
|
|
waiting_report_at = time.monotonic() + 2
|
|||
|
|
while pending:
|
|||
|
|
check_stop()
|
|||
|
|
raise_failed()
|
|||
|
|
batch, future = pending[0]
|
|||
|
|
try:
|
|||
|
|
summary, cached, key = future.result(timeout=0.25)
|
|||
|
|
except FutureTimeout:
|
|||
|
|
# Cancellation must not wait for the currently oldest request.
|
|||
|
|
check_stop()
|
|||
|
|
if time.monotonic() >= waiting_report_at:
|
|||
|
|
report()
|
|||
|
|
waiting_report_at = time.monotonic() + 2
|
|||
|
|
continue
|
|||
|
|
check_stop()
|
|||
|
|
pending.popleft()
|
|||
|
|
coverage["active_batches"] = len(pending)
|
|||
|
|
_insert_node(db, kb, summary, kind="summary", level=level + 1,
|
|||
|
|
file="通用知识阅读摘要" if generic else "全库分批阅读摘要",
|
|||
|
|
chunk_id=f"summary-level-{level + 1}",
|
|||
|
|
children=[item["id"] for item in batch])
|
|||
|
|
if level == 0:
|
|||
|
|
db.executemany("UPDATE nodes SET consumed=1,reused=? WHERE id=?",
|
|||
|
|
((int(cached), item["id"]) for item in batch))
|
|||
|
|
metric = "cached_batches" if cached else "new_read_batches"
|
|||
|
|
coverage[metric] += 1
|
|||
|
|
info[metric] += 1
|
|||
|
|
coverage["summary_cache_hits" if cached else "summary_cache_misses"] += 1
|
|||
|
|
if generic:
|
|||
|
|
manifests.setdefault(batch[0]["file"], []).append(key)
|
|||
|
|
coverage["batches_done"] += 1
|
|||
|
|
batch_count += 1
|
|||
|
|
schedule()
|
|||
|
|
report()
|
|||
|
|
waiting_report_at = time.monotonic() + 2
|
|||
|
|
except BaseException:
|
|||
|
|
stopped.set()
|
|||
|
|
for _, future in pending:
|
|||
|
|
future.cancel()
|
|||
|
|
coverage["active_batches"] = 0
|
|||
|
|
raise
|
|||
|
|
if generic and index_tokens[kb.id] and index_tokens[kb.id] == knowledge.index_revision(kb.id):
|
|||
|
|
# A manifest certifies a complete file, never a partial successful prefix.
|
|||
|
|
for name, keys in manifests.items():
|
|||
|
|
check_stop()
|
|||
|
|
file = session.exec(select(KnowledgeFile).where(KnowledgeFile.kb_id == kb.id,
|
|||
|
|
KnowledgeFile.file_name == name)).first()
|
|||
|
|
scanned = db.execute("SELECT stale,digest FROM files WHERE kb_id=? AND name=?", (kb.id, name)).fetchone()
|
|||
|
|
if file and not scanned[0] and file.content_hash == scanned[1]:
|
|||
|
|
count = db.execute("SELECT COUNT(*) FROM originals WHERE kb_id=? AND file=?", (kb.id, name)).fetchone()[0]
|
|||
|
|
knowledge.record_manifest(organization, kb.id, name, file.content_hash, file.revision,
|
|||
|
|
count, cache_signature, index_tokens[kb.id], keys)
|
|||
|
|
level += 1
|
|||
|
|
final = []
|
|||
|
|
total = 2
|
|||
|
|
for node in db.execute("SELECT * FROM nodes WHERE kb_id=? AND level=? ORDER BY seq", (kb.id, level)):
|
|||
|
|
item = _evidence(node)
|
|||
|
|
total += len(_json(item)) + (2 if final else 0)
|
|||
|
|
if total > quota:
|
|||
|
|
final = None
|
|||
|
|
break
|
|||
|
|
final.append(item)
|
|||
|
|
if final:
|
|||
|
|
result.extend(final)
|
|||
|
|
break
|
|||
|
|
if batch_count == 0 or level > 24:
|
|||
|
|
raise ValueError(f"知识库「{kb.name}」摘要无法在预算内收敛")
|
|||
|
|
check_stop()
|
|||
|
|
if len(_json(result)) > final_chars:
|
|||
|
|
raise ValueError("全库证据超出规划预算")
|
|||
|
|
coverage["phase"], coverage["evidence_count"] = "planning", len(result)
|
|||
|
|
report()
|
|||
|
|
db.close()
|
|||
|
|
check_stop()
|
|||
|
|
os.replace(staging, path)
|
|||
|
|
return result, coverage
|
|||
|
|
except Exception as exc:
|
|||
|
|
stopped.set()
|
|||
|
|
coverage["active_batches"] = 0
|
|||
|
|
coverage["phase"], coverage["complete"], coverage["source_complete"] = "failed", False, False
|
|||
|
|
coverage["warnings"].append(str(exc))
|
|||
|
|
try:
|
|||
|
|
report()
|
|||
|
|
except Exception:
|
|||
|
|
pass
|
|||
|
|
raise
|
|||
|
|
finally:
|
|||
|
|
db.close()
|
|||
|
|
staging.unlink(missing_ok=True)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def source_page(row, evidence_id, offset=0, limit=20):
|
|||
|
|
"""来源权限由调用者按导图 owner 和所选 KB 核对；这里只读取本版本快照。"""
|
|||
|
|
if not isinstance(evidence_id, str) or not re.fullmatch(r"E[1-9][0-9]*", evidence_id):
|
|||
|
|
raise ValueError("证据编号无效")
|
|||
|
|
if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
|
|||
|
|
raise ValueError("来源分页起点无效")
|
|||
|
|
if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 100:
|
|||
|
|
raise ValueError("来源分页大小必须在 1 至 100 之间")
|
|||
|
|
path = _path(row.id, row.version)
|
|||
|
|
if not path.is_file():
|
|||
|
|
raise ValueError("当前版本的原文快照不存在，请重新评估资料")
|
|||
|
|
with _connect(path, readonly=True) as db:
|
|||
|
|
node = db.execute("SELECT kb_id FROM nodes WHERE id=?", (evidence_id,)).fetchone()
|
|||
|
|
if not node or node[0] not in row.kb_ids:
|
|||
|
|
raise ValueError("证据不属于当前导图所选资料")
|
|||
|
|
cte = _source_cte()
|
|||
|
|
total = db.execute(cte + " SELECT COUNT(*) FROM raw_sources", (evidence_id,)).fetchone()[0]
|
|||
|
|
items = [dict(item) for item in db.execute(cte + """ SELECT o.id,o.kb_id,o.kb_name,o.file,o.chunk_id,o.text
|
|||
|
|
FROM originals o JOIN raw_sources s ON o.id=s.original_id ORDER BY o.id LIMIT ? OFFSET ?""",
|
|||
|
|
(evidence_id, limit, offset))]
|
|||
|
|
for item in items:
|
|||
|
|
item["id"] = f"source-{item['id']}"
|
|||
|
|
item["kind"], item["source_count"] = "raw", 1
|
|||
|
|
return {"items": items, "total": total, "offset": offset, "limit": limit}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def day_evidence(row, evidence, max_chars=FINAL_CHARS):
|
|||
|
|
"""保留传入的全部摘要，再在剩余预算内展开真实原文分片。"""
|
|||
|
|
result = [dict(item) for item in evidence]
|
|||
|
|
if len(_json(result)) > max_chars:
|
|||
|
|
raise ValueError("当天证据摘要超出模型预算")
|
|||
|
|
path = _path(row.id, row.version)
|
|||
|
|
if not path.is_file():
|
|||
|
|
return result # 兼容升级前只存业务库原文的导图。
|
|||
|
|
seen = {item["id"] for item in result}
|
|||
|
|
with _connect(path, readonly=True) as db:
|
|||
|
|
for item in evidence:
|
|||
|
|
if item.get("kind") != "summary" or item.get("kb_id") not in row.kb_ids:
|
|||
|
|
continue
|
|||
|
|
remaining = max_chars - len(_json(result))
|
|||
|
|
if remaining < 240:
|
|||
|
|
break
|
|||
|
|
cte = _source_cte()
|
|||
|
|
for node in db.execute(cte + """ SELECT n.* FROM nodes n JOIN raw_sources s
|
|||
|
|
ON n.original_id=s.original_id WHERE n.kind='raw' AND length(n.text)<=?
|
|||
|
|
ORDER BY n.seq LIMIT 128""", (item["id"], remaining - 200)):
|
|||
|
|
candidate = _evidence(node)
|
|||
|
|
if candidate["id"] not in seen and len(_json(result + [candidate])) <= max_chars:
|
|||
|
|
result.append(candidate)
|
|||
|
|
seen.add(candidate["id"])
|
|||
|
|
if max_chars - len(_json(result)) < 240:
|
|||
|
|
break
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
|
|||
|
|
def remove_corpus(draft_id):
|
|||
|
|
directory = _path(draft_id, 1).parent
|
|||
|
|
if directory.is_symlink():
|
|||
|
|
raise ValueError("资料来源路径无效")
|
|||
|
|
if directory.exists():
|
|||
|
|
shutil.rmtree(directory)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def cached_summary(row, evidence, max_chars, signature, generate):
|
|||
|
|
"""复用完全相同批次的摘要，跨草稿/版本增量命中。
|
|||
|
|
|
|||
|
|
The cache intentionally lives at the corpus root rather than under a draft
|
|||
|
|
version. A new draft selecting the same material can therefore reuse the
|
|||
|
|
expensive model result. The fingerprint still contains organization/user,
|
|||
|
|
topic, selected KB/category scope, model signature and the complete batch
|
|||
|
|
evidence, preventing cross-tenant or stale-content reuse.
|
|||
|
|
"""
|
|||
|
|
CORPUS_DIR.mkdir(parents=True, exist_ok=True)
|
|||
|
|
path = CORPUS_DIR / 'summary-cache.sqlite'
|
|||
|
|
# Keep this helper usable by legacy/unit-test draft objects that only carry
|
|||
|
|
# id/version/topic. Persisted TrainingMap rows still include the complete
|
|||
|
|
# identity fields, so cross-user or cross-organisation reuse remains isolated.
|
|||
|
|
fingerprint = hashlib.sha256(_json({
|
|||
|
|
'policy': 'training-corpus-summary-v1', 'signature': signature,
|
|||
|
|
'user': getattr(row, 'owner_user_id', 0),
|
|||
|
|
'organization': getattr(row, 'organization_id', 0),
|
|||
|
|
# Keep topic and requested days in the key because the current summary
|
|||
|
|
# prompt may include planning context. This still reuses unchanged
|
|||
|
|
# batches across draft versions and retries for the same assessment.
|
|||
|
|
'topic': getattr(row, 'topic', ''),
|
|||
|
|
'requested_days': getattr(row, 'requested_days', None),
|
|||
|
|
'kb_ids': sorted(set(getattr(row, 'kb_ids', []) or [])),
|
|||
|
|
'kb_category_ids': getattr(row, 'kb_category_ids', {}) or {},
|
|||
|
|
'evidence': evidence, 'max_chars': max_chars,
|
|||
|
|
}).encode('utf-8')).hexdigest()
|
|||
|
|
# A few assessment threads can finish at the same time. WAL + a bounded
|
|||
|
|
# busy timeout keeps cache writes from surfacing as spurious failures while
|
|||
|
|
# never holding the database lock across the model/network call below.
|
|||
|
|
lock_path = CORPUS_DIR / 'summary-cache-locks' / f'{fingerprint[:32]}.lock'
|
|||
|
|
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|||
|
|
# Per-fingerprint single-flight: identical batches wait for the first model
|
|||
|
|
# call and then reuse its result; unrelated batches remain fully parallel.
|
|||
|
|
try:
|
|||
|
|
import fcntl
|
|||
|
|
except ImportError:
|
|||
|
|
fcntl = None
|
|||
|
|
with lock_path.open('a') as lock_handle:
|
|||
|
|
if fcntl:
|
|||
|
|
fcntl.flock(lock_handle, fcntl.LOCK_EX)
|
|||
|
|
try:
|
|||
|
|
with sqlite3.connect(path, timeout=30) as cache:
|
|||
|
|
cache.execute('PRAGMA busy_timeout=30000')
|
|||
|
|
cache.execute('PRAGMA journal_mode=WAL')
|
|||
|
|
try:
|
|||
|
|
os.chmod(path, 0o600)
|
|||
|
|
except OSError:
|
|||
|
|
pass
|
|||
|
|
cache.execute('CREATE TABLE IF NOT EXISTS summaries (fingerprint TEXT PRIMARY KEY, text TEXT NOT NULL)')
|
|||
|
|
saved = cache.execute('SELECT text FROM summaries WHERE fingerprint=?', (fingerprint,)).fetchone()
|
|||
|
|
if saved and saved[0].strip() and len(saved[0]) <= max_chars:
|
|||
|
|
return saved[0], True
|
|||
|
|
# Keep the cache database transaction short. The per-fingerprint
|
|||
|
|
# lock still prevents duplicate model calls, while unrelated cache
|
|||
|
|
# keys and normal application writes are never blocked by network IO.
|
|||
|
|
summary = generate()
|
|||
|
|
if not isinstance(summary, str) or not summary.strip() or len(summary) > max_chars:
|
|||
|
|
raise ValueError('摘要超出预算或为空，未写入复用记录')
|
|||
|
|
with sqlite3.connect(path, timeout=30) as cache:
|
|||
|
|
cache.execute('PRAGMA busy_timeout=30000')
|
|||
|
|
cache.execute('PRAGMA journal_mode=WAL')
|
|||
|
|
cache.execute('INSERT OR REPLACE INTO summaries VALUES (?,?)', (fingerprint, summary))
|
|||
|
|
return summary, False
|
|||
|
|
finally:
|
|||
|
|
if fcntl:
|
|||
|
|
fcntl.flock(lock_handle, fcntl.LOCK_UN)
|