71 lines
3.0 KiB
Python
71 lines
3.0 KiB
Python
|
"""Reversible exclusions for explicitly confirmed orphan test indexes.
|
||
|
|
|
||
|
|
The manifest is local operational configuration, not an index mutation. Existing
|
||
|
|
registered files always take precedence and automatically release an exclusion.
|
||
|
|
"""
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import json
|
||
|
|
|
||
|
|
from sqlmodel import select
|
||
|
|
|
||
|
|
from ..config import DATA_DIR
|
||
|
|
from ..models import KnowledgeFile
|
||
|
|
|
||
|
|
QUARANTINE_PATH = DATA_DIR / 'rag-quarantine.json'
|
||
|
|
|
||
|
|
|
||
|
|
class QuarantineManifestError(ValueError):
|
||
|
|
"""Invalid quarantine configuration must fail explicitly, never fail open."""
|
||
|
|
|
||
|
|
|
||
|
|
def _manifest():
|
||
|
|
try:
|
||
|
|
raw = QUARANTINE_PATH.read_text(encoding='utf-8')
|
||
|
|
except FileNotFoundError:
|
||
|
|
return {}
|
||
|
|
except (OSError, UnicodeError) as exc:
|
||
|
|
raise QuarantineManifestError('知识库测试索引隔离配置无法读取，请检查 rag-quarantine.json') from exc
|
||
|
|
try:
|
||
|
|
def unique_object(pairs):
|
||
|
|
result = {}
|
||
|
|
for key, value in pairs:
|
||
|
|
if key in result:
|
||
|
|
raise ValueError('duplicate key')
|
||
|
|
result[key] = value
|
||
|
|
return result
|
||
|
|
data = json.loads(raw, object_pairs_hook=unique_object)
|
||
|
|
if not isinstance(data, dict) or set(data) != {'version', 'libraries'}:
|
||
|
|
raise ValueError('invalid fields')
|
||
|
|
if type(data['version']) is not int or data['version'] != 1 or not isinstance(data['libraries'], dict):
|
||
|
|
raise ValueError('invalid version or libraries')
|
||
|
|
result = {}
|
||
|
|
for kid, entries in data['libraries'].items():
|
||
|
|
if not kid.isascii() or not kid.isdecimal() or int(kid) <= 0 or str(int(kid)) != kid or not isinstance(entries, list):
|
||
|
|
raise ValueError('invalid library')
|
||
|
|
names = set()
|
||
|
|
for entry in entries:
|
||
|
|
if not isinstance(entry, dict) or set(entry) != {'file_name', 'reason'}:
|
||
|
|
raise ValueError('invalid entry fields')
|
||
|
|
name, reason = entry['file_name'], entry['reason']
|
||
|
|
if not isinstance(name, str) or not name.strip() or '\x00' in name or name in names:
|
||
|
|
raise ValueError('invalid or duplicate file name')
|
||
|
|
if not isinstance(reason, str) or not reason.strip():
|
||
|
|
raise ValueError('missing reason')
|
||
|
|
names.add(name)
|
||
|
|
result[int(kid)] = names
|
||
|
|
return result
|
||
|
|
except (ValueError, TypeError, AttributeError, OverflowError) as exc:
|
||
|
|
raise QuarantineManifestError('知识库测试索引隔离配置无效，请检查 rag-quarantine.json 的版本、知识库编号及文件条目') from exc
|
||
|
|
|
||
|
|
|
||
|
|
def file_exclusions(session, kid):
|
||
|
|
"""Exclude only explicitly listed names still absent from KnowledgeFile."""
|
||
|
|
if isinstance(kid, bool) or not isinstance(kid, int) or kid <= 0:
|
||
|
|
raise ValueError('知识库编号无效')
|
||
|
|
names = _manifest().get(kid, set())
|
||
|
|
if not names:
|
||
|
|
return set()
|
||
|
|
registered = session.exec(select(KnowledgeFile.file_name).where(KnowledgeFile.kb_id == kid)).all()
|
||
|
|
return names - set(registered)
|