145 lines
7.5 KiB
Python
145 lines
7.5 KiB
Python
"""知识库消费指标（交付报告 P2-3）。
|
|
|
|
三个核心指标，全部基于既有 KnowledgeUsage / Segment.references / Segment 落表，
|
|
不引入新的数据采集：
|
|
- 命中率 hit_rate：生产会话中拿到知识片段的比例（检索到片段 ÷ 检索请求）
|
|
- 重复消费率 repeat_rate：同一 Agent 反复使用同一段落导致重新开放的比例
|
|
- 引用覆盖率 reference_coverage：交付段中带引用来源的比例
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from sqlmodel import Session, select
|
|
|
|
from .models import Agent, Campaign, KnowledgeBase, KnowledgeUsage, Segment
|
|
|
|
|
|
def kb_metrics(session: Session, campaign_id: int = 0, *, owner_user_id: int | None = None,
|
|
organization_id: int | None = None, kb_ids: set[int] | None = None) -> dict:
|
|
"""返回平台级（campaign_id=0）或单任务的消费质量指标。"""
|
|
usage_rows = session.exec(select(KnowledgeUsage)).all()
|
|
if campaign_id:
|
|
usage_rows = [row for row in usage_rows if row.campaign_id == campaign_id]
|
|
if kb_ids is not None:
|
|
usage_rows = [row for row in usage_rows if int(row.kb_id or 0) in kb_ids]
|
|
if organization_id is not None or owner_user_id is not None:
|
|
campaign_ids = {int(row.campaign_id or 0) for row in usage_rows if row.campaign_id}
|
|
campaigns = {item.id: item for item in session.exec(
|
|
select(Campaign).where(Campaign.id.in_(campaign_ids))).all()} if campaign_ids else {}
|
|
usage_rows = [row for row in usage_rows if (
|
|
campaigns.get(row.campaign_id) is not None and
|
|
(organization_id is None or int(campaigns[row.campaign_id].organization_id or 1) == int(organization_id)) and
|
|
(owner_user_id is None or int(campaigns[row.campaign_id].owner_user_id or 0) == int(owner_user_id))
|
|
)]
|
|
|
|
# ---- 命中率：reserved/consumed（含已释放后再次使用）都算命中 ----
|
|
# 一个 (agent_id, segment_id) 组合代表一次生产检索请求。
|
|
requests: dict[tuple[int, int], list[KnowledgeUsage]] = {}
|
|
for row in usage_rows:
|
|
if row.segment_id:
|
|
requests.setdefault((row.agent_id, row.segment_id), []).append(row)
|
|
hit_requests = len(requests)
|
|
|
|
# 重复消费：同一 agent + 同一 chunk 出现多条记录（说明曾被重新开放再用）
|
|
pair_counts: dict[tuple[int, str], int] = {}
|
|
for row in usage_rows:
|
|
key = (row.agent_id, row.chunk_id)
|
|
pair_counts[key] = pair_counts.get(key, 0) + 1
|
|
repeated_pairs = sum(1 for count in pair_counts.values() if count > 1)
|
|
repeat_rate = round(repeated_pairs / len(pair_counts), 4) if pair_counts else 0.0
|
|
|
|
# ---- 引用覆盖率：按任务统计 ----
|
|
seg_query = select(Segment)
|
|
if campaign_id:
|
|
seg_query = seg_query.where(Segment.campaign_id == campaign_id)
|
|
if organization_id is not None or owner_user_id is not None:
|
|
campaign_ids = {int(item.id) for item in session.exec(select(Campaign).where(
|
|
*(([Campaign.organization_id == int(organization_id)] if organization_id is not None else []) +
|
|
([Campaign.owner_user_id == int(owner_user_id)] if owner_user_id is not None else []))
|
|
)).all() if item.id}
|
|
seg_query = seg_query.where(Segment.campaign_id.in_(campaign_ids or {-1}))
|
|
segments = session.exec(seg_query).all()
|
|
total_segments = len(segments)
|
|
referenced = sum(1 for seg in segments if seg.references)
|
|
reference_coverage = round(referenced / total_segments, 4) if total_segments else 0.0
|
|
|
|
# 检索总量：每次任务运行的检索请求数 = 有消耗记录的 (agent, segment) 数；
|
|
# 无消耗记录的生产段视为一次未命中的检索。
|
|
production_segments = [seg for seg in segments if seg.status in
|
|
("approved", "done", "qc", "human_review", "composing_ready", "rework")]
|
|
miss_requests = 0
|
|
for seg in production_segments:
|
|
if not any(key[1] == seg.id for key in requests):
|
|
miss_requests += 1
|
|
total_requests = hit_requests + miss_requests
|
|
hit_rate = round(hit_requests / total_requests, 4) if total_requests else 0.0
|
|
|
|
return {
|
|
"hit_rate": hit_rate,
|
|
"repeat_rate": repeat_rate,
|
|
"reference_coverage": reference_coverage,
|
|
"detail": {
|
|
"search_requests": total_requests,
|
|
"hit_requests": hit_requests,
|
|
"miss_requests": miss_requests,
|
|
"consumed_chunks": len(usage_rows),
|
|
"repeated_pairs": repeated_pairs,
|
|
"distinct_chunks": len(pair_counts),
|
|
"segments_total": total_segments,
|
|
"segments_with_references": referenced,
|
|
},
|
|
}
|
|
|
|
|
|
def kb_metrics_overview(session: Session, *, owner_user_id: int | None = None,
|
|
organization_id: int | None = None,
|
|
kb_ids: set[int] | None = None) -> dict:
|
|
"""知识库页展示：按库拆分的消费热力 + 平台级三指标。"""
|
|
allowed_kb_ids = None if kb_ids is None else {int(value) for value in kb_ids}
|
|
overall = kb_metrics(session, owner_user_id=owner_user_id, organization_id=organization_id,
|
|
kb_ids=allowed_kb_ids)
|
|
usage_rows = session.exec(select(KnowledgeUsage)).all()
|
|
if organization_id is not None or owner_user_id is not None:
|
|
campaign_ids = {int(item.id) for item in session.exec(select(Campaign).where(
|
|
*(([Campaign.organization_id == int(organization_id)] if organization_id is not None else []) +
|
|
([Campaign.owner_user_id == int(owner_user_id)] if owner_user_id is not None else []))
|
|
)).all() if item.id}
|
|
usage_rows = [row for row in usage_rows if int(row.campaign_id or 0) in campaign_ids]
|
|
if allowed_kb_ids is not None:
|
|
usage_rows = [row for row in usage_rows if int(row.kb_id or 0) in allowed_kb_ids]
|
|
selected_kb_ids = allowed_kb_ids if allowed_kb_ids is not None else {int(row.kb_id) for row in usage_rows}
|
|
kb_query = select(KnowledgeBase)
|
|
if organization_id is not None:
|
|
kb_query = kb_query.where(KnowledgeBase.organization_id.in_([0, int(organization_id)]))
|
|
kbs = session.exec(kb_query).all()
|
|
kbs = [kb for kb in kbs if int(kb.id or 0) in selected_kb_ids]
|
|
by_kb = []
|
|
for kb in kbs:
|
|
rows = [row for row in usage_rows if row.kb_id == kb.id]
|
|
consumed = sum(1 for row in rows if row.status == "consumed")
|
|
released = sum(1 for row in rows if row.status == "available")
|
|
by_kb.append({
|
|
"kb_id": kb.id,
|
|
"name": kb.name,
|
|
"file_count": kb.file_count,
|
|
"chunk_count": kb.chunk_count,
|
|
"consumed": consumed,
|
|
"released": released,
|
|
"consumption_ratio": round(consumed / kb.chunk_count, 4) if kb.chunk_count else 0.0,
|
|
})
|
|
by_kb.sort(key=lambda item: -item["consumption_ratio"])
|
|
# 最常用片段 Top5（按被不同 Agent 消费次数）
|
|
chunk_agents: dict[str, set[int]] = {}
|
|
chunk_label: dict[str, dict] = {}
|
|
for row in usage_rows:
|
|
chunk_agents.setdefault(row.chunk_id, set()).add(row.agent_id)
|
|
chunk_label[row.chunk_id] = {"file": row.file_name, "heading": row.heading,
|
|
"kb_id": row.kb_id, "excerpt": (row.excerpt or "")[:80]}
|
|
top_chunks = sorted(
|
|
({"chunk_id": cid, "agents": len(agents), **chunk_label.get(cid, {})}
|
|
for cid, agents in chunk_agents.items()),
|
|
key=lambda item: -item["agents"])[:5]
|
|
return {**overall, "by_kb": by_kb, "top_chunks": top_chunks}
|
|
|
|
|
|
__all__ = ["kb_metrics", "kb_metrics_overview"]
|