feat(server): add workbench API, DSH planner worker and RAG pipeline

server/app/ops.py

@@ -0,0 +1,198 @@
+"""运营告警引擎 + 指标看板（交付报告 P2-4 / P2-3）。
+
+告警：周期性巡检四类风险，写入 Notification（按 dedup_key 10 分钟去重，
+避免每次轮询都刷屏）：
+1. 死信任务积压（队列 dead > 0）
+2. 磁盘剩余空间不足（< 2 GB）
+3. 点数池偏低（全局余额 < 1000 点数）
+4. 任务失败段存在且超过 30 分钟未处理
+
+看板：/api/ops/dashboard 一屏返回队列、成本、知识库、Provider 健康度。
+"""
+from __future__ import annotations
+
+import shutil
+from datetime import datetime, timedelta
+
+from sqlmodel import Session, select
+
+from .models import Campaign, Notification, PointAccount, TaskQueueItem, Segment, now
+from .config import DATA_DIR
+
+ALERT_DEDUP_WINDOW_MINUTES = 10
+DISK_MIN_FREE_GB = 2.0
+POINTS_LOW_THRESHOLD = 1000
+# 充值提醒阈值：余额低于此值（或低于单任务预估的倍数）即提醒充值。默认 100 万点（=1000 万 Token）
+POINTS_RECHARGE_ALERT_THRESHOLD = 1_000_000
+
+
+def _parse_ts(value: str) -> datetime | None:
+    try:
+        return datetime.fromisoformat(str(value or "").replace(" ", "T", 1))
+    except (ValueError, TypeError):
+        return None
+
+
+def _recent_duplicate(session: Session, dedup_key: str) -> bool:
+    cutoff = datetime.now() - timedelta(minutes=ALERT_DEDUP_WINDOW_MINUTES)
+    existing = session.exec(select(Notification).where(
+        Notification.type == "alert",
+        Notification.message.contains(dedup_key),
+    )).all()
+    for row in existing:
+        created = _parse_ts(row.created_at)
+        if created and created >= cutoff:
+            return True
+    return False
+
+
+def raise_alert(session: Session, dedup_key: str, message: str) -> bool:
+    """写一条告警通知；去重窗口内相同 dedup_key 不重复发。返回是否真的发出。"""
+    if _recent_duplicate(session, dedup_key):
+        return False
+    session.add(Notification(type="alert", ref_id=0, message=f"[{dedup_key}] {message}"))
+    session.commit()
+    return True
+
+
+def run_alert_sweep(session: Session) -> list[str]:
+    """执行一轮巡检，返回本轮真正发出的告警列表。"""
+    fired: list[str] = []
+
+    # 1. 死信积压
+    dead_rows = session.exec(select(TaskQueueItem).where(TaskQueueItem.status == "dead")).all()
+    if dead_rows:
+        if raise_alert(session, "queue-dead",
+                       f"队列告警：{len(dead_rows)} 个任务进入死信队列，请在运营看板查看并重放"):
+            fired.append("queue-dead")
+
+    # 2. 磁盘空间
+    try:
+        free_gb = shutil.disk_usage(DATA_DIR).free / 1024 ** 3
+        if free_gb < DISK_MIN_FREE_GB:
+            if raise_alert(session, "disk-low",
+                           f"磁盘告警：数据目录剩余 {free_gb:.1f} GB（低于 {DISK_MIN_FREE_GB} GB），请清理或扩容"):
+                fired.append("disk-low")
+    except OSError:
+        pass
+
+    # 3. 点数池偏低（组织 1 的全部账户合计）
+    accounts = session.exec(select(PointAccount)).all()
+    total_points = sum(int(a.balance_points or 0) for a in accounts)
+    if total_points < POINTS_LOW_THRESHOLD:
+        if raise_alert(session, "points-low",
+                       f"点数告警：全局点数余额仅 {total_points}（低于 {POINTS_LOW_THRESHOLD}），AI 调用即将暂停"):
+            fired.append("points-low")
+
+    # 3.5 充值提醒：余额低于充值提醒阈值（比"即将暂停"更早预警）
+    if total_points < POINTS_RECHARGE_ALERT_THRESHOLD:
+        if raise_alert(session, "points-recharge",
+                       f"充值提醒：当前点数余额 {total_points}，已低于建议阈值 {POINTS_RECHARGE_ALERT_THRESHOLD}（约还可生成 {max(total_points // 30000, 0)} 段内容），请及时充值"):
+            fired.append("points-recharge")
+
+    # 4. 失败段滞留（存在失败任务且 30 分钟未处理）
+    campaigns = session.exec(select(Campaign).where(Campaign.has_failure == True)).all()  # noqa: E712
+    stale_cutoff = (datetime.now() - timedelta(minutes=30)).isoformat()
+    for campaign in campaigns:
+        # Campaign 没有 updated_at 字段；失败段本身有更新时间，取它与创建时间的较新者。
+        segment_times = [str(item.updated_at or "") for item in session.exec(
+            select(Segment).where(Segment.campaign_id == campaign.id)).all()]
+        updated = max([str(campaign.created_at or "")] + segment_times)
+        if updated and updated < stale_cutoff:
+            if raise_alert(session, f"campaign-failure:{campaign.id}",
+                           f"任务告警：「{campaign.name}」存在失败段且超过 30 分钟未处理，请到任务看板重试或跳过"):
+                fired.append(f"campaign-failure:{campaign.id}")
+    return fired
+
+
+def ops_dashboard(session: Session, *, owner_user_id: int | None = None,
+                  organization_id: int | None = None) -> dict:
+    """一屏运营数据：队列/任务/成本/知识库/Provider 健康。
+
+    owner_user_id 传入时按用户隔离：只统计该用户自己的任务、用量、告警；
+    None（管理员）看全组织。
+    """
+    from .task_queue import queue_stats
+    from .kb_metrics import kb_metrics_overview
+    from .models import ModelProvider, UsageEvent
+    from . import backup as backup_mod
+
+    is_admin = owner_user_id is None
+
+    queue = queue_stats(session, TaskQueueItem)
+
+    campaign_query = select(Campaign)
+    if organization_id is not None:
+        campaign_query = campaign_query.where(Campaign.organization_id == int(organization_id))
+    if not is_admin:
+        campaign_query = campaign_query.where(Campaign.owner_user_id == int(owner_user_id))
+    campaigns = session.exec(campaign_query).all()
+    active = sum(1 for c in campaigns if c.status in ("running", "parsed", "composing_ready", "human_review"))
+    failed = sum(1 for c in campaigns if c.has_failure)
+    done = sum(1 for c in campaigns if c.status == "done")
+
+    # 近 7 天用量（按用户隔离）
+    cutoff = (datetime.now() - timedelta(days=7)).isoformat()
+    events_query = select(UsageEvent).where(UsageEvent.created_at >= cutoff)
+    if organization_id is not None:
+        events_query = events_query.where(UsageEvent.organization_id == int(organization_id))
+    if not is_admin:
+        events_query = events_query.where(UsageEvent.user_id == int(owner_user_id))
+    events = session.exec(events_query).all()
+    amount_minor = sum(int(e.amount_minor or 0) for e in events)
+    tokens = sum(int(e.quantity or 0) for e in events if e.service_type == "llm")
+
+    kb = kb_metrics_overview(session, owner_user_id=owner_user_id if not is_admin else None,
+                             organization_id=organization_id)
+
+    # Provider 健康是系统级信息，仅管理员可见；普通用户不返回
+    provider_health = []
+    if is_admin:
+        providers = session.exec(select(ModelProvider)).all()
+        provider_health = [{
+            "id": p.id, "name": p.name, "enabled": p.enabled,
+            "backend": "keychain" if (p.api_key or "").startswith("keychain:") else "database",
+            "models": len(p.models or []),
+        } for p in providers]
+
+    dead_query = select(TaskQueueItem).where(TaskQueueItem.status == "dead")
+    dead_items = session.exec(dead_query).all()
+    if not is_admin:
+        # 队列项没有 user_id 字段：按 payload 里记录的任务归属过滤
+        own_campaign_ids = {int(c.id or 0) for c in campaigns}
+        dead_items = [item for item in dead_items
+                      if int((item.payload or {}).get("campaign_id", 0) or 0) in own_campaign_ids]
+    dead_list = [{"id": item.id, "task_key": item.task_key, "error": (item.error or "")[:160],
+                  "attempt": item.attempt, "updated_at": item.updated_at} for item in dead_items]
+
+    # 告警隔离：管理员看系统告警，普通用户只看自己任务相关的通知
+    if is_admin:
+        alerts = session.exec(select(Notification).where(
+            Notification.type == "alert").order_by(Notification.id.desc()).limit(20)).all()
+    else:
+        own_campaign_ids = {int(c.id or 0) for c in campaigns}
+        alerts = [n for n in session.exec(select(Notification).where(
+            Notification.type == "alert").order_by(Notification.id.desc()).limit(100)).all()
+            if int(n.ref_id or 0) in own_campaign_ids][:20]
+
+    scope = {"owner_user_id": int(owner_user_id), "scope": "self"} if not is_admin else {"scope": "org"}
+    return {
+        **scope,
+        "queue": queue,
+        "campaigns": {"active": active, "failed": failed, "done": done, "total": len(campaigns)},
+        "cost_7d": {"amount_minor": amount_minor, "tokens": tokens},
+        "knowledge": {**({"hit_rate": kb["hit_rate"], "repeat_rate": kb["repeat_rate"],
+                          "reference_coverage": kb["reference_coverage"],
+                          "by_kb": kb["by_kb"][:5]} if is_admin else
+                         {"hit_rate": 0, "repeat_rate": 0, "reference_coverage": 0, "by_kb": []})},
+        "providers": provider_health,
+        "dead_letter": dead_list,
+        "backup": backup_mod.backup_status() if is_admin else {},
+        "recent_alerts": [{"id": n.id, "message": n.message, "created_at": n.created_at}
+                          for n in alerts],
+        "generated_at": now(),
+    }
+
+
+__all__ = ["raise_alert", "run_alert_sweep", "ops_dashboard",
+           "ALERT_DEDUP_WINDOW_MINUTES", "DISK_MIN_FREE_GB", "POINTS_LOW_THRESHOLD"]