"""运营告警引擎 + 指标看板（交付报告 P2-4 / P2-3）。 告警：周期性巡检四类风险，写入 Notification（按 dedup_key 10 分钟去重， 避免每次轮询都刷屏）： 1. 死信任务积压（队列 dead > 0） 2. 磁盘剩余空间不足（< 2 GB） 3. 点数池偏低（全局余额 < 1000 点数） 4. 任务失败段存在且超过 30 分钟未处理 看板：/api/ops/dashboard 一屏返回队列、成本、知识库、Provider 健康度。 """ from __future__ import annotations import shutil from datetime import datetime, timedelta from sqlmodel import Session, select from .models import Campaign, Notification, PointAccount, TaskQueueItem, Segment, now from .config import DATA_DIR ALERT_DEDUP_WINDOW_MINUTES = 10 DISK_MIN_FREE_GB = 2.0 POINTS_LOW_THRESHOLD = 1000 # 充值提醒阈值：余额低于此值（或低于单任务预估的倍数）即提醒充值。默认 100 万点（=1000 万 Token） POINTS_RECHARGE_ALERT_THRESHOLD = 1_000_000 def _parse_ts(value: str) -> datetime | None: try: return datetime.fromisoformat(str(value or "").replace(" ", "T", 1)) except (ValueError, TypeError): return None def _recent_duplicate(session: Session, dedup_key: str) -> bool: cutoff = datetime.now() - timedelta(minutes=ALERT_DEDUP_WINDOW_MINUTES) existing = session.exec(select(Notification).where( Notification.type == "alert", Notification.message.contains(dedup_key), )).all() for row in existing: created = _parse_ts(row.created_at) if created and created >= cutoff: return True return False def raise_alert(session: Session, dedup_key: str, message: str) -> bool: """写一条告警通知；去重窗口内相同 dedup_key 不重复发。返回是否真的发出。""" if _recent_duplicate(session, dedup_key): return False session.add(Notification(type="alert", ref_id=0, message=f"[{dedup_key}] {message}")) session.commit() return True def run_alert_sweep(session: Session) -> list[str]: """执行一轮巡检，返回本轮真正发出的告警列表。""" fired: list[str] = [] # 1. 死信积压 dead_rows = session.exec(select(TaskQueueItem).where(TaskQueueItem.status == "dead")).all() if dead_rows: if raise_alert(session, "queue-dead", f"队列告警：{len(dead_rows)} 个任务进入死信队列，请在运营看板查看并重放"): fired.append("queue-dead") # 2. 磁盘空间 try: free_gb = shutil.disk_usage(DATA_DIR).free / 1024 ** 3 if free_gb < DISK_MIN_FREE_GB: if raise_alert(session, "disk-low", f"磁盘告警：数据目录剩余 {free_gb:.1f} GB（低于 {DISK_MIN_FREE_GB} GB），请清理或扩容"): fired.append("disk-low") except OSError: pass # 3. 点数池偏低（组织 1 的全部账户合计） accounts = session.exec(select(PointAccount)).all() total_points = sum(int(a.balance_points or 0) for a in accounts) if total_points < POINTS_LOW_THRESHOLD: if raise_alert(session, "points-low", f"点数告警：全局点数余额仅 {total_points}（低于 {POINTS_LOW_THRESHOLD}），AI 调用即将暂停"): fired.append("points-low") # 3.5 充值提醒：余额低于充值提醒阈值（比"即将暂停"更早预警） if total_points < POINTS_RECHARGE_ALERT_THRESHOLD: if raise_alert(session, "points-recharge", f"充值提醒：当前点数余额 {total_points}，已低于建议阈值 {POINTS_RECHARGE_ALERT_THRESHOLD}（约还可生成 {max(total_points // 30000, 0)} 段内容），请及时充值"): fired.append("points-recharge") # 4. 失败段滞留（存在失败任务且 30 分钟未处理） campaigns = session.exec(select(Campaign).where(Campaign.has_failure == True)).all() # noqa: E712 stale_cutoff = (datetime.now() - timedelta(minutes=30)).isoformat() for campaign in campaigns: # Campaign 没有 updated_at 字段；失败段本身有更新时间，取它与创建时间的较新者。 segment_times = [str(item.updated_at or "") for item in session.exec( select(Segment).where(Segment.campaign_id == campaign.id)).all()] updated = max([str(campaign.created_at or "")] + segment_times) if updated and updated < stale_cutoff: if raise_alert(session, f"campaign-failure:{campaign.id}", f"任务告警：「{campaign.name}」存在失败段且超过 30 分钟未处理，请到任务看板重试或跳过"): fired.append(f"campaign-failure:{campaign.id}") return fired def ops_dashboard(session: Session, *, owner_user_id: int | None = None, organization_id: int | None = None) -> dict: """一屏运营数据：队列/任务/成本/知识库/Provider 健康。 owner_user_id 传入时按用户隔离：只统计该用户自己的任务、用量、告警； None（管理员）看全组织。 """ from .task_queue import queue_stats from .kb_metrics import kb_metrics_overview from .models import ModelProvider, UsageEvent from . import backup as backup_mod is_admin = owner_user_id is None queue = queue_stats(session, TaskQueueItem) campaign_query = select(Campaign) if organization_id is not None: campaign_query = campaign_query.where(Campaign.organization_id == int(organization_id)) if not is_admin: campaign_query = campaign_query.where(Campaign.owner_user_id == int(owner_user_id)) campaigns = session.exec(campaign_query).all() active = sum(1 for c in campaigns if c.status in ("running", "parsed", "composing_ready", "human_review")) failed = sum(1 for c in campaigns if c.has_failure) done = sum(1 for c in campaigns if c.status == "done") # 近 7 天用量（按用户隔离） cutoff = (datetime.now() - timedelta(days=7)).isoformat() events_query = select(UsageEvent).where(UsageEvent.created_at >= cutoff) if organization_id is not None: events_query = events_query.where(UsageEvent.organization_id == int(organization_id)) if not is_admin: events_query = events_query.where(UsageEvent.user_id == int(owner_user_id)) events = session.exec(events_query).all() amount_minor = sum(int(e.amount_minor or 0) for e in events) tokens = sum(int(e.quantity or 0) for e in events if e.service_type == "llm") kb = kb_metrics_overview(session, owner_user_id=owner_user_id if not is_admin else None, organization_id=organization_id) # Provider 健康是系统级信息，仅管理员可见；普通用户不返回 provider_health = [] if is_admin: providers = session.exec(select(ModelProvider)).all() provider_health = [{ "id": p.id, "name": p.name, "enabled": p.enabled, "backend": "keychain" if (p.api_key or "").startswith("keychain:") else "database", "models": len(p.models or []), } for p in providers] dead_query = select(TaskQueueItem).where(TaskQueueItem.status == "dead") dead_items = session.exec(dead_query).all() if not is_admin: # 队列项没有 user_id 字段：按 payload 里记录的任务归属过滤 own_campaign_ids = {int(c.id or 0) for c in campaigns} dead_items = [item for item in dead_items if int((item.payload or {}).get("campaign_id", 0) or 0) in own_campaign_ids] dead_list = [{"id": item.id, "task_key": item.task_key, "error": (item.error or "")[:160], "attempt": item.attempt, "updated_at": item.updated_at} for item in dead_items] # 告警隔离：管理员看系统告警，普通用户只看自己任务相关的通知 if is_admin: alerts = session.exec(select(Notification).where( Notification.type == "alert").order_by(Notification.id.desc()).limit(20)).all() else: own_campaign_ids = {int(c.id or 0) for c in campaigns} alerts = [n for n in session.exec(select(Notification).where( Notification.type == "alert").order_by(Notification.id.desc()).limit(100)).all() if int(n.ref_id or 0) in own_campaign_ids][:20] scope = {"owner_user_id": int(owner_user_id), "scope": "self"} if not is_admin else {"scope": "org"} return { **scope, "queue": queue, "campaigns": {"active": active, "failed": failed, "done": done, "total": len(campaigns)}, "cost_7d": {"amount_minor": amount_minor, "tokens": tokens}, "knowledge": {**({"hit_rate": kb["hit_rate"], "repeat_rate": kb["repeat_rate"], "reference_coverage": kb["reference_coverage"], "by_kb": kb["by_kb"][:5]} if is_admin else {"hit_rate": 0, "repeat_rate": 0, "reference_coverage": 0, "by_kb": []})}, "providers": provider_health, "dead_letter": dead_list, "backup": backup_mod.backup_status() if is_admin else {}, "recent_alerts": [{"id": n.id, "message": n.message, "created_at": n.created_at} for n in alerts], "generated_at": now(), } __all__ = ["raise_alert", "run_alert_sweep", "ops_dashboard", "ALERT_DEDUP_WINDOW_MINUTES", "DISK_MIN_FREE_GB", "POINTS_LOW_THRESHOLD"]