199 lines
9.3 KiB
Python
199 lines
9.3 KiB
Python
"""运营告警引擎 + 指标看板（交付报告 P2-4 / P2-3）。
|
||
|
||
告警：周期性巡检四类风险，写入 Notification（按 dedup_key 10 分钟去重，
|
||
避免每次轮询都刷屏）：
|
||
1. 死信任务积压（队列 dead > 0）
|
||
2. 磁盘剩余空间不足（< 2 GB）
|
||
3. 点数池偏低（全局余额 < 1000 点数）
|
||
4. 任务失败段存在且超过 30 分钟未处理
|
||
|
||
看板：/api/ops/dashboard 一屏返回队列、成本、知识库、Provider 健康度。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import shutil
|
||
from datetime import datetime, timedelta
|
||
|
||
from sqlmodel import Session, select
|
||
|
||
from .models import Campaign, Notification, PointAccount, TaskQueueItem, Segment, now
|
||
from .config import DATA_DIR
|
||
|
||
ALERT_DEDUP_WINDOW_MINUTES = 10
|
||
DISK_MIN_FREE_GB = 2.0
|
||
POINTS_LOW_THRESHOLD = 1000
|
||
# 充值提醒阈值：余额低于此值（或低于单任务预估的倍数）即提醒充值。默认 100 万点（=1000 万 Token）
|
||
POINTS_RECHARGE_ALERT_THRESHOLD = 1_000_000
|
||
|
||
|
||
def _parse_ts(value: str) -> datetime | None:
|
||
try:
|
||
return datetime.fromisoformat(str(value or "").replace(" ", "T", 1))
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
|
||
def _recent_duplicate(session: Session, dedup_key: str) -> bool:
|
||
cutoff = datetime.now() - timedelta(minutes=ALERT_DEDUP_WINDOW_MINUTES)
|
||
existing = session.exec(select(Notification).where(
|
||
Notification.type == "alert",
|
||
Notification.message.contains(dedup_key),
|
||
)).all()
|
||
for row in existing:
|
||
created = _parse_ts(row.created_at)
|
||
if created and created >= cutoff:
|
||
return True
|
||
return False
|
||
|
||
|
||
def raise_alert(session: Session, dedup_key: str, message: str) -> bool:
|
||
"""写一条告警通知；去重窗口内相同 dedup_key 不重复发。返回是否真的发出。"""
|
||
if _recent_duplicate(session, dedup_key):
|
||
return False
|
||
session.add(Notification(type="alert", ref_id=0, message=f"[{dedup_key}] {message}"))
|
||
session.commit()
|
||
return True
|
||
|
||
|
||
def run_alert_sweep(session: Session) -> list[str]:
|
||
"""执行一轮巡检，返回本轮真正发出的告警列表。"""
|
||
fired: list[str] = []
|
||
|
||
# 1. 死信积压
|
||
dead_rows = session.exec(select(TaskQueueItem).where(TaskQueueItem.status == "dead")).all()
|
||
if dead_rows:
|
||
if raise_alert(session, "queue-dead",
|
||
f"队列告警：{len(dead_rows)} 个任务进入死信队列，请在运营看板查看并重放"):
|
||
fired.append("queue-dead")
|
||
|
||
# 2. 磁盘空间
|
||
try:
|
||
free_gb = shutil.disk_usage(DATA_DIR).free / 1024 ** 3
|
||
if free_gb < DISK_MIN_FREE_GB:
|
||
if raise_alert(session, "disk-low",
|
||
f"磁盘告警：数据目录剩余 {free_gb:.1f} GB（低于 {DISK_MIN_FREE_GB} GB），请清理或扩容"):
|
||
fired.append("disk-low")
|
||
except OSError:
|
||
pass
|
||
|
||
# 3. 点数池偏低（组织 1 的全部账户合计）
|
||
accounts = session.exec(select(PointAccount)).all()
|
||
total_points = sum(int(a.balance_points or 0) for a in accounts)
|
||
if total_points < POINTS_LOW_THRESHOLD:
|
||
if raise_alert(session, "points-low",
|
||
f"点数告警：全局点数余额仅 {total_points}（低于 {POINTS_LOW_THRESHOLD}），AI 调用即将暂停"):
|
||
fired.append("points-low")
|
||
|
||
# 3.5 充值提醒：余额低于充值提醒阈值（比"即将暂停"更早预警）
|
||
if total_points < POINTS_RECHARGE_ALERT_THRESHOLD:
|
||
if raise_alert(session, "points-recharge",
|
||
f"充值提醒：当前点数余额 {total_points}，已低于建议阈值 {POINTS_RECHARGE_ALERT_THRESHOLD}（约还可生成 {max(total_points // 30000, 0)} 段内容），请及时充值"):
|
||
fired.append("points-recharge")
|
||
|
||
# 4. 失败段滞留（存在失败任务且 30 分钟未处理）
|
||
campaigns = session.exec(select(Campaign).where(Campaign.has_failure == True)).all() # noqa: E712
|
||
stale_cutoff = (datetime.now() - timedelta(minutes=30)).isoformat()
|
||
for campaign in campaigns:
|
||
# Campaign 没有 updated_at 字段；失败段本身有更新时间，取它与创建时间的较新者。
|
||
segment_times = [str(item.updated_at or "") for item in session.exec(
|
||
select(Segment).where(Segment.campaign_id == campaign.id)).all()]
|
||
updated = max([str(campaign.created_at or "")] + segment_times)
|
||
if updated and updated < stale_cutoff:
|
||
if raise_alert(session, f"campaign-failure:{campaign.id}",
|
||
f"任务告警：「{campaign.name}」存在失败段且超过 30 分钟未处理，请到任务看板重试或跳过"):
|
||
fired.append(f"campaign-failure:{campaign.id}")
|
||
return fired
|
||
|
||
|
||
def ops_dashboard(session: Session, *, owner_user_id: int | None = None,
|
||
organization_id: int | None = None) -> dict:
|
||
"""一屏运营数据：队列/任务/成本/知识库/Provider 健康。
|
||
|
||
owner_user_id 传入时按用户隔离：只统计该用户自己的任务、用量、告警；
|
||
None（管理员）看全组织。
|
||
"""
|
||
from .task_queue import queue_stats
|
||
from .kb_metrics import kb_metrics_overview
|
||
from .models import ModelProvider, UsageEvent
|
||
from . import backup as backup_mod
|
||
|
||
is_admin = owner_user_id is None
|
||
|
||
queue = queue_stats(session, TaskQueueItem)
|
||
|
||
campaign_query = select(Campaign)
|
||
if organization_id is not None:
|
||
campaign_query = campaign_query.where(Campaign.organization_id == int(organization_id))
|
||
if not is_admin:
|
||
campaign_query = campaign_query.where(Campaign.owner_user_id == int(owner_user_id))
|
||
campaigns = session.exec(campaign_query).all()
|
||
active = sum(1 for c in campaigns if c.status in ("running", "parsed", "composing_ready", "human_review"))
|
||
failed = sum(1 for c in campaigns if c.has_failure)
|
||
done = sum(1 for c in campaigns if c.status == "done")
|
||
|
||
# 近 7 天用量（按用户隔离）
|
||
cutoff = (datetime.now() - timedelta(days=7)).isoformat()
|
||
events_query = select(UsageEvent).where(UsageEvent.created_at >= cutoff)
|
||
if organization_id is not None:
|
||
events_query = events_query.where(UsageEvent.organization_id == int(organization_id))
|
||
if not is_admin:
|
||
events_query = events_query.where(UsageEvent.user_id == int(owner_user_id))
|
||
events = session.exec(events_query).all()
|
||
amount_minor = sum(int(e.amount_minor or 0) for e in events)
|
||
tokens = sum(int(e.quantity or 0) for e in events if e.service_type == "llm")
|
||
|
||
kb = kb_metrics_overview(session, owner_user_id=owner_user_id if not is_admin else None,
|
||
organization_id=organization_id)
|
||
|
||
# Provider 健康是系统级信息，仅管理员可见；普通用户不返回
|
||
provider_health = []
|
||
if is_admin:
|
||
providers = session.exec(select(ModelProvider)).all()
|
||
provider_health = [{
|
||
"id": p.id, "name": p.name, "enabled": p.enabled,
|
||
"backend": "keychain" if (p.api_key or "").startswith("keychain:") else "database",
|
||
"models": len(p.models or []),
|
||
} for p in providers]
|
||
|
||
dead_query = select(TaskQueueItem).where(TaskQueueItem.status == "dead")
|
||
dead_items = session.exec(dead_query).all()
|
||
if not is_admin:
|
||
# 队列项没有 user_id 字段：按 payload 里记录的任务归属过滤
|
||
own_campaign_ids = {int(c.id or 0) for c in campaigns}
|
||
dead_items = [item for item in dead_items
|
||
if int((item.payload or {}).get("campaign_id", 0) or 0) in own_campaign_ids]
|
||
dead_list = [{"id": item.id, "task_key": item.task_key, "error": (item.error or "")[:160],
|
||
"attempt": item.attempt, "updated_at": item.updated_at} for item in dead_items]
|
||
|
||
# 告警隔离：管理员看系统告警，普通用户只看自己任务相关的通知
|
||
if is_admin:
|
||
alerts = session.exec(select(Notification).where(
|
||
Notification.type == "alert").order_by(Notification.id.desc()).limit(20)).all()
|
||
else:
|
||
own_campaign_ids = {int(c.id or 0) for c in campaigns}
|
||
alerts = [n for n in session.exec(select(Notification).where(
|
||
Notification.type == "alert").order_by(Notification.id.desc()).limit(100)).all()
|
||
if int(n.ref_id or 0) in own_campaign_ids][:20]
|
||
|
||
scope = {"owner_user_id": int(owner_user_id), "scope": "self"} if not is_admin else {"scope": "org"}
|
||
return {
|
||
**scope,
|
||
"queue": queue,
|
||
"campaigns": {"active": active, "failed": failed, "done": done, "total": len(campaigns)},
|
||
"cost_7d": {"amount_minor": amount_minor, "tokens": tokens},
|
||
"knowledge": {**({"hit_rate": kb["hit_rate"], "repeat_rate": kb["repeat_rate"],
|
||
"reference_coverage": kb["reference_coverage"],
|
||
"by_kb": kb["by_kb"][:5]} if is_admin else
|
||
{"hit_rate": 0, "repeat_rate": 0, "reference_coverage": 0, "by_kb": []})},
|
||
"providers": provider_health,
|
||
"dead_letter": dead_list,
|
||
"backup": backup_mod.backup_status() if is_admin else {},
|
||
"recent_alerts": [{"id": n.id, "message": n.message, "created_at": n.created_at}
|
||
for n in alerts],
|
||
"generated_at": now(),
|
||
}
|
||
|
||
|
||
__all__ = ["raise_alert", "run_alert_sweep", "ops_dashboard",
|
||
"ALERT_DEDUP_WINDOW_MINUTES", "DISK_MIN_FREE_GB", "POINTS_LOW_THRESHOLD"]
|