356 lines
19 KiB
Python
356 lines
19 KiB
Python
"""LLM 多 Provider 客户端：重试/错误分类/拒审检测/调用日志"""
|
||
import json
|
||
import re
|
||
import time
|
||
import hashlib
|
||
import uuid
|
||
from sqlmodel import Session
|
||
from openai import OpenAI
|
||
|
||
from .hermes_core import retry_sleep
|
||
from .secret_store import resolve_secret
|
||
from .models import Campaign, CampaignRun, LLMCall, ModelProvider
|
||
from .billing import (consume_points, pricing_for, record_usage_event,
|
||
require_ai_points)
|
||
from . import system_log
|
||
|
||
REJECT_PATTERNS = [
|
||
"抱歉", "无法生成", "不能生成", "我不能", "无法满足", "露骨", "合规",
|
||
"安全地", "作为AI", "作为人工智能", "色情内容", "不合适",
|
||
]
|
||
|
||
def humanize_error(kind: str, raw: str) -> str:
|
||
"""把错误翻译成用户能行动的话；限流类明确告知原因和换模型建议。"""
|
||
if kind == "rate_limit":
|
||
wait = ""
|
||
m = re.search(r"try again in ([\d.]+)s", raw, re.IGNORECASE)
|
||
if m:
|
||
wait = f"（建议约 {m.group(1)} 秒后重试）"
|
||
return (f"模型服务商限流（429）：请求太频繁或超出并发额度{wait}。"
|
||
"系统会自动放慢重试；若持续失败，建议在「模型管理」更换其他模型或 Provider")
|
||
if kind == "auth":
|
||
return "模型服务认证失败：API Key 无效或已过期，请到「模型管理」核对密钥"
|
||
if kind == "model_not_found":
|
||
return "模型不存在或未开通：请到「模型管理」更换该 Provider 下的其他模型"
|
||
if kind == "timeout":
|
||
return "模型响应超时：可能是网络波动或服务商繁忙，系统会自动重试"
|
||
if kind == "connection":
|
||
return "连接模型服务失败：请检查 Provider 地址/网络；持续失败建议更换其他模型"
|
||
if kind == "balance":
|
||
return "服务商账户余额不足：请到服务商平台充值，或更换其他 Provider"
|
||
return raw
|
||
|
||
|
||
def classify_error(e: Exception) -> str:
|
||
msg = str(e).lower()
|
||
if "timeout" in msg or "timed out" in msg or "read" in msg and "deadline" in msg:
|
||
return "timeout"
|
||
if "429" in msg or "rate limit" in msg or "quota" in msg:
|
||
return "rate_limit"
|
||
if "401" in msg or "403" in msg or "invalid api key" in msg or "authentication" in msg:
|
||
return "auth"
|
||
if "404" in msg or "model" in msg and "not found" in msg or "does not exist" in msg:
|
||
return "model_not_found"
|
||
if "balance" in msg or "insufficient" in msg or "余额" in msg:
|
||
return "balance"
|
||
if "connection" in msg or "connect" in msg:
|
||
return "connection"
|
||
return "other"
|
||
|
||
def is_rejection(text: str) -> bool:
|
||
head = text[:120]
|
||
return any(p in head for p in REJECT_PATTERNS)
|
||
|
||
def retryable(kind: str) -> bool:
|
||
return kind in ("timeout", "rate_limit", "connection")
|
||
|
||
def tools_unsupported(e: Exception) -> bool:
|
||
message = str(e).lower()
|
||
return ("tool" in message or "function" in message) and any(token in message for token in (
|
||
"unsupported", "not support", "unknown parameter", "invalid parameter", "not allowed",
|
||
))
|
||
|
||
def get_provider(session: Session, provider_id: int) -> ModelProvider:
|
||
p = session.get(ModelProvider, provider_id)
|
||
if not p or not p.enabled:
|
||
raise ValueError(f"Provider {provider_id} 不可用")
|
||
return p
|
||
|
||
def _payload_hash(value) -> str:
|
||
try:
|
||
raw = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str)
|
||
except Exception:
|
||
raw = str(value)
|
||
return hashlib.sha256(raw.encode("utf-8", errors="ignore")).hexdigest()
|
||
|
||
|
||
def _write_ai_log(session: Session, *, level: str, message: str,
|
||
trace_id: str = "", status: str = "ok") -> None:
|
||
"""Best-effort AI observability; logging must never break generation or tests."""
|
||
try:
|
||
# Ordinary AI lifecycle messages use the bounded writer so generation
|
||
# does not wait on SQLite. ``emit`` still falls back to synchronous
|
||
# persistence for ERROR and other critical levels.
|
||
system_log.emit(session, level=level, source="ai", message=message,
|
||
trace_id=trace_id, status=status)
|
||
except Exception:
|
||
# Fake sessions and a temporarily unavailable log table must not mask the
|
||
# provider response. LLMCall/UsageEvent remain the primary audit trail.
|
||
try:
|
||
rollback = getattr(session, "rollback", None)
|
||
if rollback:
|
||
rollback()
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _usage_owner(session: Session, user_id: int, organization_id: int,
|
||
campaign_id: int = 0, run_id: int = 0) -> tuple[int, int]:
|
||
"""运行器调用未显式传用户时，从持久化运行记录恢复归属。"""
|
||
user = int(user_id or 0)
|
||
organization = int(organization_id or 0)
|
||
run = session.get(CampaignRun, run_id) if run_id else None
|
||
campaign = session.get(Campaign, campaign_id) if campaign_id else None
|
||
if not user:
|
||
user = int((run.user_id if run else 0) or (campaign.owner_user_id if campaign else 0) or 0)
|
||
if not organization:
|
||
organization = int((run.organization_id if run else 0) or
|
||
(campaign.organization_id if campaign else 1) or 1)
|
||
return user, organization
|
||
|
||
def chat(session: Session, messages: list, provider_id: int, model: str,
|
||
campaign_id: int = 0, segment_id: int = 0, agent_id: int = 0, agent_name: str = "",
|
||
run_id: int = 0, trace_id: str = "", span_id: str = "", attempt: int = 1,
|
||
tools: list | None = None, max_retries: int = 2, temperature: float = 0.8,
|
||
user_id: int = 0, organization_id: int = 0, _failover_left: int = 1,
|
||
sdk_retries: int | None = None, on_rate_limit=None, before_request=None) -> dict:
|
||
"""调用 LLM，返回 {text, status, error, provider, model, tokens, duration_ms}
|
||
status: ok | rejected | failed
|
||
Hermes 融合：主 Provider 失败且属于可切换错误时，自动 failover 到下一个
|
||
启用 Provider 重试（每条调用链最多一次）。"""
|
||
p = get_provider(session, provider_id)
|
||
resolved_user, resolved_org = _usage_owner(session, user_id, organization_id,
|
||
campaign_id, run_id)
|
||
try:
|
||
require_ai_points(session, user_id=resolved_user, organization_id=resolved_org)
|
||
except ValueError as exc:
|
||
# 点数不足也是一次可追踪的 AI 操作，写入系统日志供管理员定位。
|
||
_write_ai_log(session, level="WARNING",
|
||
message=f"AI 调用未执行 · 模型：{model} · 原因：{exc}",
|
||
trace_id=trace_id, status="failed")
|
||
return {"text": "", "status": "failed", "error": str(exc),
|
||
"provider": p.name, "model": model, "tokens": 0, "duration_ms": 0}
|
||
api_key = resolve_secret(p.api_key, f"provider-{p.id}")
|
||
if not api_key:
|
||
error = '没有读取到可用模型密钥，请在模型管理重新填写并保存；已保存时请检查服务环境与密钥存储'
|
||
_write_ai_log(session, level='ERROR', message=f'AI 调用未执行 · Provider：{p.name} · 原因：{error}',
|
||
trace_id=trace_id, status='failed')
|
||
return {'text': '', 'status': 'failed', 'error': error, 'provider': p.name,
|
||
'model': model, 'tokens': 0, 'duration_ms': 0, 'error_kind': 'credential_unavailable'}
|
||
client = OpenAI(base_url=p.base_url,
|
||
api_key=api_key,
|
||
timeout=120, **({'max_retries': sdk_retries} if sdk_retries is not None else {}))
|
||
t0 = time.time()
|
||
retry_attempt = 0
|
||
last_err = ""
|
||
prompt_hash = _payload_hash(messages)
|
||
tool_fallback_used = False
|
||
while True:
|
||
if before_request is not None:
|
||
before_request()
|
||
try:
|
||
request = {"model": model, "messages": messages, "temperature": temperature}
|
||
if tools:
|
||
request["tools"] = tools
|
||
request["tool_choice"] = "auto"
|
||
resp = client.chat.completions.create(**request)
|
||
text = resp.choices[0].message.content or ""
|
||
raw_calls = getattr(resp.choices[0].message, "tool_calls", None) or []
|
||
tool_calls = [{
|
||
"id": str(getattr(call, "id", "")),
|
||
"type": str(getattr(call, "type", "function")),
|
||
"function": {
|
||
"name": str(getattr(getattr(call, "function", None), "name", "")),
|
||
"arguments": str(getattr(getattr(call, "function", None), "arguments", "{}")),
|
||
},
|
||
} for call in raw_calls]
|
||
dt = int((time.time() - t0) * 1000)
|
||
prompt_tk = resp.usage.prompt_tokens if resp.usage else 0
|
||
comp_tk = resp.usage.completion_tokens if resp.usage else 0
|
||
status = "rejected" if is_rejection(text) else "ok"
|
||
point_charged = _log(session, run_id, campaign_id, segment_id, agent_id, agent_name, p.name, model,
|
||
status, "", prompt_tk, comp_tk, dt, trace_id, span_id, attempt,
|
||
prompt_hash, _payload_hash({"text": text, "tool_calls": tool_calls}),
|
||
provider_id=p.id or provider_id, user_id=resolved_user,
|
||
organization_id=resolved_org)
|
||
if status == "ok" and resolved_user and not point_charged:
|
||
session.commit()
|
||
_write_ai_log(session, level="WARNING",
|
||
message=f"内容生成未交付 · Agent：{agent_name or '系统'} · 模型：{model} · 原因：点数余额不足 · 输入 {prompt_tk} Token · 输出 {comp_tk} Token",
|
||
trace_id=trace_id, status="failed")
|
||
return {"text": "", "status": "failed", "error": "点数余额不足，本次生成未交付，请联系管理员增加点数",
|
||
"provider": p.name, "model": model, "tokens": 0, "duration_ms": dt}
|
||
if status == "rejected":
|
||
session.commit()
|
||
elif status == "ok":
|
||
# _log() flushes the usage event and point charge. Leaving
|
||
# that transaction open while the caller waits for the next
|
||
# model request turns SQLite's single writer lock into a
|
||
# system-wide stall (health checks, estimates and drafts then
|
||
# appear as 524). Commit each completed call before emitting
|
||
# the asynchronous observability record so long AI jobs do not
|
||
# hold the database lock across network waits.
|
||
session.commit()
|
||
# 记录 AI 调用结果与拒答原因，但只保留有限摘要，不落库完整提示词/正文。
|
||
summary = f"输出 {len(text)} 字符"
|
||
if status == "rejected":
|
||
summary += f" · 模型返回：{system_log.safe_detail(text, limit=240)}"
|
||
_write_ai_log(session, level="WARNING" if status == "rejected" else "INFO",
|
||
message=f"AI{'内容生成' if campaign_id or segment_id else '调用'}完成 · Agent：{agent_name or '系统'} · 模型：{model} · {summary} · 输入 {prompt_tk} Token · 输出 {comp_tk} Token",
|
||
trace_id=trace_id, status="rejected" if status == "rejected" else "success")
|
||
return {"text": text, "status": status, "provider": p.name, "model": model,
|
||
"tokens": prompt_tk + comp_tk, "duration_ms": dt, "tool_calls": tool_calls}
|
||
except Exception as e:
|
||
last_err = str(e)
|
||
kind = classify_error(e)
|
||
if kind == "rate_limit" and on_rate_limit is not None:
|
||
on_rate_limit()
|
||
if tools and not tool_fallback_used and tools_unsupported(e):
|
||
# 某些 OpenAI 兼容 Provider 不实现 tools；保留同一上下文退化为单轮输出。
|
||
tools = None
|
||
tool_fallback_used = True
|
||
continue
|
||
retry_attempt += 1
|
||
if retryable(kind) and retry_attempt <= max_retries:
|
||
# Hermes 融合：抖动退避 + 服务端 Retry-After，防并发重试尖峰。
|
||
headers = getattr(e, "response", None)
|
||
headers = getattr(headers, "headers", None) if headers is not None else None
|
||
time.sleep(retry_sleep(retry_attempt, retry_after=headers))
|
||
continue
|
||
dt = int((time.time() - t0) * 1000)
|
||
_log(session, run_id, campaign_id, segment_id, agent_id, agent_name, p.name, model,
|
||
"error", f"{kind}: {last_err[:300]}", 0, 0, dt, trace_id, span_id,
|
||
attempt, prompt_hash, "", provider_id=p.id or provider_id,
|
||
user_id=resolved_user, organization_id=resolved_org)
|
||
session.commit()
|
||
_write_ai_log(session, level="ERROR",
|
||
message=f"AI{'内容生成' if campaign_id or segment_id else '调用'}失败 · Agent：{agent_name or '系统'} · 模型：{model} · 错误类型：{kind} · 详情：{system_log.safe_detail(last_err, limit=300)}",
|
||
trace_id=trace_id, status="failed")
|
||
# Hermes 融合：主 Provider 报可切换错误时，自动切下一个 Provider 重试一次。
|
||
if _failover_left > 0:
|
||
from .failover import failover_target, should_failover
|
||
if should_failover(kind):
|
||
target = failover_target(session, p.id or provider_id, model)
|
||
if target:
|
||
alt_provider, alt_model = target
|
||
session.commit()
|
||
result = chat(
|
||
session, messages, alt_provider.id, alt_model,
|
||
campaign_id=campaign_id, segment_id=segment_id,
|
||
agent_id=agent_id, agent_name=agent_name, run_id=run_id,
|
||
trace_id=trace_id, span_id=span_id, attempt=attempt,
|
||
tools=tools, max_retries=max_retries, temperature=temperature,
|
||
user_id=resolved_user, organization_id=resolved_org,
|
||
_failover_left=0, sdk_retries=sdk_retries, on_rate_limit=on_rate_limit,
|
||
before_request=before_request)
|
||
if result.get("status") == "ok":
|
||
result["failover"] = {"from_provider": p.name, "from_error": kind}
|
||
return result
|
||
return {"text": "", "status": "failed",
|
||
"error": humanize_error(kind, last_err[:300]),
|
||
"provider": p.name, "model": model, "tokens": 0, "duration_ms": dt,
|
||
"error_kind": kind}
|
||
|
||
def chat_json(session: Session, messages: list, provider_id: int, model: str, **kw) -> dict:
|
||
"""要求模型输出 JSON，容错解析；解析失败自动重试 1 次"""
|
||
msgs = messages + [{"role": "user", "content":
|
||
"只输出一个合法的 JSON 对象，不要输出任何其他文字、markdown 代码块标记或解释。"}]
|
||
r = chat(session, msgs, provider_id, model, **kw)
|
||
if r["status"] != "ok":
|
||
return r
|
||
text = r["text"].strip()
|
||
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text)
|
||
try:
|
||
r["json"] = json.loads(text)
|
||
return r
|
||
except Exception:
|
||
m = re.search(r"\{[\s\S]*\}", text)
|
||
if m:
|
||
try:
|
||
r["json"] = json.loads(m.group(0))
|
||
return r
|
||
except Exception:
|
||
pass
|
||
# JSON 解析失败 → 自动重试 1 次（换措辞）
|
||
r2 = chat(session, msgs, provider_id, model, **kw)
|
||
if r2["status"] != "ok":
|
||
return r2
|
||
text2 = re.sub(r"^```(?:json)?\s*|\s*```$", "", r2["text"].strip())
|
||
try:
|
||
r2["json"] = json.loads(text2)
|
||
except Exception:
|
||
m2 = re.search(r"\{[\s\S]*\}", text2)
|
||
if m2:
|
||
try:
|
||
r2["json"] = json.loads(m2.group(0))
|
||
except Exception:
|
||
r2["status"] = "failed"
|
||
r2["error"] = "JSON 解析失败（重试后仍失败）"
|
||
else:
|
||
r2["status"] = "failed"
|
||
r2["error"] = "JSON 解析失败（重试后仍失败）"
|
||
return r2
|
||
|
||
def _log(session, run_id, campaign_id, segment_id, agent_id, agent_name, provider, model,
|
||
status, error, pt, ct, dt, trace_id="", span_id="", attempt=1,
|
||
prompt_hash="", output_hash="", provider_id=0, user_id=0, organization_id=1):
|
||
"""写入调用日志并同步追加一条不可变用量事件。"""
|
||
owner = session.get(Campaign, campaign_id) if campaign_id else None
|
||
run = session.get(CampaignRun, run_id) if run_id else None
|
||
resolved_user = int(user_id or (run.user_id if run else 0) or (owner.owner_user_id if owner else 0) or 0)
|
||
resolved_org = int(organization_id or (run.organization_id if run else 1) or (owner.organization_id if owner else 1) or 1)
|
||
call = LLMCall(run_id=run_id, campaign_id=campaign_id, segment_id=segment_id, agent_id=agent_id,
|
||
agent_name=agent_name, provider=provider, model=model,
|
||
status=status, error=error, prompt_tokens=pt,
|
||
completion_tokens=ct, duration_ms=dt, trace_id=trace_id,
|
||
span_id=span_id, attempt=attempt, prompt_hash=prompt_hash,
|
||
output_hash=output_hash, user_id=resolved_user,
|
||
organization_id=resolved_org, billing_status="measured" if pt or ct else "estimated")
|
||
session.add(call)
|
||
session.flush()
|
||
rates = pricing_for(session, provider_id, model)
|
||
event_key = f"llm:{trace_id or uuid.uuid4().hex}:{span_id or uuid.uuid4().hex}:{attempt}:{call.id}"
|
||
billable_tokens = max(1, int(pt or 0) + int(ct or 0)) if status == "ok" else 0
|
||
event = record_usage_event(
|
||
session, idempotency_key=event_key, user_id=resolved_user, organization_id=resolved_org,
|
||
campaign_id=campaign_id, run_id=run_id, segment_id=segment_id, trace_id=trace_id,
|
||
llm_call_id=call.id, agent_id=agent_id, provider_id=provider_id,
|
||
provider=provider, model=model, service_type="llm", unit="token",
|
||
quantity=billable_tokens,
|
||
input_quantity=pt, output_quantity=ct, input_rate=rates["input_rate"],
|
||
output_rate=rates["output_rate"], pricing_version=rates["version"],
|
||
status="measured" if pt or ct else "estimated",
|
||
metadata={"status": status, "error": error[:300], "prompt_hash": prompt_hash,
|
||
"output_hash": output_hash,
|
||
# 用量明细直接展示用户能理解的业务动作，而不是只显示 LLM。
|
||
"operation": "内容生成" if (campaign_id or segment_id) else "AI 调用",
|
||
"agent_name": agent_name or "系统"},
|
||
)
|
||
call.usage_event_id = event.id
|
||
call.billing_status = event.status
|
||
session.add(call)
|
||
if status != "ok" or not resolved_user:
|
||
return True
|
||
charged = consume_points(
|
||
session, user_id=resolved_user, organization_id=resolved_org,
|
||
points=billable_tokens * 10, reason=f"AI 模型调用：{model}",
|
||
idempotency_key=f"points:{event.idempotency_key}",
|
||
metadata={"usage_event_id": event.id, "llm_call_id": call.id,
|
||
"tokens": billable_tokens, "model": model},
|
||
)
|
||
if not charged:
|
||
event.usage_metadata = {**(event.usage_metadata or {}),
|
||
"point_charge": "insufficient"}
|
||
session.add(event)
|
||
return charged
|