"""LLM 多 Provider 客户端:重试/错误分类/拒审检测/调用日志""" import json import re import time import hashlib import uuid from sqlmodel import Session from openai import OpenAI from .hermes_core import retry_sleep from .secret_store import resolve_secret from .models import Campaign, CampaignRun, LLMCall, ModelProvider from .billing import (consume_points, pricing_for, record_usage_event, require_ai_points) from . import system_log REJECT_PATTERNS = [ "抱歉", "无法生成", "不能生成", "我不能", "无法满足", "露骨", "合规", "安全地", "作为AI", "作为人工智能", "色情内容", "不合适", ] def humanize_error(kind: str, raw: str) -> str: """把错误翻译成用户能行动的话;限流类明确告知原因和换模型建议。""" if kind == "rate_limit": wait = "" m = re.search(r"try again in ([\d.]+)s", raw, re.IGNORECASE) if m: wait = f"(建议约 {m.group(1)} 秒后重试)" return (f"模型服务商限流(429):请求太频繁或超出并发额度{wait}。" "系统会自动放慢重试;若持续失败,建议在「模型管理」更换其他模型或 Provider") if kind == "auth": return "模型服务认证失败:API Key 无效或已过期,请到「模型管理」核对密钥" if kind == "model_not_found": return "模型不存在或未开通:请到「模型管理」更换该 Provider 下的其他模型" if kind == "timeout": return "模型响应超时:可能是网络波动或服务商繁忙,系统会自动重试" if kind == "connection": return "连接模型服务失败:请检查 Provider 地址/网络;持续失败建议更换其他模型" if kind == "balance": return "服务商账户余额不足:请到服务商平台充值,或更换其他 Provider" return raw def classify_error(e: Exception) -> str: msg = str(e).lower() if "timeout" in msg or "timed out" in msg or "read" in msg and "deadline" in msg: return "timeout" if "429" in msg or "rate limit" in msg or "quota" in msg: return "rate_limit" if "401" in msg or "403" in msg or "invalid api key" in msg or "authentication" in msg: return "auth" if "404" in msg or "model" in msg and "not found" in msg or "does not exist" in msg: return "model_not_found" if "balance" in msg or "insufficient" in msg or "余额" in msg: return "balance" if "connection" in msg or "connect" in msg: return "connection" return "other" def is_rejection(text: str) -> bool: head = text[:120] return any(p in head for p in REJECT_PATTERNS) def retryable(kind: str) -> bool: return kind in ("timeout", "rate_limit", "connection") def tools_unsupported(e: Exception) -> bool: message = str(e).lower() return ("tool" in message or "function" in message) and any(token in message for token in ( "unsupported", "not support", "unknown parameter", "invalid parameter", "not allowed", )) def get_provider(session: Session, provider_id: int) -> ModelProvider: p = session.get(ModelProvider, provider_id) if not p or not p.enabled: raise ValueError(f"Provider {provider_id} 不可用") return p def _payload_hash(value) -> str: try: raw = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) except Exception: raw = str(value) return hashlib.sha256(raw.encode("utf-8", errors="ignore")).hexdigest() def _write_ai_log(session: Session, *, level: str, message: str, trace_id: str = "", status: str = "ok") -> None: """Best-effort AI observability; logging must never break generation or tests.""" try: # Ordinary AI lifecycle messages use the bounded writer so generation # does not wait on SQLite. ``emit`` still falls back to synchronous # persistence for ERROR and other critical levels. system_log.emit(session, level=level, source="ai", message=message, trace_id=trace_id, status=status) except Exception: # Fake sessions and a temporarily unavailable log table must not mask the # provider response. LLMCall/UsageEvent remain the primary audit trail. try: rollback = getattr(session, "rollback", None) if rollback: rollback() except Exception: pass def _usage_owner(session: Session, user_id: int, organization_id: int, campaign_id: int = 0, run_id: int = 0) -> tuple[int, int]: """运行器调用未显式传用户时,从持久化运行记录恢复归属。""" user = int(user_id or 0) organization = int(organization_id or 0) run = session.get(CampaignRun, run_id) if run_id else None campaign = session.get(Campaign, campaign_id) if campaign_id else None if not user: user = int((run.user_id if run else 0) or (campaign.owner_user_id if campaign else 0) or 0) if not organization: organization = int((run.organization_id if run else 0) or (campaign.organization_id if campaign else 1) or 1) return user, organization def chat(session: Session, messages: list, provider_id: int, model: str, campaign_id: int = 0, segment_id: int = 0, agent_id: int = 0, agent_name: str = "", run_id: int = 0, trace_id: str = "", span_id: str = "", attempt: int = 1, tools: list | None = None, max_retries: int = 2, temperature: float = 0.8, user_id: int = 0, organization_id: int = 0, _failover_left: int = 1, sdk_retries: int | None = None, on_rate_limit=None, before_request=None) -> dict: """调用 LLM,返回 {text, status, error, provider, model, tokens, duration_ms} status: ok | rejected | failed Hermes 融合:主 Provider 失败且属于可切换错误时,自动 failover 到下一个 启用 Provider 重试(每条调用链最多一次)。""" p = get_provider(session, provider_id) resolved_user, resolved_org = _usage_owner(session, user_id, organization_id, campaign_id, run_id) try: require_ai_points(session, user_id=resolved_user, organization_id=resolved_org) except ValueError as exc: # 点数不足也是一次可追踪的 AI 操作,写入系统日志供管理员定位。 _write_ai_log(session, level="WARNING", message=f"AI 调用未执行 · 模型:{model} · 原因:{exc}", trace_id=trace_id, status="failed") return {"text": "", "status": "failed", "error": str(exc), "provider": p.name, "model": model, "tokens": 0, "duration_ms": 0} api_key = resolve_secret(p.api_key, f"provider-{p.id}") if not api_key: error = '没有读取到可用模型密钥,请在模型管理重新填写并保存;已保存时请检查服务环境与密钥存储' _write_ai_log(session, level='ERROR', message=f'AI 调用未执行 · Provider:{p.name} · 原因:{error}', trace_id=trace_id, status='failed') return {'text': '', 'status': 'failed', 'error': error, 'provider': p.name, 'model': model, 'tokens': 0, 'duration_ms': 0, 'error_kind': 'credential_unavailable'} client = OpenAI(base_url=p.base_url, api_key=api_key, timeout=120, **({'max_retries': sdk_retries} if sdk_retries is not None else {})) t0 = time.time() retry_attempt = 0 last_err = "" prompt_hash = _payload_hash(messages) tool_fallback_used = False while True: if before_request is not None: before_request() try: request = {"model": model, "messages": messages, "temperature": temperature} if tools: request["tools"] = tools request["tool_choice"] = "auto" resp = client.chat.completions.create(**request) text = resp.choices[0].message.content or "" raw_calls = getattr(resp.choices[0].message, "tool_calls", None) or [] tool_calls = [{ "id": str(getattr(call, "id", "")), "type": str(getattr(call, "type", "function")), "function": { "name": str(getattr(getattr(call, "function", None), "name", "")), "arguments": str(getattr(getattr(call, "function", None), "arguments", "{}")), }, } for call in raw_calls] dt = int((time.time() - t0) * 1000) prompt_tk = resp.usage.prompt_tokens if resp.usage else 0 comp_tk = resp.usage.completion_tokens if resp.usage else 0 status = "rejected" if is_rejection(text) else "ok" point_charged = _log(session, run_id, campaign_id, segment_id, agent_id, agent_name, p.name, model, status, "", prompt_tk, comp_tk, dt, trace_id, span_id, attempt, prompt_hash, _payload_hash({"text": text, "tool_calls": tool_calls}), provider_id=p.id or provider_id, user_id=resolved_user, organization_id=resolved_org) if status == "ok" and resolved_user and not point_charged: session.commit() _write_ai_log(session, level="WARNING", message=f"内容生成未交付 · Agent:{agent_name or '系统'} · 模型:{model} · 原因:点数余额不足 · 输入 {prompt_tk} Token · 输出 {comp_tk} Token", trace_id=trace_id, status="failed") return {"text": "", "status": "failed", "error": "点数余额不足,本次生成未交付,请联系管理员增加点数", "provider": p.name, "model": model, "tokens": 0, "duration_ms": dt} if status == "rejected": session.commit() elif status == "ok": # _log() flushes the usage event and point charge. Leaving # that transaction open while the caller waits for the next # model request turns SQLite's single writer lock into a # system-wide stall (health checks, estimates and drafts then # appear as 524). Commit each completed call before emitting # the asynchronous observability record so long AI jobs do not # hold the database lock across network waits. session.commit() # 记录 AI 调用结果与拒答原因,但只保留有限摘要,不落库完整提示词/正文。 summary = f"输出 {len(text)} 字符" if status == "rejected": summary += f" · 模型返回:{system_log.safe_detail(text, limit=240)}" _write_ai_log(session, level="WARNING" if status == "rejected" else "INFO", message=f"AI{'内容生成' if campaign_id or segment_id else '调用'}完成 · Agent:{agent_name or '系统'} · 模型:{model} · {summary} · 输入 {prompt_tk} Token · 输出 {comp_tk} Token", trace_id=trace_id, status="rejected" if status == "rejected" else "success") return {"text": text, "status": status, "provider": p.name, "model": model, "tokens": prompt_tk + comp_tk, "duration_ms": dt, "tool_calls": tool_calls} except Exception as e: last_err = str(e) kind = classify_error(e) if kind == "rate_limit" and on_rate_limit is not None: on_rate_limit() if tools and not tool_fallback_used and tools_unsupported(e): # 某些 OpenAI 兼容 Provider 不实现 tools;保留同一上下文退化为单轮输出。 tools = None tool_fallback_used = True continue retry_attempt += 1 if retryable(kind) and retry_attempt <= max_retries: # Hermes 融合:抖动退避 + 服务端 Retry-After,防并发重试尖峰。 headers = getattr(e, "response", None) headers = getattr(headers, "headers", None) if headers is not None else None time.sleep(retry_sleep(retry_attempt, retry_after=headers)) continue dt = int((time.time() - t0) * 1000) _log(session, run_id, campaign_id, segment_id, agent_id, agent_name, p.name, model, "error", f"{kind}: {last_err[:300]}", 0, 0, dt, trace_id, span_id, attempt, prompt_hash, "", provider_id=p.id or provider_id, user_id=resolved_user, organization_id=resolved_org) session.commit() _write_ai_log(session, level="ERROR", message=f"AI{'内容生成' if campaign_id or segment_id else '调用'}失败 · Agent:{agent_name or '系统'} · 模型:{model} · 错误类型:{kind} · 详情:{system_log.safe_detail(last_err, limit=300)}", trace_id=trace_id, status="failed") # Hermes 融合:主 Provider 报可切换错误时,自动切下一个 Provider 重试一次。 if _failover_left > 0: from .failover import failover_target, should_failover if should_failover(kind): target = failover_target(session, p.id or provider_id, model) if target: alt_provider, alt_model = target session.commit() result = chat( session, messages, alt_provider.id, alt_model, campaign_id=campaign_id, segment_id=segment_id, agent_id=agent_id, agent_name=agent_name, run_id=run_id, trace_id=trace_id, span_id=span_id, attempt=attempt, tools=tools, max_retries=max_retries, temperature=temperature, user_id=resolved_user, organization_id=resolved_org, _failover_left=0, sdk_retries=sdk_retries, on_rate_limit=on_rate_limit, before_request=before_request) if result.get("status") == "ok": result["failover"] = {"from_provider": p.name, "from_error": kind} return result return {"text": "", "status": "failed", "error": humanize_error(kind, last_err[:300]), "provider": p.name, "model": model, "tokens": 0, "duration_ms": dt, "error_kind": kind} def chat_json(session: Session, messages: list, provider_id: int, model: str, **kw) -> dict: """要求模型输出 JSON,容错解析;解析失败自动重试 1 次""" msgs = messages + [{"role": "user", "content": "只输出一个合法的 JSON 对象,不要输出任何其他文字、markdown 代码块标记或解释。"}] r = chat(session, msgs, provider_id, model, **kw) if r["status"] != "ok": return r text = r["text"].strip() text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text) try: r["json"] = json.loads(text) return r except Exception: m = re.search(r"\{[\s\S]*\}", text) if m: try: r["json"] = json.loads(m.group(0)) return r except Exception: pass # JSON 解析失败 → 自动重试 1 次(换措辞) r2 = chat(session, msgs, provider_id, model, **kw) if r2["status"] != "ok": return r2 text2 = re.sub(r"^```(?:json)?\s*|\s*```$", "", r2["text"].strip()) try: r2["json"] = json.loads(text2) except Exception: m2 = re.search(r"\{[\s\S]*\}", text2) if m2: try: r2["json"] = json.loads(m2.group(0)) except Exception: r2["status"] = "failed" r2["error"] = "JSON 解析失败(重试后仍失败)" else: r2["status"] = "failed" r2["error"] = "JSON 解析失败(重试后仍失败)" return r2 def _log(session, run_id, campaign_id, segment_id, agent_id, agent_name, provider, model, status, error, pt, ct, dt, trace_id="", span_id="", attempt=1, prompt_hash="", output_hash="", provider_id=0, user_id=0, organization_id=1): """写入调用日志并同步追加一条不可变用量事件。""" owner = session.get(Campaign, campaign_id) if campaign_id else None run = session.get(CampaignRun, run_id) if run_id else None resolved_user = int(user_id or (run.user_id if run else 0) or (owner.owner_user_id if owner else 0) or 0) resolved_org = int(organization_id or (run.organization_id if run else 1) or (owner.organization_id if owner else 1) or 1) call = LLMCall(run_id=run_id, campaign_id=campaign_id, segment_id=segment_id, agent_id=agent_id, agent_name=agent_name, provider=provider, model=model, status=status, error=error, prompt_tokens=pt, completion_tokens=ct, duration_ms=dt, trace_id=trace_id, span_id=span_id, attempt=attempt, prompt_hash=prompt_hash, output_hash=output_hash, user_id=resolved_user, organization_id=resolved_org, billing_status="measured" if pt or ct else "estimated") session.add(call) session.flush() rates = pricing_for(session, provider_id, model) event_key = f"llm:{trace_id or uuid.uuid4().hex}:{span_id or uuid.uuid4().hex}:{attempt}:{call.id}" billable_tokens = max(1, int(pt or 0) + int(ct or 0)) if status == "ok" else 0 event = record_usage_event( session, idempotency_key=event_key, user_id=resolved_user, organization_id=resolved_org, campaign_id=campaign_id, run_id=run_id, segment_id=segment_id, trace_id=trace_id, llm_call_id=call.id, agent_id=agent_id, provider_id=provider_id, provider=provider, model=model, service_type="llm", unit="token", quantity=billable_tokens, input_quantity=pt, output_quantity=ct, input_rate=rates["input_rate"], output_rate=rates["output_rate"], pricing_version=rates["version"], status="measured" if pt or ct else "estimated", metadata={"status": status, "error": error[:300], "prompt_hash": prompt_hash, "output_hash": output_hash, # 用量明细直接展示用户能理解的业务动作,而不是只显示 LLM。 "operation": "内容生成" if (campaign_id or segment_id) else "AI 调用", "agent_name": agent_name or "系统"}, ) call.usage_event_id = event.id call.billing_status = event.status session.add(call) if status != "ok" or not resolved_user: return True charged = consume_points( session, user_id=resolved_user, organization_id=resolved_org, points=billable_tokens * 10, reason=f"AI 模型调用:{model}", idempotency_key=f"points:{event.idempotency_key}", metadata={"usage_event_id": event.id, "llm_call_id": call.id, "tokens": billable_tokens, "model": model}, ) if not charged: event.usage_metadata = {**(event.usage_metadata or {}), "point_charge": "insufficient"} session.add(event) return charged