未来
2bd8b61e91
新功能: - 大纲/章节生成改为服务端后台任务，支持断线续传 - 后台任务队列排队执行，按用户排队（同用户串行不同用户并发） - 章节管理页面添加后台任务列表弹窗和进度面板 - 章节状态添加 pending（待处理）选项 - 集成json5容错解析器 + 上下文感知JSON修复 - SSE流式生成添加心跳保活，防止连接超时 - SSEPostClient添加credentials:include修复network error - 每章最大伏笔数从2调整为5 - 添加大纲读区伏笔的功能 Bug修复: - 修复AI生成JSON中未转义引号/中文标点/多对象属性值未合并 - 修复JSON非法转义字符清洗和中文引号处理 - 修复MCP插件TimeoutError/连接失败上下文清理 - MCP插件后台注册添加重试机制 - 续写模式添加缺失的mcp_references参数 - 修复Alembic迁移链分叉 - 使用torch CPU版本加速Docker构建
435 lines
17 KiB
Python
435 lines
17 KiB
Python
"""数据库连接和会话管理 - PostgreSQL 多用户数据隔离"""
|
||
import asyncio
|
||
from typing import Dict, Any
|
||
from datetime import datetime
|
||
from sqlalchemy import select, text
|
||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
|
||
from sqlalchemy.orm import declarative_base
|
||
from fastapi import Request, HTTPException
|
||
from app.config import settings
|
||
from app.logger import get_logger
|
||
|
||
logger = get_logger(__name__)
|
||
|
||
# 创建基类
|
||
Base = declarative_base()
|
||
|
||
# 导入所有模型，确保 Base.metadata 能够发现它们
|
||
# 这必须在 Base 创建之后、init_db 之前导入
|
||
from app.models import (
|
||
Project, Outline, Character, Chapter, GenerationHistory,
|
||
Settings, WritingStyle, ProjectDefaultStyle,
|
||
RelationshipType, CharacterRelationship, Organization, OrganizationMember,
|
||
StoryMemory, PlotAnalysis, AnalysisTask, BatchGenerationTask,
|
||
RegenerationTask, Career, CharacterCareer, User, MCPPlugin, PromptTemplate,
|
||
BackgroundTask
|
||
)
|
||
|
||
# 引擎缓存：每个用户一个引擎
|
||
_engine_cache: Dict[str, Any] = {}
|
||
|
||
# 锁管理：用于保护引擎创建过程
|
||
_engine_locks: Dict[str, asyncio.Lock] = {}
|
||
_cache_lock = asyncio.Lock()
|
||
|
||
# 会话统计（用于监控连接泄漏）
|
||
_session_stats = {
|
||
"created": 0,
|
||
"closed": 0,
|
||
"active": 0,
|
||
"errors": 0,
|
||
"generator_exits": 0,
|
||
"last_check": None
|
||
}
|
||
|
||
|
||
async def get_engine(user_id: str):
|
||
"""获取或创建用户专属的数据库引擎（线程安全）
|
||
|
||
PostgreSQL: 所有用户共享一个数据库，通过user_id字段隔离数据
|
||
|
||
Args:
|
||
user_id: 用户ID
|
||
|
||
Returns:
|
||
用户专属的异步引擎
|
||
"""
|
||
# PostgreSQL模式：所有用户共享同一个引擎
|
||
cache_key = "shared_postgres"
|
||
if cache_key in _engine_cache:
|
||
return _engine_cache[cache_key]
|
||
|
||
async with _cache_lock:
|
||
if cache_key not in _engine_cache:
|
||
# 检测数据库类型
|
||
is_sqlite = 'sqlite' in settings.database_url.lower()
|
||
|
||
# 基础引擎参数
|
||
engine_args = {
|
||
"echo": settings.database_echo_pool,
|
||
"echo_pool": settings.database_echo_pool,
|
||
"future": True,
|
||
}
|
||
|
||
if is_sqlite:
|
||
# SQLite 配置（使用 NullPool，不支持连接池参数）
|
||
engine_args["connect_args"] = {
|
||
"check_same_thread": False,
|
||
"timeout": 30.0, # 等待锁释放的超时时间（秒）
|
||
}
|
||
# 启用连接前检测以支持更好的并发
|
||
engine_args["pool_pre_ping"] = True
|
||
|
||
logger.info("📊 使用 SQLite 数据库（NullPool，超时30秒，WAL模式）")
|
||
else:
|
||
# PostgreSQL 配置（完整连接池支持）
|
||
connect_args = {
|
||
"server_settings": {
|
||
"application_name": settings.app_name,
|
||
"jit": "off",
|
||
"search_path": "public",
|
||
},
|
||
"command_timeout": 60,
|
||
"statement_cache_size": 500,
|
||
}
|
||
|
||
engine_args.update({
|
||
"pool_size": settings.database_pool_size,
|
||
"max_overflow": settings.database_max_overflow,
|
||
"pool_timeout": settings.database_pool_timeout,
|
||
"pool_pre_ping": settings.database_pool_pre_ping,
|
||
"pool_recycle": settings.database_pool_recycle,
|
||
"pool_use_lifo": settings.database_pool_use_lifo,
|
||
"pool_reset_on_return": settings.database_pool_reset_on_return,
|
||
"max_identifier_length": settings.database_max_identifier_length,
|
||
"connect_args": connect_args
|
||
})
|
||
|
||
total_connections = settings.database_pool_size + settings.database_max_overflow
|
||
estimated_concurrent_users = total_connections * 2
|
||
|
||
logger.info(
|
||
f"📊 PostgreSQL 连接池配置:\n"
|
||
f" ├─ 核心连接: {settings.database_pool_size}\n"
|
||
f" ├─ 溢出连接: {settings.database_max_overflow}\n"
|
||
f" ├─ 总连接数: {total_connections}\n"
|
||
f" ├─ 获取超时: {settings.database_pool_timeout}秒\n"
|
||
f" ├─ 连接回收: {settings.database_pool_recycle}秒\n"
|
||
f" └─ 预估并发: {estimated_concurrent_users}+用户"
|
||
)
|
||
|
||
engine = create_async_engine(settings.database_url, **engine_args)
|
||
_engine_cache[cache_key] = engine
|
||
|
||
# 如果是 SQLite，启用 WAL 模式以支持读写并发
|
||
if is_sqlite:
|
||
try:
|
||
from sqlalchemy import event
|
||
from sqlalchemy.pool import NullPool
|
||
|
||
@event.listens_for(engine.sync_engine, "connect")
|
||
def set_sqlite_pragma(dbapi_conn, connection_record):
|
||
cursor = dbapi_conn.cursor()
|
||
cursor.execute("PRAGMA journal_mode=WAL")
|
||
cursor.execute("PRAGMA synchronous=NORMAL")
|
||
cursor.execute("PRAGMA cache_size=-64000") # 64MB 缓存
|
||
cursor.execute("PRAGMA busy_timeout=30000") # 30秒超时
|
||
cursor.close()
|
||
|
||
logger.info("✅ SQLite WAL 模式已启用（支持读写并发）")
|
||
except Exception as e:
|
||
logger.warning(f"⚠️ 启用 WAL 模式失败: {e}，使用默认配置")
|
||
|
||
return _engine_cache[cache_key]
|
||
|
||
|
||
async def get_db(request: Request):
|
||
"""获取数据库会话的依赖函数
|
||
|
||
从 request.state.user_id 获取用户ID，然后返回该用户的数据库会话
|
||
"""
|
||
user_id = getattr(request.state, "user_id", None)
|
||
|
||
if not user_id:
|
||
raise HTTPException(status_code=401, detail="未登录或用户ID缺失")
|
||
|
||
engine = await get_engine(user_id)
|
||
|
||
AsyncSessionLocal = async_sessionmaker(
|
||
engine,
|
||
class_=AsyncSession,
|
||
expire_on_commit=False
|
||
)
|
||
|
||
session = AsyncSessionLocal()
|
||
session_id = id(session)
|
||
|
||
global _session_stats
|
||
_session_stats["created"] += 1
|
||
_session_stats["active"] += 1
|
||
|
||
# logger.debug(f"📊 会话创建 [User:{user_id}][ID:{session_id}] - 活跃:{_session_stats['active']}, 总创建:{_session_stats['created']}, 总关闭:{_session_stats['closed']}")
|
||
|
||
try:
|
||
yield session
|
||
if session.in_transaction():
|
||
await session.rollback()
|
||
except GeneratorExit:
|
||
_session_stats["generator_exits"] += 1
|
||
logger.warning(f"⚠️ GeneratorExit [User:{user_id}][ID:{session_id}] - SSE连接断开（总计:{_session_stats['generator_exits']}次）")
|
||
try:
|
||
if session.in_transaction():
|
||
await session.rollback()
|
||
logger.info(f"✅ 事务已回滚 [User:{user_id}][ID:{session_id}]（GeneratorExit）")
|
||
except Exception as rollback_error:
|
||
_session_stats["errors"] += 1
|
||
logger.error(f"❌ GeneratorExit回滚失败 [User:{user_id}][ID:{session_id}]: {str(rollback_error)}")
|
||
except Exception as e:
|
||
_session_stats["errors"] += 1
|
||
logger.error(f"❌ 会话异常 [User:{user_id}][ID:{session_id}]: {str(e)}")
|
||
try:
|
||
if session.in_transaction():
|
||
await session.rollback()
|
||
logger.info(f"✅ 事务已回滚 [User:{user_id}][ID:{session_id}]（异常）")
|
||
except Exception as rollback_error:
|
||
logger.error(f"❌ 异常回滚失败 [User:{user_id}][ID:{session_id}]: {str(rollback_error)}")
|
||
raise
|
||
finally:
|
||
try:
|
||
if session.in_transaction():
|
||
await session.rollback()
|
||
logger.warning(f"⚠️ finally中发现未提交事务 [User:{user_id}][ID:{session_id}]，已回滚")
|
||
|
||
await session.close()
|
||
|
||
_session_stats["closed"] += 1
|
||
_session_stats["active"] -= 1
|
||
_session_stats["last_check"] = datetime.now().isoformat()
|
||
|
||
# logger.debug(f"📊 会话关闭 [User:{user_id}][ID:{session_id}] - 活跃:{_session_stats['active']}, 总创建:{_session_stats['created']}, 总关闭:{_session_stats['closed']}, 错误:{_session_stats['errors']}")
|
||
|
||
# 使用优化后的会话监控阈值
|
||
if _session_stats["active"] > settings.database_session_leak_threshold:
|
||
logger.error(f"🚨 严重告警：活跃会话数 {_session_stats['active']} 超过泄漏阈值 {settings.database_session_leak_threshold}！")
|
||
elif _session_stats["active"] > settings.database_session_max_active:
|
||
logger.warning(f"⚠️ 警告：活跃会话数 {_session_stats['active']} 超过警告阈值 {settings.database_session_max_active}，可能存在连接泄漏！")
|
||
elif _session_stats["active"] < 0:
|
||
logger.error(f"🚨 活跃会话数异常: {_session_stats['active']}，统计可能不准确！")
|
||
|
||
except Exception as e:
|
||
_session_stats["errors"] += 1
|
||
logger.error(f"❌ 关闭会话时出错 [User:{user_id}][ID:{session_id}]: {str(e)}", exc_info=True)
|
||
try:
|
||
await session.close()
|
||
except Exception:
|
||
pass
|
||
|
||
async def init_db(user_id: str = None):
|
||
"""
|
||
初始化数据库（已弃用）
|
||
|
||
⚠️ 此函数已弃用，仅保留用于向后兼容
|
||
|
||
新的最佳实践:
|
||
- 表结构管理: 使用 'alembic upgrade head'
|
||
- 用户配置: Settings 在首次访问时自动创建（延迟初始化）
|
||
|
||
Args:
|
||
user_id: 用户ID (已不再使用)
|
||
"""
|
||
logger.warning(
|
||
"⚠️ init_db() 已弃用且无实际作用！\n"
|
||
" - 表结构: 由 Alembic 管理\n"
|
||
" - 用户配置: Settings API 自动创建\n"
|
||
" 建议移除此调用"
|
||
)
|
||
|
||
|
||
async def close_db():
|
||
"""关闭所有数据库连接"""
|
||
try:
|
||
logger.info("正在关闭所有数据库连接...")
|
||
for user_id, engine in _engine_cache.items():
|
||
await engine.dispose()
|
||
logger.info(f"用户 {user_id} 的数据库连接已关闭")
|
||
_engine_cache.clear()
|
||
logger.info("所有数据库连接已关闭")
|
||
except Exception as e:
|
||
logger.error(f"关闭数据库连接失败: {str(e)}", exc_info=True)
|
||
raise
|
||
|
||
async def get_database_stats():
|
||
"""获取数据库连接和会话统计信息
|
||
|
||
Returns:
|
||
dict: 包含数据库统计信息的字典
|
||
"""
|
||
from app.config import settings
|
||
|
||
# 获取连接池详细状态
|
||
pool_stats = {}
|
||
cache_key = "shared_postgres"
|
||
if cache_key in _engine_cache:
|
||
engine = _engine_cache[cache_key]
|
||
try:
|
||
pool = engine.pool
|
||
pool_stats = {
|
||
"size": pool.size(), # 当前连接池大小
|
||
"checked_in": pool.checkedin(), # 可用连接数
|
||
"checked_out": pool.checkedout(), # 正在使用的连接数
|
||
"overflow": pool.overflow(), # 溢出连接数
|
||
"usage_percent": (pool.checkedout() / (settings.database_pool_size + settings.database_max_overflow)) * 100,
|
||
}
|
||
except Exception as e:
|
||
logger.warning(f"获取连接池状态失败: {e}")
|
||
pool_stats = {"error": str(e)}
|
||
|
||
stats = {
|
||
"session_stats": {
|
||
"created": _session_stats["created"],
|
||
"closed": _session_stats["closed"],
|
||
"active": _session_stats["active"],
|
||
"errors": _session_stats["errors"],
|
||
"generator_exits": _session_stats["generator_exits"],
|
||
"last_check": _session_stats["last_check"],
|
||
},
|
||
"pool_stats": pool_stats, # 新增：连接池实时状态
|
||
"engine_cache": {
|
||
"total_engines": len(_engine_cache),
|
||
"engine_keys": list(_engine_cache.keys()),
|
||
},
|
||
"config": {
|
||
"database_type": "PostgreSQL",
|
||
"pool_size": settings.database_pool_size,
|
||
"max_overflow": settings.database_max_overflow,
|
||
"total_connections": settings.database_pool_size + settings.database_max_overflow,
|
||
"pool_timeout": settings.database_pool_timeout,
|
||
"pool_recycle": settings.database_pool_recycle,
|
||
"session_max_active_threshold": settings.database_session_max_active,
|
||
"session_leak_threshold": settings.database_session_leak_threshold,
|
||
},
|
||
"health": {
|
||
"status": "healthy",
|
||
"warnings": [],
|
||
"errors": [],
|
||
}
|
||
}
|
||
|
||
# 健康检查
|
||
if _session_stats["active"] > settings.database_session_leak_threshold:
|
||
stats["health"]["status"] = "critical"
|
||
stats["health"]["errors"].append(
|
||
f"活跃会话数 {_session_stats['active']} 超过泄漏阈值 {settings.database_session_leak_threshold}"
|
||
)
|
||
elif _session_stats["active"] > settings.database_session_max_active:
|
||
stats["health"]["status"] = "warning"
|
||
stats["health"]["warnings"].append(
|
||
f"活跃会话数 {_session_stats['active']} 超过警告阈值 {settings.database_session_max_active}"
|
||
)
|
||
|
||
if _session_stats["active"] < 0:
|
||
stats["health"]["status"] = "error"
|
||
stats["health"]["errors"].append(f"活跃会话数异常: {_session_stats['active']}")
|
||
|
||
# 连接池使用率检查
|
||
if pool_stats and "usage_percent" in pool_stats:
|
||
usage = pool_stats["usage_percent"]
|
||
if usage > 90:
|
||
stats["health"]["status"] = "warning"
|
||
stats["health"]["warnings"].append(f"连接池使用率过高: {usage:.1f}%")
|
||
elif usage > 95:
|
||
stats["health"]["status"] = "critical"
|
||
stats["health"]["errors"].append(f"连接池几乎耗尽: {usage:.1f}%")
|
||
|
||
error_rate = (_session_stats["errors"] / max(_session_stats["created"], 1)) * 100
|
||
if error_rate > 5:
|
||
if stats["health"]["status"] == "healthy":
|
||
stats["health"]["status"] = "warning"
|
||
stats["health"]["warnings"].append(f"会话错误率过高: {error_rate:.2f}%")
|
||
|
||
stats["health"]["error_rate"] = f"{error_rate:.2f}%"
|
||
|
||
return stats
|
||
|
||
|
||
async def check_database_health(user_id: str = None) -> dict:
|
||
"""检查数据库连接健康状态
|
||
|
||
Args:
|
||
user_id: 可选的用户ID，如果提供则检查特定用户的数据库
|
||
|
||
Returns:
|
||
dict: 健康检查结果
|
||
"""
|
||
result = {
|
||
"healthy": True,
|
||
"checks": {},
|
||
"timestamp": datetime.now().isoformat()
|
||
}
|
||
|
||
try:
|
||
# 检查引擎是否存在
|
||
cache_key = "shared_postgres"
|
||
if user_id:
|
||
engine = await get_engine(user_id)
|
||
else:
|
||
if cache_key not in _engine_cache:
|
||
result["checks"]["engine"] = {"status": "not_initialized", "healthy": True}
|
||
return result
|
||
engine = _engine_cache[cache_key]
|
||
|
||
# 测试数据库连接
|
||
AsyncSessionLocal = async_sessionmaker(
|
||
engine,
|
||
class_=AsyncSession,
|
||
expire_on_commit=False
|
||
)
|
||
|
||
async with AsyncSessionLocal() as session:
|
||
# 执行简单查询测试连接
|
||
await session.execute(text("SELECT 1"))
|
||
result["checks"]["connection"] = {"status": "ok", "healthy": True}
|
||
|
||
# 检查连接池状态（仅PostgreSQL）
|
||
if hasattr(engine.pool, 'size'):
|
||
pool_status = {
|
||
"size": engine.pool.size(),
|
||
"checked_in": engine.pool.checkedin(),
|
||
"checked_out": engine.pool.checkedout(),
|
||
"overflow": engine.pool.overflow(),
|
||
"healthy": True
|
||
}
|
||
|
||
# 连接池健康检查
|
||
if engine.pool.overflow() >= settings.database_max_overflow:
|
||
pool_status["healthy"] = False
|
||
pool_status["warning"] = "连接池溢出已满"
|
||
result["healthy"] = False
|
||
|
||
result["checks"]["pool"] = pool_status
|
||
|
||
except Exception as e:
|
||
result["healthy"] = False
|
||
result["checks"]["error"] = {
|
||
"status": "error",
|
||
"message": str(e),
|
||
"healthy": False
|
||
}
|
||
logger.error(f"数据库健康检查失败: {str(e)}", exc_info=True)
|
||
|
||
return result
|
||
|
||
|
||
async def reset_session_stats():
|
||
"""重置会话统计信息（用于测试或维护）"""
|
||
global _session_stats
|
||
_session_stats = {
|
||
"created": 0,
|
||
"closed": 0,
|
||
"active": 0,
|
||
"errors": 0,
|
||
"generator_exits": 0,
|
||
"last_check": datetime.now().isoformat()
|
||
}
|
||
logger.info("✅ 会话统计信息已重置")
|
||
return _session_stats |