update:1.更新mcp插件功能,目前只支持remote调用

This commit is contained in:
xiamuceer
2025-11-07 22:14:20 +08:00
parent 1e998920e3
commit 88115a45c5
26 changed files with 4088 additions and 138 deletions
+4
View File
@@ -0,0 +1,4 @@
"""MCP插件系统"""
from .registry import mcp_registry
__all__ = ["mcp_registry"]
+345
View File
@@ -0,0 +1,345 @@
"""HTTP MCP客户端 - 实现JSON-RPC 2.0协议"""
import httpx
from typing import Dict, Any, List, Optional
from app.logger import get_logger
import time
logger = get_logger(__name__)
class MCPError(Exception):
"""MCP错误"""
pass
class HTTPMCPClient:
"""HTTP模式MCP客户端(类似Cursor/Claude Code实现)"""
def __init__(
self,
url: str,
headers: Optional[Dict[str, str]] = None,
env: Optional[Dict[str, str]] = None,
timeout: float = 60.0,
http_client: Optional[httpx.AsyncClient] = None
):
"""
初始化HTTP MCP客户端
Args:
url: MCP服务器URL
headers: HTTP请求头
env: 环境变量(用于API Key等)
timeout: 超时时间(秒)
http_client: 可选的共享HTTP客户端(用于连接池复用)
"""
self.url = url.rstrip('/')
self.headers = headers or {}
self.env = env or {}
self.timeout = timeout
# 设置MCP必需的Accept头
# MCP服务器要求客户端必须接受 application/json 和 text/event-stream
if 'Accept' not in self.headers:
self.headers['Accept'] = 'application/json, text/event-stream'
# 设置Content-Type
if 'Content-Type' not in self.headers:
self.headers['Content-Type'] = 'application/json'
# 如果env中有API Key,添加到headers
if 'API_KEY' in self.env:
self.headers['Authorization'] = f'Bearer {self.env["API_KEY"]}'
# 使用共享客户端或创建新客户端
self._owns_client = http_client is None
if http_client:
self.client = http_client
else:
self.client = httpx.AsyncClient(
timeout=httpx.Timeout(timeout),
headers=self.headers
)
self._request_id = 0
def _next_request_id(self) -> int:
"""获取下一个请求ID"""
self._request_id += 1
return self._request_id
async def _call_jsonrpc(
self,
method: str,
params: Optional[Dict[str, Any]] = None
) -> Dict[str, Any]:
"""
调用JSON-RPC 2.0方法
Args:
method: 方法名
params: 参数
Returns:
响应结果
Raises:
MCPError: 调用失败时抛出
"""
request_id = self._next_request_id()
payload = {
"jsonrpc": "2.0",
"id": request_id,
"method": method,
"params": params or {}
}
try:
logger.debug(f"MCP请求: {method} -> {self.url}")
response = await self.client.post(
self.url,
json=payload,
headers=self.headers # 显式传递headers(对于共享客户端很重要)
)
response.raise_for_status()
# 获取响应内容
response_text = response.text
content_type = response.headers.get('content-type', '')
# 如果是空响应
if not response_text or response_text.strip() == '':
raise MCPError("服务器返回空响应")
# 处理SSE格式响应
if 'text/event-stream' in content_type or response_text.startswith('event:'):
logger.debug("检测到SSE格式响应,开始解析")
data = self._parse_sse_response(response_text)
else:
# 标准JSON响应
try:
data = response.json()
except ValueError as e:
logger.error(f"JSON解析失败,响应内容: {response_text[:500]}")
raise MCPError(f"无法解析JSON响应: {str(e)}")
# 检查JSON-RPC错误
if "error" in data:
error = data["error"]
error_msg = error.get("message", "Unknown error")
error_code = error.get("code", -1)
logger.error(f"MCP错误 [{error_code}]: {error_msg}")
raise MCPError(f"[{error_code}] {error_msg}")
if "result" not in data:
raise MCPError("响应中缺少result字段")
return data["result"]
except httpx.HTTPStatusError as e:
logger.error(f"HTTP错误 {e.response.status_code}: {e.response.text}")
raise MCPError(f"HTTP错误 {e.response.status_code}: {e.response.text}")
except httpx.RequestError as e:
logger.error(f"请求错误: {str(e)}")
raise MCPError(f"请求错误: {str(e)}")
except MCPError:
raise
except Exception as e:
logger.error(f"未知错误: {str(e)}")
raise MCPError(f"未知错误: {str(e)}")
def _parse_sse_response(self, sse_text: str) -> Dict[str, Any]:
"""
解析SSE格式的响应
SSE格式示例:
event: message
data: {"result": {...}}
Args:
sse_text: SSE格式的文本
Returns:
解析后的JSON数据
"""
import json
lines = sse_text.strip().split('\n')
data_lines = []
for line in lines:
line = line.strip()
if line.startswith('data:'):
# 提取data后面的内容
data_content = line[5:].strip()
data_lines.append(data_content)
if not data_lines:
raise MCPError("SSE响应中没有找到data字段")
# 合并所有data行(某些SSE可能分多行)
full_data = ''.join(data_lines)
try:
return json.loads(full_data)
except json.JSONDecodeError as e:
logger.error(f"解析SSE data失败: {full_data[:200]}")
raise MCPError(f"SSE data不是有效的JSON: {str(e)}")
async def list_tools(self) -> List[Dict[str, Any]]:
"""
列举可用工具
Returns:
工具列表
"""
try:
result = await self._call_jsonrpc("tools/list")
tools = result.get("tools", [])
logger.info(f"获取到 {len(tools)} 个工具")
return tools
except Exception as e:
logger.error(f"获取工具列表失败: {e}")
raise
async def call_tool(
self,
tool_name: str,
arguments: Dict[str, Any]
) -> Any:
"""
调用工具
Args:
tool_name: 工具名称
arguments: 工具参数
Returns:
工具执行结果
"""
try:
logger.info(f"调用工具: {tool_name}")
logger.debug(f"参数: {arguments}")
result = await self._call_jsonrpc(
"tools/call",
{
"name": tool_name,
"arguments": arguments
}
)
# MCP返回的result通常包含content数组
if isinstance(result, dict) and "content" in result:
content = result["content"]
if isinstance(content, list) and len(content) > 0:
# 提取第一个content项的text
first_content = content[0]
if isinstance(first_content, dict) and "text" in first_content:
return first_content["text"]
return first_content
return content
return result
except Exception as e:
logger.error(f"调用工具失败: {tool_name}, 错误: {e}")
raise
async def list_resources(self) -> List[Dict[str, Any]]:
"""
列举可用资源
Returns:
资源列表
"""
try:
result = await self._call_jsonrpc("resources/list")
resources = result.get("resources", [])
logger.info(f"获取到 {len(resources)} 个资源")
return resources
except Exception as e:
logger.error(f"获取资源列表失败: {e}")
raise
async def read_resource(self, uri: str) -> Any:
"""
读取资源
Args:
uri: 资源URI
Returns:
资源内容
"""
try:
result = await self._call_jsonrpc(
"resources/read",
{"uri": uri}
)
return result
except Exception as e:
logger.error(f"读取资源失败: {uri}, 错误: {e}")
raise
async def test_connection(self) -> Dict[str, Any]:
"""
测试连接
Returns:
测试结果
"""
start_time = time.time()
try:
# 尝试列举工具来测试连接
tools = await self.list_tools()
end_time = time.time()
response_time = round((end_time - start_time) * 1000, 2)
return {
"success": True,
"message": "连接测试成功",
"response_time_ms": response_time,
"tools_count": len(tools),
"tools": tools
}
except MCPError as e:
end_time = time.time()
response_time = round((end_time - start_time) * 1000, 2)
return {
"success": False,
"message": "连接测试失败",
"response_time_ms": response_time,
"error": str(e),
"error_type": "MCPError",
"suggestions": [
"请检查服务器URL是否正确",
"请确认API Key是否有效",
"请检查网络连接"
]
}
except Exception as e:
end_time = time.time()
response_time = round((end_time - start_time) * 1000, 2)
return {
"success": False,
"message": "连接测试失败",
"response_time_ms": response_time,
"error": str(e),
"error_type": type(e).__name__,
"suggestions": [
"请检查服务器是否在线",
"请确认配置是否正确"
]
}
async def close(self):
"""关闭客户端(仅在拥有客户端所有权时关闭)"""
if self._owns_client and self.client:
await self.client.aclose()
+349
View File
@@ -0,0 +1,349 @@
"""MCP插件注册表 - 管理运行时插件实例"""
import asyncio
import time
import httpx
from typing import Dict, Optional, Any, List, Tuple
from collections import OrderedDict
from app.mcp.http_client import HTTPMCPClient, MCPError
from app.models.mcp_plugin import MCPPlugin
from app.logger import get_logger
logger = get_logger(__name__)
class MCPPluginRegistry:
"""MCP插件注册表 - 管理运行时插件实例(多用户优化版)"""
def __init__(self, max_clients: int = 1000, client_ttl: int = 3600):
"""
初始化注册表
Args:
max_clients: 最大缓存客户端数量
client_ttl: 客户端过期时间(秒),默认1小时
"""
# 存储格式: {plugin_id: (client, last_access_time)}
self._clients: OrderedDict[str, Tuple[HTTPMCPClient, float]] = OrderedDict()
# 细粒度锁:每个用户一个锁
self._user_locks: Dict[str, asyncio.Lock] = {}
self._locks_lock = asyncio.Lock() # 保护locks字典本身
# 配置参数
self._max_clients = max_clients
self._client_ttl = client_ttl
# 共享HTTP客户端池(用于所有MCP HTTP请求)
self._shared_http_client = httpx.AsyncClient(
limits=httpx.Limits(
max_keepalive_connections=100,
max_connections=200,
keepalive_expiry=30.0
),
timeout=httpx.Timeout(connect=10.0, read=60.0, write=10.0, pool=5.0),
headers={
"User-Agent": "MuMuAINovel-MCP-Client/1.0"
}
)
# 启动后台清理任务
self._cleanup_task = None
self._start_cleanup_task()
def _start_cleanup_task(self):
"""启动后台清理任务"""
if self._cleanup_task is None:
self._cleanup_task = asyncio.create_task(self._cleanup_loop())
logger.info("✅ MCP插件注册表后台清理任务已启动")
async def _cleanup_loop(self):
"""后台清理过期客户端"""
while True:
try:
await asyncio.sleep(300) # 每5分钟清理一次
await self._cleanup_expired_clients()
except asyncio.CancelledError:
break
except Exception as e:
logger.error(f"清理任务异常: {e}")
async def _cleanup_expired_clients(self):
"""清理过期的客户端"""
now = time.time()
expired_ids = []
# 收集过期的plugin_id
for plugin_id, (client, last_access) in list(self._clients.items()):
if now - last_access > self._client_ttl:
expired_ids.append(plugin_id)
if expired_ids:
logger.info(f"🧹 清理 {len(expired_ids)} 个过期的MCP客户端")
for plugin_id in expired_ids:
# 提取user_id来获取对应的锁
user_id = plugin_id.split(':', 1)[0]
user_lock = await self._get_user_lock(user_id)
async with user_lock:
if plugin_id in self._clients:
await self._unload_plugin_unsafe(plugin_id)
async def _get_user_lock(self, user_id: str) -> asyncio.Lock:
"""
获取用户专属的锁(细粒度锁)
Args:
user_id: 用户ID
Returns:
该用户的锁对象
"""
async with self._locks_lock:
if user_id not in self._user_locks:
self._user_locks[user_id] = asyncio.Lock()
return self._user_locks[user_id]
def _touch_client(self, plugin_id: str):
"""
更新客户端的最后访问时间(LRU)
Args:
plugin_id: 插件ID
"""
if plugin_id in self._clients:
client, _ = self._clients[plugin_id]
self._clients[plugin_id] = (client, time.time())
# 移到末尾(LRU
self._clients.move_to_end(plugin_id)
async def _evict_lru_client(self):
"""驱逐最久未使用的客户端(当达到max_clients限制时)"""
if len(self._clients) >= self._max_clients:
# 获取最旧的plugin_id
oldest_id = next(iter(self._clients))
logger.info(f"📤 达到最大客户端数量限制,驱逐: {oldest_id}")
await self._unload_plugin_unsafe(oldest_id)
async def load_plugin(self, plugin: MCPPlugin) -> bool:
"""
从配置加载插件
Args:
plugin: 插件配置
Returns:
是否加载成功
"""
# 使用细粒度锁(只锁定当前用户)
user_lock = await self._get_user_lock(plugin.user_id)
async with user_lock:
try:
plugin_id = f"{plugin.user_id}:{plugin.plugin_name}"
# 如果已加载,先卸载
if plugin_id in self._clients:
await self._unload_plugin_unsafe(plugin_id)
# 检查是否需要驱逐LRU客户端
await self._evict_lru_client()
# 目前只支持HTTP类型
if plugin.plugin_type == "http":
if not plugin.server_url:
logger.error(f"HTTP插件缺少server_url: {plugin.plugin_name}")
return False
# 使用共享HTTP连接池创建客户端
client = HTTPMCPClient(
url=plugin.server_url,
headers=plugin.headers or {},
env=plugin.env or {},
timeout=plugin.config.get('timeout', 60.0) if plugin.config else 60.0,
http_client=self._shared_http_client # 传入共享连接池
)
# 存储客户端和当前时间戳
self._clients[plugin_id] = (client, time.time())
logger.info(f"✅ 加载MCP插件: {plugin_id}")
return True
else:
logger.warning(f"暂不支持的插件类型: {plugin.plugin_type}")
return False
except Exception as e:
logger.error(f"加载插件失败 {plugin.plugin_name}: {e}")
return False
async def unload_plugin(self, user_id: str, plugin_name: str):
"""
卸载插件
Args:
user_id: 用户ID
plugin_name: 插件名称
"""
# 使用细粒度锁(只锁定当前用户)
user_lock = await self._get_user_lock(user_id)
async with user_lock:
plugin_id = f"{user_id}:{plugin_name}"
await self._unload_plugin_unsafe(plugin_id)
async def _unload_plugin_unsafe(self, plugin_id: str):
"""卸载插件(不加锁,内部使用)"""
if plugin_id in self._clients:
client, _ = self._clients[plugin_id] # 解包 (client, timestamp)
try:
await client.close()
except Exception as e:
logger.error(f"关闭插件客户端失败 {plugin_id}: {e}")
del self._clients[plugin_id]
logger.info(f"卸载MCP插件: {plugin_id}")
async def reload_plugin(self, plugin: MCPPlugin) -> bool:
"""
重新加载插件
Args:
plugin: 插件配置
Returns:
是否重载成功
"""
await self.unload_plugin(plugin.user_id, plugin.plugin_name)
return await self.load_plugin(plugin)
def get_client(self, user_id: str, plugin_name: str) -> Optional[HTTPMCPClient]:
"""
获取插件客户端(支持LRU访问时间更新)
Args:
user_id: 用户ID
plugin_name: 插件名称
Returns:
客户端实例或None
"""
plugin_id = f"{user_id}:{plugin_name}"
entry = self._clients.get(plugin_id)
if entry:
# 更新访问时间(LRU
self._touch_client(plugin_id)
return entry[0] # 返回客户端对象
return None
async def call_tool(
self,
user_id: str,
plugin_name: str,
tool_name: str,
arguments: Dict[str, Any]
) -> Any:
"""
调用插件工具
Args:
user_id: 用户ID
plugin_name: 插件名称
tool_name: 工具名称
arguments: 工具参数
Returns:
工具执行结果
Raises:
ValueError: 插件不存在或未启用
MCPError: 工具调用失败
"""
client = self.get_client(user_id, plugin_name)
if not client:
raise ValueError(f"插件未加载: {plugin_name}")
try:
result = await client.call_tool(tool_name, arguments)
logger.info(f"✅ 工具调用成功: {plugin_name}.{tool_name}")
# logger.info(f"✅ 工具返回内容: {result}")
return result
except Exception as e:
logger.error(f"❌ 工具调用失败: {plugin_name}.{tool_name}, 错误: {e}")
raise
async def get_plugin_tools(
self,
user_id: str,
plugin_name: str
) -> List[Dict[str, Any]]:
"""
获取插件的工具列表
Args:
user_id: 用户ID
plugin_name: 插件名称
Returns:
工具列表
"""
client = self.get_client(user_id, plugin_name)
if not client:
raise ValueError(f"插件未加载: {plugin_name}")
try:
tools = await client.list_tools()
return tools
except Exception as e:
logger.error(f"获取工具列表失败: {plugin_name}, 错误: {e}")
raise
async def test_plugin(
self,
user_id: str,
plugin_name: str
) -> Dict[str, Any]:
"""
测试插件连接
Args:
user_id: 用户ID
plugin_name: 插件名称
Returns:
测试结果
"""
client = self.get_client(user_id, plugin_name)
if not client:
raise ValueError(f"插件未加载: {plugin_name}")
return await client.test_connection()
async def cleanup_all(self):
"""清理所有插件和资源"""
# 停止后台清理任务
if self._cleanup_task:
self._cleanup_task.cancel()
try:
await self._cleanup_task
except asyncio.CancelledError:
pass
# 清理所有客户端
plugin_ids = list(self._clients.keys())
for plugin_id in plugin_ids:
user_id = plugin_id.split(':', 1)[0]
user_lock = await self._get_user_lock(user_id)
async with user_lock:
await self._unload_plugin_unsafe(plugin_id)
# 关闭共享HTTP客户端
try:
await self._shared_http_client.aclose()
except Exception as e:
logger.error(f"关闭共享HTTP客户端失败: {e}")
logger.info("✅ 已清理所有MCP插件和资源")
# 全局注册表实例
mcp_registry = MCPPluginRegistry()