From aaf4386f91c26fc41322befba1a3a8a15579362a Mon Sep 17 00:00:00 2001 From: xiamuceer Date: Tue, 4 Nov 2025 20:46:53 +0800 Subject: [PATCH] =?UTF-8?q?fix:1.=E4=BF=AE=E5=A4=8D=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E8=A2=AB=E8=A6=86=E7=9B=96=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 2 - .gitignore | 3 +- Dockerfile | 8 +-- backend/app/services/memory_service.py | 69 +++++++++++++++---- .../added_tokens.json | 0 .../chat_template.jinja | 0 .../vocab.txt | 0 .../refs/main | 0 .../1_Pooling/config.json | 0 .../README.md | 0 .../config.json | 0 .../config_sentence_transformers.json | 0 .../model.safetensors | 0 .../modules.json | 0 .../sentence_bert_config.json | 0 .../special_tokens_map.json | 0 .../tokenizer.json | 0 .../tokenizer_config.json | 0 18 files changed, 60 insertions(+), 22 deletions(-) rename backend/{data/models => embedding}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/added_tokens.json (100%) rename backend/{data/models => embedding}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/chat_template.jinja (100%) rename backend/{data/models => embedding}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/vocab.txt (100%) rename backend/{data/models => embedding}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/refs/main (100%) rename backend/{data/models => embedding}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/1_Pooling/config.json (100%) rename backend/{data/models => embedding}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/README.md (100%) rename backend/{data/models => embedding}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/config.json (100%) rename backend/{data/models => embedding}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/config_sentence_transformers.json (100%) rename backend/{data/models => embedding}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/model.safetensors (100%) rename backend/{data/models => embedding}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/modules.json (100%) rename backend/{data/models => embedding}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/sentence_bert_config.json (100%) rename backend/{data/models => embedding}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/special_tokens_map.json (100%) rename backend/{data/models => embedding}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/tokenizer.json (100%) rename backend/{data/models => embedding}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/tokenizer_config.json (100%) diff --git a/.dockerignore b/.dockerignore index e7c354e..ef6162d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -43,8 +43,6 @@ backend/data/*.db # ChromaDB数据库（不包含在镜像中，会在运行时生成） backend/data/chroma_db/ -# 注意：backend/data/models/ 目录需要打包（包含预下载的Embedding模型） -# 所以不要添加到 .dockerignore 中 # 日志文件 logs/ diff --git a/.gitignore b/.gitignore index 7247932..a7ce302 100644 --- a/.gitignore +++ b/.gitignore @@ -102,8 +102,7 @@ dmypy.json # Jupyter Notebook .ipynb_checkpoints -data/* -!/data/models/ +data/ docs/ data_old/ backend/migrate_all_databases.py diff --git a/Dockerfile b/Dockerfile index 23877c5..bec63d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,11 +53,11 @@ COPY backend/ ./ COPY --from=frontend-builder /frontend/dist ./static # 创建必要的目录 -RUN mkdir -p /app/data /app/logs +RUN mkdir -p /app/data /app/logs /app/embedding -# 复制预下载的Embedding模型（如果存在） +# 复制预下载的Embedding模型到独立目录（避免被docker-compose的data挂载覆盖） # 这样可以避免首次运行时联网下载约420MB的模型文件 -COPY backend/data/models /app/data/models +COPY backend/data/models /app/embedding # 复制环境变量示例文件 COPY backend/.env.example ./.env.example @@ -74,7 +74,7 @@ ENV APP_PORT=8000 ENV TRANSFORMERS_OFFLINE=1 ENV HF_DATASETS_OFFLINE=1 ENV HF_HUB_OFFLINE=1 -ENV SENTENCE_TRANSFORMERS_HOME=/app/data/models +ENV SENTENCE_TRANSFORMERS_HOME=/app/embedding # 健康检查 HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ diff --git a/backend/app/services/memory_service.py b/backend/app/services/memory_service.py index 9700827..71e4186 100644 --- a/backend/app/services/memory_service.py +++ b/backend/app/services/memory_service.py @@ -10,11 +10,10 @@ import hashlib logger = get_logger(__name__) -# 配置离线模式，避免联网检查 -os.environ['TRANSFORMERS_OFFLINE'] = '1' -os.environ['HF_DATASETS_OFFLINE'] = '1' -os.environ['HF_HUB_OFFLINE'] = '1' # 禁用HuggingFace Hub联网 -os.environ['SENTENCE_TRANSFORMERS_HOME'] = 'data/models' # 设置模型缓存目录 +# 配置模型缓存目录（不设置离线模式，让它自动选择） +# 如果本地有模型就用本地的，没有才联网下载 +if 'SENTENCE_TRANSFORMERS_HOME' not in os.environ: + os.environ['SENTENCE_TRANSFORMERS_HOME'] = 'embedding' class MemoryService: @@ -46,33 +45,75 @@ class MemoryService: logger.info("🔄 正在加载Embedding模型...") # 确保模型缓存目录存在 - model_cache_dir = 'data/models' + model_cache_dir = 'embedding' os.makedirs(model_cache_dir, exist_ok=True) + # 调试信息：打印环境变量和路径 + logger.info(f"📂 当前工作目录: {os.getcwd()}") + logger.info(f"📂 模型缓存目录: {os.path.abspath(model_cache_dir)}") + logger.info(f"🔧 SENTENCE_TRANSFORMERS_HOME: {os.environ.get('SENTENCE_TRANSFORMERS_HOME', '未设置')}") + logger.info(f"🔧 TRANSFORMERS_OFFLINE: {os.environ.get('TRANSFORMERS_OFFLINE', '未设置')}") + logger.info(f"🔧 HF_HUB_OFFLINE: {os.environ.get('HF_HUB_OFFLINE', '未设置')}") + + # 检查模型目录内容 + if os.path.exists(model_cache_dir): + logger.info(f"📁 模型目录存在，检查内容...") + try: + items = os.listdir(model_cache_dir) + logger.info(f"📁 模型目录内容: {items}") + + # 检查是否有预期的模型文件夹 + expected_model_dir = os.path.join(model_cache_dir, 'models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2') + if os.path.exists(expected_model_dir): + logger.info(f"✅ 找到本地模型目录: {expected_model_dir}") + # 检查快照目录 + snapshots_dir = os.path.join(expected_model_dir, 'snapshots') + if os.path.exists(snapshots_dir): + snapshots = os.listdir(snapshots_dir) + logger.info(f"📁 模型快照: {snapshots}") + else: + logger.warning(f"⚠️ 未找到本地模型目录: {expected_model_dir}") + except Exception as e: + logger.error(f"❌ 检查模型目录失败: {str(e)}") + else: + logger.warning(f"⚠️ 模型目录不存在: {os.path.abspath(model_cache_dir)}") + try: + logger.info("🔄 尝试加载主模型: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2") # 优先使用本地缓存的模型 # cache_folder会让模型优先从本地加载，只有不存在时才联网下载 + # 注意：不要设置local_files_only=True，这会阻止fallback到联网下载 self.embedding_model = SentenceTransformer( - 'paraphrase-multilingual-MiniLM-L12-v2', + 'sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2', cache_folder=model_cache_dir, - device='cpu' # 明确指定使用CPU + device='cpu', # 明确指定使用CPU + trust_remote_code=False # 安全起见 ) - logger.info("✅ Embedding模型加载成功") + logger.info("✅ Embedding模型加载成功 (paraphrase-multilingual-MiniLM-L12-v2)") except Exception as e: logger.warning(f"⚠️ 无法加载多语言模型: {str(e)}") - logger.info("🔄 尝试使用备用模型...") + logger.error(f"❌ 详细错误: {repr(e)}") + import traceback + logger.error(f"❌ 错误堆栈:\n{traceback.format_exc()}") + logger.info("🔄 尝试使用备用模型: sentence-transformers/all-MiniLM-L6-v2") try: # 降级到更小的模型作为备选 self.embedding_model = SentenceTransformer( - 'all-MiniLM-L6-v2', + 'sentence-transformers/all-MiniLM-L6-v2', cache_folder=model_cache_dir, - device='cpu' + device='cpu', + trust_remote_code=False ) - logger.info("✅ 使用备用Embedding模型") + logger.info("✅ 使用备用Embedding模型 (all-MiniLM-L6-v2)") except Exception as e2: logger.error(f"❌ 所有模型加载失败: {str(e2)}") + logger.error(f"❌ 详细错误: {repr(e2)}") + import traceback + logger.error(f"❌ 错误堆栈:\n{traceback.format_exc()}") logger.error("💡 模型首次使用需要联网下载（约420MB）") - logger.error(" 或手动下载模型文件到 data/models 目录") + logger.error(" 或手动下载模型文件到 embedding 目录") + logger.error(f"💡 期望的模型目录结构:") + logger.error(f" {os.path.abspath(model_cache_dir)}/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/") raise RuntimeError("无法加载任何Embedding模型") self._initialized = True diff --git a/backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/added_tokens.json b/backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/added_tokens.json similarity index 100% rename from backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/added_tokens.json rename to backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/added_tokens.json diff --git a/backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/chat_template.jinja b/backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/chat_template.jinja similarity index 100% rename from backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/chat_template.jinja rename to backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/chat_template.jinja diff --git a/backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/vocab.txt b/backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/vocab.txt similarity index 100% rename from backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/vocab.txt rename to backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/.no_exist/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/vocab.txt diff --git a/backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/refs/main b/backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/refs/main similarity index 100% rename from backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/refs/main rename to backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/refs/main diff --git a/backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/1_Pooling/config.json b/backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/1_Pooling/config.json similarity index 100% rename from backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/1_Pooling/config.json rename to backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/1_Pooling/config.json diff --git a/backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/README.md b/backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/README.md similarity index 100% rename from backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/README.md rename to backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/README.md diff --git a/backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/config.json b/backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/config.json similarity index 100% rename from backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/config.json rename to backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/config.json diff --git a/backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/config_sentence_transformers.json b/backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/config_sentence_transformers.json similarity index 100% rename from backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/config_sentence_transformers.json rename to backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/config_sentence_transformers.json diff --git a/backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/model.safetensors b/backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/model.safetensors similarity index 100% rename from backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/model.safetensors rename to backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/model.safetensors diff --git a/backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/modules.json b/backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/modules.json similarity index 100% rename from backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/modules.json rename to backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/modules.json diff --git a/backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/sentence_bert_config.json b/backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/sentence_bert_config.json similarity index 100% rename from backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/sentence_bert_config.json rename to backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/sentence_bert_config.json diff --git a/backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/special_tokens_map.json b/backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/special_tokens_map.json similarity index 100% rename from backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/special_tokens_map.json rename to backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/special_tokens_map.json diff --git a/backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/tokenizer.json b/backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/tokenizer.json similarity index 100% rename from backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/tokenizer.json rename to backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/tokenizer.json diff --git a/backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/tokenizer_config.json b/backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/tokenizer_config.json similarity index 100% rename from backend/data/models/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/tokenizer_config.json rename to backend/embedding/models--sentence-transformers--paraphrase-multilingual-MiniLM-L12-v2/snapshots/86741b4e3f5cb7765a600d3a3d55a0f6a6cb443d/tokenizer_config.json