package chatpipeline import ( "context" "encoding/json" "fmt" "regexp" "strings" "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" ) // PluginQueryUnderstand performs query rewriting and intent classification. // It uses conversation history and an LLM to optimise the user's original query // and determine the downstream pipeline behaviour. type PluginQueryUnderstand struct { modelService interfaces.ModelService messageService interfaces.MessageService memoryService interfaces.MemoryService config *config.Config } var rewriteImageSepPattern = regexp.MustCompile(`(?s)^(.*?)\s*\n?---\n(.*)$`) type queryUnderstandOutput struct { RewriteQuery string `json:"rewrite_query"` Intent types.QueryIntent `json:"intent"` ImageDescription string `json:"image_description"` } // NewPluginQueryUnderstand creates a new query-understanding plugin instance // and registers it with the event manager. func NewPluginQueryUnderstand(eventManager *EventManager, modelService interfaces.ModelService, messageService interfaces.MessageService, memoryService interfaces.MemoryService, config *config.Config, ) *PluginQueryUnderstand { res := &PluginQueryUnderstand{ modelService: modelService, messageService: messageService, memoryService: memoryService, config: config, } eventManager.Register(res) return res } // ActivationEvents returns the list of event types this plugin responds to. func (p *PluginQueryUnderstand) ActivationEvents() []types.EventType { return []types.EventType{types.QUERY_UNDERSTAND} } // OnEvent processes triggered events. // Handles three input combinations: // - Text only: standard rewrite + intent classification (uses chat model) // - Text + images: multimodal rewrite + intent + image description (uses VLM/vision model) // - Images only: multimodal analysis + intent + image description (uses VLM/vision model) func (p *PluginQueryUnderstand) OnEvent(ctx context.Context, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, ) *PluginError { chatManage.RewriteQuery = chatManage.Query hasImages := len(chatManage.Images) > 0 needRewrite := chatManage.EnableRewrite if !needRewrite && !hasImages { pipelineInfo(ctx, "QueryUnderstand", "skip", map[string]interface{}{ "session_id": chatManage.SessionID, "reason": "rewrite_disabled_no_images", }) return next() } pipelineInfo(ctx, "QueryUnderstand", "input", map[string]interface{}{ "session_id": chatManage.SessionID, "tenant_id": chatManage.TenantID, "user_query": chatManage.Query, "has_images": hasImages, "enable_rewrite": chatManage.EnableRewrite, }) // --- Load and prepare conversation history --- var historyList []*types.History if len(chatManage.History) > 0 { historyList = chatManage.History pipelineInfo(ctx, "QueryUnderstand", "history_reused", map[string]interface{}{ "session_id": chatManage.SessionID, "rounds": len(historyList), }) } else { historyList = p.loadHistory(ctx, chatManage) } // --- Select the appropriate model --- rewriteModel, useImages := p.selectModel(ctx, chatManage, hasImages) if rewriteModel == nil { pipelineError(ctx, "QueryUnderstand", "get_model", map[string]interface{}{ "session_id": chatManage.SessionID, }) return next() } // --- Build prompts --- systemContent, userContent := p.buildPrompts(ctx, chatManage, historyList) userMsg := chat.Message{Role: "user", Content: userContent} if useImages { userMsg.Images = chatManage.Images } maxTokens := 150 if useImages { maxTokens = 500 } // --- Call model --- thinking := false modelCtx := types.WithLLMCallMetadata(ctx, "query_rewrite", "") response, err := rewriteModel.Chat(modelCtx, []chat.Message{ {Role: "system", Content: systemContent}, userMsg, }, &chat.ChatOptions{ Temperature: 0.3, MaxCompletionTokens: maxTokens, Thinking: &thinking, }) if err != nil { pipelineError(ctx, "QueryUnderstand", "model_call", map[string]interface{}{ "session_id": chatManage.SessionID, "error": err.Error(), }) return next() } // --- Parse structured output --- p.parseOutput(chatManage, response.Content) // Persist image description asynchronously — this DB write does not affect // the current pipeline result, so it can run in the background. if chatManage.ImageDescription != "" && chatManage.UserMessageID != "" { go p.updateUserMessageImageCaption(context.WithoutCancel(ctx), chatManage) } // --- Apply intent-specific system prompt override --- if !chatManage.NeedsRetrieval() { if applyIntentPromptOverride(chatManage, p.config.Conversation.IntentSystemPrompts) { pipelineInfo(ctx, "QueryUnderstand", "prompt_override", map[string]interface{}{ "session_id": chatManage.SessionID, "intent": chatManage.Intent, }) } } pipelineInfo(ctx, "QueryUnderstand", "output", map[string]interface{}{ "session_id": chatManage.SessionID, "rewrite_query": chatManage.RewriteQuery, "intent": chatManage.Intent, "has_image_desc": chatManage.ImageDescription != "", "has_prompt_override": chatManage.SystemPromptOverride != "", "original_output": response.Content, }) return next() } // updateUserMessageImageCaption writes the generated ImageDescription back to // the stored user message so that subsequent turns can see it in history. func (p *PluginQueryUnderstand) updateUserMessageImageCaption(ctx context.Context, chatManage *types.ChatManage) { msg, err := p.messageService.GetMessage(ctx, chatManage.SessionID, chatManage.UserMessageID) if err != nil { pipelineWarn(ctx, "QueryUnderstand", "get_user_message", map[string]interface{}{ "session_id": chatManage.SessionID, "user_message_id": chatManage.UserMessageID, "error": err.Error(), }) return } if len(msg.Images) == 0 { return } msg.Images[0].Caption = chatManage.ImageDescription if err := p.messageService.UpdateMessageImages(ctx, chatManage.SessionID, chatManage.UserMessageID, msg.Images); err != nil { pipelineWarn(ctx, "QueryUnderstand", "update_image_caption", map[string]interface{}{ "session_id": chatManage.SessionID, "user_message_id": chatManage.UserMessageID, "error": err.Error(), }) } } // loadHistory fetches and processes conversation history for rewrite context. func (p *PluginQueryUnderstand) loadHistory(ctx context.Context, chatManage *types.ChatManage) []*types.History { // Honor the multi-turn-disabled signal: chatManage.MaxRounds == 0 is set // explicitly by applyAgentOverridesToChatManage when the custom agent has // MultiTurnEnabled=false. We must not silently fall back to the global // default, otherwise rewrite + image analysis would still pull old turns // into the context and leak through chatManage.History. if chatManage.MaxRounds <= 0 { return nil } maxRounds := chatManage.MaxRounds historyList, err := loadAndProcessHistory(ctx, p.messageService, chatManage.SessionID, maxRounds, 20) if err != nil { pipelineWarn(ctx, "QueryUnderstand", "history_fetch", map[string]interface{}{ "session_id": chatManage.SessionID, "error": err.Error(), }) return nil } chatManage.History = historyList if len(historyList) > 0 { pipelineInfo(ctx, "QueryUnderstand", "history_ready", map[string]interface{}{ "session_id": chatManage.SessionID, "history_rounds": len(historyList), }) } return historyList } // selectModel picks the model for query understanding. When images are present // it prefers a vision-capable model. Returns (model, useImages). func (p *PluginQueryUnderstand) selectModel(ctx context.Context, chatManage *types.ChatManage, hasImages bool) (chat.Chat, bool) { if hasImages { if chatManage.ChatModelSupportsVision { m, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID) if err == nil { return m, true } pipelineWarn(ctx, "QueryUnderstand", "vision_model_fallback", map[string]interface{}{ "session_id": chatManage.SessionID, "error": err.Error(), }) } if chatManage.VLMModelID != "" { m, err := p.modelService.GetChatModel(ctx, chatManage.VLMModelID) if err == nil { return m, true } pipelineWarn(ctx, "QueryUnderstand", "vlm_model_fallback", map[string]interface{}{ "session_id": chatManage.SessionID, "vlm_model_id": chatManage.VLMModelID, "error": err.Error(), }) } pipelineWarn(ctx, "QueryUnderstand", "no_vision_model", map[string]interface{}{ "session_id": chatManage.SessionID, }) } textModelID := chatManage.ChatModelID if chatManage.QueryUnderstandModelID != "" { textModelID = chatManage.QueryUnderstandModelID } m, err := p.modelService.GetChatModel(ctx, textModelID) if err != nil { // Fall back to ChatModelID when a dedicated query-understand model was // configured but cannot be resolved (e.g. deleted / disabled). if chatManage.QueryUnderstandModelID != "" && textModelID != chatManage.ChatModelID { pipelineWarn(ctx, "QueryUnderstand", "query_understand_model_fallback", map[string]interface{}{ "session_id": chatManage.SessionID, "query_understand_model_id": chatManage.QueryUnderstandModelID, "error": err.Error(), }) if fallback, fbErr := p.modelService.GetChatModel(ctx, chatManage.ChatModelID); fbErr == nil { return fallback, false } } pipelineError(ctx, "QueryUnderstand", "get_model", map[string]interface{}{ "session_id": chatManage.SessionID, "chat_model_id": textModelID, "error": err.Error(), }) return nil, false } return m, false } // buildPrompts constructs system and user prompts with placeholder replacement. func (p *PluginQueryUnderstand) buildPrompts( ctx context.Context, chatManage *types.ChatManage, historyList []*types.History, ) (string, string) { userPrompt := p.config.Conversation.RewritePromptUser if chatManage.RewritePromptUser != "" { userPrompt = chatManage.RewritePromptUser } systemPrompt := p.config.Conversation.RewritePromptSystem if chatManage.RewritePromptSystem != "" { systemPrompt = chatManage.RewritePromptSystem } conversationText := formatConversationHistory(historyList) queryContent := chatManage.Query if len(chatManage.Images) > 0 { queryContent += fmt.Sprintf("\n\n", len(chatManage.Images)) } else { queryContent += "\n\n" } if len(chatManage.Attachments) > 0 { queryContent += chatManage.Attachments.BuildPrompt() } else { queryContent += "\n" } queryContent += p.memoryBackground(ctx, chatManage) vals := types.PlaceholderValues{ "conversation": conversationText, "query": queryContent, "language": chatManage.Language, } return types.RenderPromptPlaceholders(systemPrompt, vals), types.RenderPromptPlaceholders(userPrompt, vals) } // memoryBackground gives the rewriter who is asking. // // This is the point where long-term memory stops being a paragraph appended to // the answer prompt and starts changing what gets retrieved. "How do I tune the // segmentation" is a different search for someone who works on medical imaging // than for someone who works on autonomous driving, and the only place that // difference can be applied is before retrieval runs. // // It is deliberately advisory rather than a filter. Memory narrows nothing and // excludes no knowledge base: a stale note about last quarter's project must // not be able to make this quarter's documents unreachable. func (p *PluginQueryUnderstand) memoryBackground(ctx context.Context, chatManage *types.ChatManage) string { if p.memoryService == nil { return "" } memCtx := p.memoryService.RetrievalContextFor(ctx) if memCtx.Empty() { return "" } var b strings.Builder b.WriteString("\n\n") if memCtx.Background != "" { b.WriteString("\n" + memCtx.Background) } if len(memCtx.Interests) > 0 { b.WriteString("\n长期关注:" + strings.Join(memCtx.Interests, "、")) } if len(memCtx.Documents) > 0 { b.WriteString("\n常查资料:" + strings.Join(memCtx.Documents, "、")) } b.WriteString("\n") // Deliberately does not add to chatManage.UsedMemories. What this reads is // the whole standing background, unfiltered — that is the right input for a // rewriter, but reporting it would claim every turn recalled memories that // have nothing to do with the question. Which memories this turn actually // used is decided in MEMORY_RECALL, by relevance, and the profile entries // here are already reported from there. fields := map[string]interface{}{ "session_id": chatManage.SessionID, "interests": len(memCtx.Interests), "documents": len(memCtx.Documents), "items": len(memCtx.Items), } if len(memCtx.Interests) > 0 { fields["interest_previews"] = memCtx.Interests } pipelineInfo(ctx, "QueryUnderstand", "memory_background", fields) return b.String() } // parseOutput extracts the rewritten query, intent classification, and optional // image description from the model's structured JSON output. // // Expected format: {"rewrite_query":"...","intent":"kb_search","image_description":"..."} func (p *PluginQueryUnderstand) parseOutput(chatManage *types.ChatManage, raw string) { content := strings.TrimSpace(raw) if content == "" { return } if output, ok := parseStructuredQueryOutput(content); ok { if rewrite := strings.TrimSpace(output.RewriteQuery); rewrite != "" { chatManage.RewriteQuery = rewrite } chatManage.Intent = output.Intent chatManage.ImageDescription = strings.TrimSpace(output.ImageDescription) return } // On parse failure, keep the original query and intent. } func parseStructuredQueryOutput(raw string) (queryUnderstandOutput, bool) { content := strings.TrimSpace(raw) if content == "" { return queryUnderstandOutput{}, false } if parsed, ok := parseStructuredQueryOutputJSON(content); ok { return parsed, true } // Be tolerant to occasional markdown wrappers or extra prose. start := strings.Index(content, "{") end := strings.LastIndex(content, "}") if start < 0 || end <= start { return queryUnderstandOutput{}, false } candidate := content[start : end+1] if parsed, ok := parseStructuredQueryOutputJSON(candidate); ok { return parsed, true } return queryUnderstandOutput{}, false } func parseStructuredQueryOutputJSON(content string) (queryUnderstandOutput, bool) { var obj map[string]json.RawMessage if err := json.Unmarshal([]byte(content), &obj); err != nil { return queryUnderstandOutput{}, false } out := queryUnderstandOutput{ RewriteQuery: strings.TrimSpace(firstStringField(obj, "rewrite_query", "rewritten_query", "query", "question")), } intentStr := strings.TrimSpace(firstStringField(obj, "intent")) if intentStr != "" { out.Intent = types.QueryIntent(intentStr) } desc := strings.TrimSpace(firstStringField(obj, "image_description", "image_desc", "image_text", "image_ocr_text", "description")) ocr := strings.TrimSpace(firstStringField(obj, "ocr_text", "ocr", "full_ocr", "image_ocr", "ocr_content")) combined, set := mergeImageDescAndOCR(desc, ocr) if set { out.ImageDescription = combined } return out, true } func firstStringField(obj map[string]json.RawMessage, keys ...string) string { for _, key := range keys { raw, ok := obj[key] if !ok || len(raw) == 0 { continue } var s string if err := json.Unmarshal(raw, &s); err == nil { return s } } return "" } func mergeImageDescAndOCR(desc, ocr string) (string, bool) { if desc == "" && ocr == "" { return "", false } if desc == "" { return ocr, true } if ocr == "" { return desc, true } if strings.Contains(desc, ocr) { return desc, true } return desc + "\n\n[OCR]\n" + ocr, true } // applyIntentPromptOverride resolves the system-prompt override for the current // non-retrieval intent. Agent-level overrides take precedence; otherwise the // tenant/global IntentSystemPrompts map is consulted. Whitespace-only agent // overrides are treated as unset and fall through to the global default. Returns // true when a non-empty override was applied. func applyIntentPromptOverride(chatManage *types.ChatManage, globalPrompts map[string]string) bool { intentKey := string(chatManage.Intent) if raw, ok := chatManage.IntentPromptOverrides[intentKey]; ok && strings.TrimSpace(raw) != "" { chatManage.SystemPromptOverride = raw } if chatManage.SystemPromptOverride == "" { if prompt, ok := globalPrompts[intentKey]; ok { chatManage.SystemPromptOverride = prompt } } return chatManage.SystemPromptOverride != "" } // formatConversationHistory formats conversation history for prompt template. func formatConversationHistory(historyList []*types.History) string { if len(historyList) == 0 { return "" } var builder strings.Builder for _, h := range historyList { builder.WriteString("------BEGIN------\n") builder.WriteString("User question: ") builder.WriteString(h.Query) builder.WriteString("\nAssistant answer: ") builder.WriteString(h.Answer) builder.WriteString("\n------END------\n") } return builder.String() }