WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/models/vlm/concurrency_wrapper.go

@@ -0,0 +1,38 @@
+package vlm
+
+import (
+	"context"
+
+	"github.com/Tencent/WeKnora/internal/models/limiter"
+)
+
+// Multimodal enrichment (image OCR / caption) is a high-volume, slow background
+// stage that hits the same provider budget as chat. Like chat, it must be
+// governed at the client layer so an image-heavy ingestion storm can't burst
+// the whole worker pool against one VLM provider. Only background (asynq
+// worker) calls are throttled — see limiter.Gate / types.IsBackgroundTask.
+type concurrencyVLM struct {
+	inner VLM
+	// limit is this model's configured per-model background cap; 0 falls back
+	// to the process-wide default (see limiter.GateN).
+	limit int
+}
+
+func (w *concurrencyVLM) GetModelName() string { return w.inner.GetModelName() }
+func (w *concurrencyVLM) GetModelID() string   { return w.inner.GetModelID() }
+
+func (w *concurrencyVLM) Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error) {
+	release := limiter.GateNamedN(ctx, w.inner.GetModelID(), w.inner.GetModelName(), w.limit)
+	defer release()
+	return w.inner.Predict(ctx, imgBytes, prompt)
+}
+
+// wrapVLMConcurrency installs the background concurrency governor as the
+// outermost VLM decorator. Always applied; a cheap passthrough when no limiter
+// is installed or the call is interactive.
+func wrapVLMConcurrency(v VLM, limit int, err error) (VLM, error) {
+	if err != nil || v == nil {
+		return v, err
+	}
+	return &concurrencyVLM{inner: v, limit: limit}, nil
+}

internal/models/vlm/config_from_model_test.go

@@ -0,0 +1,59 @@
+package vlm
+
+import (
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestConfigFromModel_RemoteDefaultsToOpenAI(t *testing.T) {
+	m := &types.Model{
+		ID:     "v1",
+		Name:   "gpt-4o",
+		Source: types.ModelSourceRemote,
+		Parameters: types.ModelParameters{
+			BaseURL:       "https://api.example.com/v1",
+			APIKey:        "sk",
+			Provider:      "openai",
+			ExtraConfig:   map[string]string{"x": "y"},
+			CustomHeaders: map[string]string{"H": "v"},
+		},
+	}
+	cfg := ConfigFromModel(m, "app", "secret")
+	if cfg.InterfaceType != "openai" {
+		t.Errorf("expected openai default for remote, got %q", cfg.InterfaceType)
+	}
+	if cfg.CustomHeaders["H"] != "v" {
+		t.Errorf("CustomHeaders not propagated: %+v", cfg.CustomHeaders)
+	}
+	if cfg.Extra["x"] != "y" {
+		t.Errorf("ExtraConfig not propagated as Extra: %+v", cfg.Extra)
+	}
+	if cfg.AppID != "app" || cfg.AppSecret != "secret" {
+		t.Errorf("cloud creds mismatch: %+v", cfg)
+	}
+}
+
+func TestConfigFromModel_LocalDefaultsToOllama(t *testing.T) {
+	m := &types.Model{
+		Name:   "qwen2-vl",
+		Source: types.ModelSourceLocal,
+	}
+	cfg := ConfigFromModel(m, "", "")
+	if cfg.InterfaceType != "ollama" {
+		t.Errorf("expected ollama default for local, got %q", cfg.InterfaceType)
+	}
+}
+
+func TestConfigFromModel_RespectsExplicitInterface(t *testing.T) {
+	m := &types.Model{
+		Name:   "qwen2-vl",
+		Source: types.ModelSourceRemote,
+		Parameters: types.ModelParameters{
+			InterfaceType: "ollama",
+		},
+	}
+	if got := ConfigFromModel(m, "", "").InterfaceType; got != "ollama" {
+		t.Errorf("expected explicit interface to win, got %q", got)
+	}
+}

internal/models/vlm/langfuse_wrapper.go

@@ -0,0 +1,72 @@
+package vlm
+
+import (
+	"context"
+
+	"github.com/Tencent/WeKnora/internal/tracing/langfuse"
+)
+
+// langfuseVLM wraps a VLM and reports each Predict call as a Langfuse
+// generation. The raw image bytes are NOT uploaded — Langfuse traces are
+// designed for text. We include image count and total byte size in the
+// metadata, plus the text prompt, which matches how Langfuse's own VLM
+// integrations report multimodal calls.
+type langfuseVLM struct {
+	inner VLM
+}
+
+func (l *langfuseVLM) GetModelName() string { return l.inner.GetModelName() }
+func (l *langfuseVLM) GetModelID() string   { return l.inner.GetModelID() }
+
+func (l *langfuseVLM) Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error) {
+	mgr := langfuse.GetManager()
+	if !mgr.Enabled() {
+		return l.inner.Predict(ctx, imgBytes, prompt)
+	}
+
+	totalImgSize := 0
+	for _, b := range imgBytes {
+		totalImgSize += len(b)
+	}
+
+	genCtx, gen := mgr.StartGeneration(ctx, langfuse.GenerationOptions{
+		Name:  "vlm.predict",
+		Model: l.inner.GetModelName(),
+		Input: map[string]interface{}{
+			"prompt":      prompt,
+			"image_count": len(imgBytes),
+		},
+		Metadata: map[string]interface{}{
+			"model_id":          l.inner.GetModelID(),
+			"image_count":       len(imgBytes),
+			"image_bytes_total": totalImgSize,
+		},
+	})
+
+	result, err := l.inner.Predict(genCtx, imgBytes, prompt)
+
+	// VLMs don't return token usage; approximate prompt tokens for cost
+	// tracking in Langfuse (users can configure per-model pricing in the UI).
+	promptTokens := len([]rune(prompt))/4 + 1
+	outputTokens := len([]rune(result)) / 4
+	usage := &langfuse.TokenUsage{
+		Input:  promptTokens,
+		Output: outputTokens,
+		Total:  promptTokens + outputTokens,
+		Unit:   "TOKENS",
+	}
+
+	gen.Finish(result, usage, err)
+	return result, err
+}
+
+// wrapVLMLangfuse applies the Langfuse decorator when the manager is enabled.
+func wrapVLMLangfuse(v VLM, err error) (VLM, error) {
+	if err != nil || v == nil {
+		return v, err
+	}
+	if !langfuse.GetManager().Enabled() {
+		return v, nil
+	}
+	return &langfuseVLM{inner: v}, nil
+}

internal/models/vlm/llm_debug.go

@@ -0,0 +1,63 @@
+package vlm
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/logger"
+)
+
+// debugVLM wraps a VLM with LLM debug logging.
+type debugVLM struct {
+	inner VLM
+}
+
+func (d *debugVLM) Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error) {
+	start := time.Now()
+	result, err := d.inner.Predict(ctx, imgBytes, prompt)
+	logVLMDebug(ctx, d.inner.GetModelName(), imgBytes, prompt, result, err, time.Since(start))
+	return result, err
+}
+
+func (d *debugVLM) GetModelName() string { return d.inner.GetModelName() }
+func (d *debugVLM) GetModelID() string   { return d.inner.GetModelID() }
+
+func logVLMDebug(ctx context.Context, model string, imgBytes [][]byte, prompt string, response string, callErr error, dur time.Duration) {
+	if !logger.LLMDebugEnabled() {
+		return
+	}
+
+	record := &logger.LLMCallRecord{
+		CallType: "VLM",
+		Model:    model,
+		Duration: dur,
+	}
+
+	// Input section
+	var inputBuf strings.Builder
+	inputBuf.WriteString(fmt.Sprintf("Images: count=%d", len(imgBytes)))
+	totalSize := 0
+	for _, img := range imgBytes {
+		totalSize += len(img)
+	}
+	inputBuf.WriteString(fmt.Sprintf(", total_size=%d bytes\n\n", totalSize))
+	inputBuf.WriteString("[prompt]\n")
+	inputBuf.WriteString(prompt)
+	inputBuf.WriteString("\n")
+	record.Sections = append(record.Sections, logger.RecordSection{Title: "Input", Content: inputBuf.String()})
+
+	// Response section
+	if response != "" {
+		record.Sections = append(record.Sections, logger.RecordSection{
+			Title:   "Response",
+			Content: response,
+		})
+	}
+
+	if callErr != nil {
+		record.Error = callErr.Error()
+	}
+	logger.LLMDebugLog(ctx, record)
+}

internal/models/vlm/ollama.go

@@ -0,0 +1,74 @@
+package vlm
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/Tencent/WeKnora/internal/logger"
+	"github.com/Tencent/WeKnora/internal/models/utils/ollama"
+	ollamaapi "github.com/ollama/ollama/api"
+)
+
+// OllamaVLM implements VLM via the local Ollama service.
+type OllamaVLM struct {
+	modelName     string
+	modelID       string
+	ollamaService *ollama.OllamaService
+}
+
+// NewOllamaVLM creates an Ollama-backed VLM instance.
+func NewOllamaVLM(config *Config, ollamaService *ollama.OllamaService) (*OllamaVLM, error) {
+	if ollamaService == nil {
+		return nil, fmt.Errorf("ollama service is required for local VLM model")
+	}
+	return &OllamaVLM{
+		modelName:     config.ModelName,
+		modelID:       config.ModelID,
+		ollamaService: ollamaService,
+	}, nil
+}
+
+// Predict sends an image with a text prompt to the Ollama vision model.
+func (v *OllamaVLM) Predict(ctx context.Context, imgBytesList [][]byte, prompt string) (string, error) {
+	streamFlag := false
+	var images []ollamaapi.ImageData
+	for _, imgBytes := range imgBytesList {
+		if len(imgBytes) > 0 {
+			images = append(images, imgBytes)
+		}
+	}
+
+	chatReq := &ollamaapi.ChatRequest{
+		Model: v.modelName,
+		Messages: []ollamaapi.Message{
+			{
+				Role:    "user",
+				Content: prompt,
+				Images:  images,
+			},
+		},
+		Stream:  &streamFlag,
+		Options: map[string]interface{}{"temperature": 0.1},
+	}
+
+	totalImageSize := 0
+	for _, img := range imgBytesList {
+		totalImageSize += len(img)
+	}
+	logger.Infof(ctx, "[VLM] Calling Ollama API, model=%s, numImages=%d, totalImageSize=%d", v.modelName, len(images), totalImageSize)
+
+	var result string
+	err := v.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error {
+		result = resp.Message.Content
+		return nil
+	})
+	if err != nil {
+		return "", fmt.Errorf("Ollama VLM request: %w", err)
+	}
+
+	logger.Infof(ctx, "[VLM] Ollama response received, len=%d", len(result))
+	return result, nil
+}
+
+func (v *OllamaVLM) GetModelName() string { return v.modelName }
+func (v *OllamaVLM) GetModelID() string   { return v.modelID }

internal/models/vlm/remote_api.go

@@ -0,0 +1,214 @@
+package vlm
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/logger"
+	"github.com/Tencent/WeKnora/internal/models/provider"
+	secutils "github.com/Tencent/WeKnora/internal/utils"
+	openai "github.com/sashabaranov/go-openai"
+)
+
+const (
+	// defaultTimeout is the fallback HTTP timeout for a single VLM request.
+	// Dense scanned-PDF OCR (full-page text + layout extraction) can take well
+	// over a minute on slow endpoints, so this is intentionally generous and
+	// can be raised further via VLM_HTTP_TIMEOUT_SECONDS.
+	defaultTimeout = 180 * time.Second
+	defaultMaxToks = 5000
+	defaultTemp    = float32(0.1)
+)
+
+// vlmHTTPTimeout returns the HTTP client timeout for VLM requests, read from
+// the VLM_HTTP_TIMEOUT_SECONDS env var when set (and positive), falling back to
+// defaultTimeout otherwise. Shared by all OpenAI-compatible VLM backends.
+func vlmHTTPTimeout() time.Duration {
+	if v := strings.TrimSpace(os.Getenv("VLM_HTTP_TIMEOUT_SECONDS")); v != "" {
+		if secs, err := strconv.Atoi(v); err == nil && secs > 0 {
+			return time.Duration(secs) * time.Second
+		}
+	}
+	return defaultTimeout
+}
+
+// RemoteAPIVLM implements VLM via an OpenAI-compatible chat completions API.
+type RemoteAPIVLM struct {
+	modelName   string
+	modelID     string
+	client      *openai.Client
+	baseURL     string
+	temperature float32
+}
+
+// NewRemoteAPIVLM creates a remote-API backed VLM instance.
+func NewRemoteAPIVLM(config *Config) (*RemoteAPIVLM, error) {
+	if err := validateVLMBaseURL(config.BaseURL); err != nil {
+		return nil, err
+	}
+
+	providerName := provider.ProviderName(config.Provider)
+	if providerName == "" {
+		providerName = provider.DetectProvider(config.BaseURL)
+	}
+
+	var apiCfg openai.ClientConfig
+	if providerName == provider.ProviderAzureOpenAI {
+		apiCfg = openai.DefaultAzureConfig(config.APIKey, config.BaseURL)
+		apiCfg.AzureModelMapperFunc = func(model string) string {
+			return model
+		}
+		if config.Extra != nil {
+			if v, ok := config.Extra["api_version"]; ok {
+				if vs, ok := v.(string); ok && vs != "" {
+					apiCfg.APIVersion = vs
+				}
+			}
+		}
+	} else {
+		apiCfg = openai.DefaultConfig(config.APIKey)
+		if config.BaseURL != "" {
+			apiCfg.BaseURL = config.BaseURL
+		}
+	}
+	httpClient := newVLMHTTPClient(vlmHTTPTimeout())
+
+	// 注入用户自定义 HTTP header（类似 OpenAI Python SDK 的 extra_headers）
+	if len(config.CustomHeaders) > 0 {
+		apiCfg.HTTPClient = secutils.WrapHTTPClientWithHeaders(httpClient, config.CustomHeaders)
+	} else {
+		apiCfg.HTTPClient = httpClient
+	}
+
+	temp := defaultTemp
+	if config.Extra != nil {
+		if v, ok := config.Extra["temperature"]; ok {
+			if vs, ok := v.(string); ok {
+				if f, err := strconv.ParseFloat(vs, 32); err == nil {
+					temp = float32(f)
+				}
+			}
+		}
+	}
+
+	return &RemoteAPIVLM{
+		modelName:   config.ModelName,
+		modelID:     config.ModelID,
+		client:      openai.NewClientWithConfig(apiCfg),
+		baseURL:     config.BaseURL,
+		temperature: temp,
+	}, nil
+}
+
+// Predict sends an image with a text prompt to the OpenAI-compatible API.
+func (v *RemoteAPIVLM) Predict(ctx context.Context, imgBytesList [][]byte, prompt string) (string, error) {
+	var parts []openai.ChatMessagePart
+
+	// Add text prompt first
+	parts = append(parts, openai.ChatMessagePart{
+		Type: openai.ChatMessagePartTypeText,
+		Text: prompt,
+	})
+
+	// Add images
+	for _, imgBytes := range imgBytesList {
+		if len(imgBytes) > 0 {
+			mimeType := detectImageMIME(imgBytes)
+			b64 := base64.StdEncoding.EncodeToString(imgBytes)
+			dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType, b64)
+			parts = append(parts, openai.ChatMessagePart{
+				Type: openai.ChatMessagePartTypeImageURL,
+				ImageURL: &openai.ChatMessageImageURL{
+					URL:    dataURI,
+					Detail: openai.ImageURLDetailAuto,
+				},
+			})
+		}
+	}
+
+	req := openai.ChatCompletionRequest{
+		Model: v.modelName,
+		Messages: []openai.ChatCompletionMessage{
+			{
+				Role:         openai.ChatMessageRoleUser,
+				MultiContent: parts,
+			},
+		},
+		MaxTokens:   defaultMaxToks,
+		Temperature: v.temperature,
+	}
+	shapeReasoningVLMRequest(&req)
+
+	totalImageSize := 0
+	for _, img := range imgBytesList {
+		totalImageSize += len(img)
+	}
+	logger.Infof(ctx, "[VLM] Calling OpenAI-compatible API, model=%s, baseURL=%s, numImages=%d, totalImageSize=%d",
+		v.modelName, v.baseURL, len(imgBytesList), totalImageSize)
+
+	resp, err := v.client.CreateChatCompletion(ctx, req)
+	if err != nil {
+		return "", fmt.Errorf("OpenAI VLM request: %w", err)
+	}
+	if len(resp.Choices) == 0 {
+		return "", fmt.Errorf("OpenAI VLM returned no choices")
+	}
+
+	choice := resp.Choices[0]
+	content := choice.Message.Content
+	if strings.TrimSpace(content) == "" && choice.FinishReason == openai.FinishReasonLength {
+		// Reasoning models spend max_completion_tokens on reasoning before any
+		// visible output, so an exhausted budget yields an empty message rather
+		// than an API error. Returning "" here would be recorded as
+		// "no_extracted_content" and look identical to an image with no text.
+		return "", fmt.Errorf(
+			"OpenAI VLM returned no content: completion truncated at %d tokens (finish_reason=length)",
+			defaultMaxToks,
+		)
+	}
+	logger.Infof(ctx, "[VLM] OpenAI response received, len=%d", len(content))
+	return content, nil
+}
+
+// shapeReasoningVLMRequest adapts an OpenAI-compatible VLM request for
+// reasoning (o-series) and GPT-5 models, which reject `max_tokens` and every
+// non-default sampling parameter.
+//
+// This mirrors shapeOpenAIReasoning in internal/models/chat, which fixed the
+// same incompatibility on the chat path for issue #1283. The VLM path was
+// never wired to it, so image OCR and captioning failed for every one of these
+// models (issue #2537).
+//
+// Both quirks have to be handled together: migrating max_tokens alone still
+// fails, because the VLM default temperature (0.1) is itself rejected.
+func shapeReasoningVLMRequest(req *openai.ChatCompletionRequest) {
+	if !provider.IsOpenAIReasoningOrGPT5Model(req.Model) {
+		return
+	}
+	if req.MaxCompletionTokens == 0 && req.MaxTokens > 0 {
+		req.MaxCompletionTokens = req.MaxTokens
+	}
+	req.MaxTokens = 0
+	req.Temperature = 0
+	req.TopP = 0
+	req.FrequencyPenalty = 0
+	req.PresencePenalty = 0
+}
+
+func (v *RemoteAPIVLM) GetModelName() string { return v.modelName }
+func (v *RemoteAPIVLM) GetModelID() string   { return v.modelID }
+
+// detectImageMIME returns the MIME type for the given image bytes.
+func detectImageMIME(data []byte) string {
+	ct := http.DetectContentType(data)
+	if strings.HasPrefix(ct, "image/") {
+		return ct
+	}
+	return "image/png"
+}

internal/models/vlm/remote_api_reasoning_test.go

@@ -0,0 +1,293 @@
+package vlm
+
+import (
+	"encoding/json"
+	"errors"
+	"math"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	openai "github.com/sashabaranov/go-openai"
+)
+
+// TestShapeReasoningVLMRequest 验证 GPT-5 / o-series 的请求参数改写。
+// 见 issue #2537：这些模型必须使用 max_completion_tokens 替代 max_tokens，
+// 且不接受非默认的采样参数。
+func TestShapeReasoningVLMRequest(t *testing.T) {
+	cases := []struct {
+		name                    string
+		model                   string
+		maxTokens               int
+		maxCompletionTokens     int
+		temperature             float32
+		wantMaxTokens           int
+		wantMaxCompletionTokens int
+		wantTemperature         float32
+	}{
+		{
+			name:  "gpt-5 migrates max_tokens and drops temperature",
+			model: "gpt-5", maxTokens: 5000, temperature: 0.1,
+			wantMaxTokens: 0, wantMaxCompletionTokens: 5000, wantTemperature: 0,
+		},
+		{
+			name:  "gpt-5-nano is shaped", // the model reported in issue #2537
+			model: "gpt-5-nano", maxTokens: 5000, temperature: 0.1,
+			wantMaxTokens: 0, wantMaxCompletionTokens: 5000, wantTemperature: 0,
+		},
+		{
+			name:  "gpt-5 mixed case is shaped",
+			model: "GPT-5.4-Mini", maxTokens: 5000, temperature: 0.1,
+			wantMaxTokens: 0, wantMaxCompletionTokens: 5000, wantTemperature: 0,
+		},
+		{
+			name:  "o1-mini is shaped",
+			model: "o1-mini", maxTokens: 5000, temperature: 0.1,
+			wantMaxTokens: 0, wantMaxCompletionTokens: 5000, wantTemperature: 0,
+		},
+		{
+			name:  "o3 is shaped",
+			model: "o3", maxTokens: 5000, temperature: 0.1,
+			wantMaxTokens: 0, wantMaxCompletionTokens: 5000, wantTemperature: 0,
+		},
+		{
+			name:  "o4-mini is shaped",
+			model: "o4-mini", maxTokens: 5000, temperature: 0.1,
+			wantMaxTokens: 0, wantMaxCompletionTokens: 5000, wantTemperature: 0,
+		},
+		{
+			name:  "explicit max_completion_tokens is preserved",
+			model: "gpt-5", maxTokens: 5000, maxCompletionTokens: 128, temperature: 0.1,
+			wantMaxTokens: 0, wantMaxCompletionTokens: 128, wantTemperature: 0,
+		},
+		{
+			name:  "gpt-4o is left untouched",
+			model: "gpt-4o", maxTokens: 5000, temperature: 0.1,
+			wantMaxTokens: 5000, wantMaxCompletionTokens: 0, wantTemperature: 0.1,
+		},
+		{
+			name:  "qwen-vl is left untouched",
+			model: "qwen2.5-vl-7b-instruct", maxTokens: 5000, temperature: 0.1,
+			wantMaxTokens: 5000, wantMaxCompletionTokens: 0, wantTemperature: 0.1,
+		},
+		{
+			name:  "empty model is left untouched",
+			model: "", maxTokens: 5000, temperature: 0.1,
+			wantMaxTokens: 5000, wantMaxCompletionTokens: 0, wantTemperature: 0.1,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			req := openai.ChatCompletionRequest{
+				Model:               tc.model,
+				MaxTokens:           tc.maxTokens,
+				MaxCompletionTokens: tc.maxCompletionTokens,
+				Temperature:         tc.temperature,
+			}
+			shapeReasoningVLMRequest(&req)
+
+			if req.MaxTokens != tc.wantMaxTokens {
+				t.Errorf("MaxTokens = %d, want %d", req.MaxTokens, tc.wantMaxTokens)
+			}
+			if req.MaxCompletionTokens != tc.wantMaxCompletionTokens {
+				t.Errorf("MaxCompletionTokens = %d, want %d", req.MaxCompletionTokens, tc.wantMaxCompletionTokens)
+			}
+			if req.Temperature != tc.wantTemperature {
+				t.Errorf("Temperature = %v, want %v", req.Temperature, tc.wantTemperature)
+			}
+		})
+	}
+}
+
+// newVLMChatTestServer emulates an OpenAI-compatible chat completions endpoint
+// and records the last decoded request body.
+func newVLMChatTestServer(t *testing.T, lastRequest *map[string]interface{}) *httptest.Server {
+	t.Helper()
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var req map[string]interface{}
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+			t.Errorf("decode VLM request: %v", err)
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		*lastRequest = req
+
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{
+			"id": "chatcmpl-test",
+			"object": "chat.completion",
+			"choices": [
+				{"index": 0, "message": {"role": "assistant", "content": "extracted text"}, "finish_reason": "stop"}
+			]
+		}`))
+	}))
+}
+
+// testPNG is a minimal byte slice that http.DetectContentType reports as a PNG.
+var testPNG = []byte("\x89PNG\r\n\x1a\n" + strings.Repeat("\x00", 16))
+
+// TestRemoteAPIVLMSendsMaxCompletionTokensForReasoningModel is the regression
+// test for issue #2537: with a GPT-5 / o-series vision model, every OCR and
+// caption call failed with
+//
+//	"this model is not supported MaxTokens, please use MaxCompletionTokens"
+//
+// The request was rejected client-side by go-openai's reasoning validator, so
+// it never reached the server and no image chunk was ever created.
+func TestRemoteAPIVLMSendsMaxCompletionTokensForReasoningModel(t *testing.T) {
+	withVLMSSRFWhitelist(t, "127.0.0.1")
+
+	var lastRequest map[string]interface{}
+	server := newVLMChatTestServer(t, &lastRequest)
+	defer server.Close()
+
+	v, err := NewRemoteAPIVLM(&Config{
+		BaseURL:   server.URL,
+		ModelName: "gpt-5-nano",
+		APIKey:    "sk-test",
+	})
+	if err != nil {
+		t.Fatalf("NewRemoteAPIVLM: %v", err)
+	}
+
+	content, err := v.Predict(t.Context(), [][]byte{testPNG}, "extract the text")
+	if err != nil {
+		t.Fatalf("Predict: %v", err)
+	}
+	if content != "extracted text" {
+		t.Errorf("content = %q, want %q", content, "extracted text")
+	}
+
+	if _, ok := lastRequest["max_tokens"]; ok {
+		t.Errorf("request carries max_tokens, which reasoning models reject: %v", lastRequest["max_tokens"])
+	}
+	if got, ok := lastRequest["max_completion_tokens"]; !ok {
+		t.Error("request is missing max_completion_tokens")
+	} else if got != float64(defaultMaxToks) {
+		t.Errorf("max_completion_tokens = %v, want %d", got, defaultMaxToks)
+	}
+	// Temperature 0.1 is itself rejected for these models, so migrating
+	// max_tokens alone would not have been enough.
+	if _, ok := lastRequest["temperature"]; ok {
+		t.Errorf("request carries temperature, which reasoning models reject: %v", lastRequest["temperature"])
+	}
+}
+
+// TestRemoteAPIVLMKeepsMaxTokensForNonReasoningModel guards against the fix
+// regressing ordinary vision models, which still expect max_tokens.
+func TestRemoteAPIVLMKeepsMaxTokensForNonReasoningModel(t *testing.T) {
+	withVLMSSRFWhitelist(t, "127.0.0.1")
+
+	var lastRequest map[string]interface{}
+	server := newVLMChatTestServer(t, &lastRequest)
+	defer server.Close()
+
+	v, err := NewRemoteAPIVLM(&Config{
+		BaseURL:   server.URL,
+		ModelName: "gpt-4o",
+		APIKey:    "sk-test",
+	})
+	if err != nil {
+		t.Fatalf("NewRemoteAPIVLM: %v", err)
+	}
+
+	if _, err := v.Predict(t.Context(), [][]byte{testPNG}, "extract the text"); err != nil {
+		t.Fatalf("Predict: %v", err)
+	}
+
+	if got, ok := lastRequest["max_tokens"]; !ok {
+		t.Error("request is missing max_tokens")
+	} else if got != float64(defaultMaxToks) {
+		t.Errorf("max_tokens = %v, want %d", got, defaultMaxToks)
+	}
+	if _, ok := lastRequest["max_completion_tokens"]; ok {
+		t.Error("request carries max_completion_tokens for a non-reasoning model")
+	}
+	if got, ok := lastRequest["temperature"]; !ok {
+		t.Error("request is missing temperature")
+	} else if f, isFloat := got.(float64); !isFloat || math.Abs(f-float64(defaultTemp)) > 1e-6 {
+		t.Errorf("temperature = %v, want %v", got, defaultTemp)
+	}
+}
+
+// TestRemoteAPIVLMReportsTruncatedCompletion covers the other way a reasoning
+// model yields nothing: max_completion_tokens also covers reasoning tokens, so
+// an exhausted budget returns an empty message with finish_reason=length
+// instead of an API error. Reporting that as an error keeps it out of the
+// "no_extracted_content" bucket, where issue #2537 notes the failure is
+// indistinguishable from an image that genuinely has no text.
+func TestRemoteAPIVLMReportsTruncatedCompletion(t *testing.T) {
+	withVLMSSRFWhitelist(t, "127.0.0.1")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{
+			"id": "chatcmpl-test",
+			"object": "chat.completion",
+			"choices": [
+				{"index": 0, "message": {"role": "assistant", "content": ""}, "finish_reason": "length"}
+			]
+		}`))
+	}))
+	defer server.Close()
+
+	v, err := NewRemoteAPIVLM(&Config{
+		BaseURL:   server.URL,
+		ModelName: "gpt-5-nano",
+		APIKey:    "sk-test",
+	})
+	if err != nil {
+		t.Fatalf("NewRemoteAPIVLM: %v", err)
+	}
+
+	_, err = v.Predict(t.Context(), [][]byte{testPNG}, "extract the text")
+	if err == nil {
+		t.Fatal("Predict returned nil error for a truncated completion")
+	}
+	if !strings.Contains(err.Error(), "truncated") {
+		t.Errorf("error = %q, want it to mention truncation", err.Error())
+	}
+}
+
+// TestRemoteAPIVLMUnshapedReasoningRequestIsRejected pins the upstream
+// behavior this fix works around: without the shaping, go-openai rejects the
+// request before it leaves the process. It fails identically for max_tokens
+// and for a non-default temperature.
+func TestRemoteAPIVLMUnshapedReasoningRequestIsRejected(t *testing.T) {
+	withVLMSSRFWhitelist(t, "127.0.0.1")
+
+	var lastRequest map[string]interface{}
+	server := newVLMChatTestServer(t, &lastRequest)
+	defer server.Close()
+
+	v, err := NewRemoteAPIVLM(&Config{
+		BaseURL:   server.URL,
+		ModelName: "gpt-5-nano",
+		APIKey:    "sk-test",
+	})
+	if err != nil {
+		t.Fatalf("NewRemoteAPIVLM: %v", err)
+	}
+
+	unshaped := openai.ChatCompletionRequest{
+		Model:     "gpt-5-nano",
+		Messages:  []openai.ChatCompletionMessage{{Role: openai.ChatMessageRoleUser, Content: "hi"}},
+		MaxTokens: defaultMaxToks,
+	}
+	_, err = v.client.CreateChatCompletion(t.Context(), unshaped)
+	if !errors.Is(err, openai.ErrReasoningModelMaxTokensDeprecated) {
+		t.Errorf("max_tokens error = %v, want ErrReasoningModelMaxTokensDeprecated", err)
+	}
+
+	tempOnly := openai.ChatCompletionRequest{
+		Model:       "gpt-5-nano",
+		Messages:    []openai.ChatCompletionMessage{{Role: openai.ChatMessageRoleUser, Content: "hi"}},
+		Temperature: defaultTemp,
+	}
+	_, err = v.client.CreateChatCompletion(t.Context(), tempOnly)
+	if !errors.Is(err, openai.ErrReasoningModelLimitationsOther) {
+		t.Errorf("temperature error = %v, want ErrReasoningModelLimitationsOther", err)
+	}
+}

internal/models/vlm/transport.go

@@ -0,0 +1,25 @@
+package vlm
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	secutils "github.com/Tencent/WeKnora/internal/utils"
+)
+
+func validateVLMBaseURL(baseURL string) error {
+	if baseURL == "" {
+		return nil
+	}
+	if err := secutils.ValidateURLForSSRF(baseURL); err != nil {
+		return fmt.Errorf("base URL SSRF check failed: %w", err)
+	}
+	return nil
+}
+
+func newVLMHTTPClient(timeout time.Duration) *http.Client {
+	cfg := secutils.DefaultSSRFSafeHTTPClientConfig()
+	cfg.Timeout = timeout
+	return secutils.NewSSRFSafeHTTPClient(cfg)
+}

internal/models/vlm/transport_security_test.go

@@ -0,0 +1,26 @@
+package vlm
+
+import (
+	"testing"
+
+	secutils "github.com/Tencent/WeKnora/internal/utils"
+)
+
+func withVLMSSRFWhitelist(t *testing.T, raw string) {
+	t.Helper()
+	t.Setenv("SSRF_WHITELIST", raw)
+	secutils.ResetSSRFWhitelistForTest()
+	t.Cleanup(secutils.ResetSSRFWhitelistForTest)
+}
+
+func TestRemoteAPIVLMRejectsInternalBaseURL(t *testing.T) {
+	withVLMSSRFWhitelist(t, "")
+
+	_, err := NewRemoteAPIVLM(&Config{
+		BaseURL:   "http://169.254.169.254/latest/meta-data/",
+		ModelName: "vlm-test",
+	})
+	if err == nil {
+		t.Fatalf("NewRemoteAPIVLM returned nil error for blocked internal BaseURL")
+	}
+}

internal/models/vlm/vlm.go

@@ -0,0 +1,141 @@
+package vlm
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/Tencent/WeKnora/internal/logger"
+	"github.com/Tencent/WeKnora/internal/models/provider"
+	"github.com/Tencent/WeKnora/internal/models/utils/ollama"
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// VLM defines the interface for Vision Language Model operations.
+type VLM interface {
+	// Predict sends one or more images with a text prompt to the VLM and returns the generated text.
+	Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error)
+
+	GetModelName() string
+	GetModelID() string
+}
+
+// Config holds the configuration needed to create a VLM instance.
+type Config struct {
+	Source        types.ModelSource
+	BaseURL       string
+	ModelName     string
+	APIKey        string
+	ModelID       string
+	InterfaceType string // "ollama" or "openai" (default)
+	Provider      string
+	// MaxConcurrency caps concurrent background calls to this model; 0 falls
+	// back to the process-wide default (see limiter.GateN).
+	MaxConcurrency int
+	Extra          map[string]any
+	// CustomHeaders 允许在调用远程 API 时附加自定义 HTTP 请求头（类似 OpenAI Python SDK 的 extra_headers）。
+	CustomHeaders map[string]string
+	AppID         string
+	AppSecret     string
+}
+
+// ConfigFromModel 根据 types.Model 构造 vlm.Config。
+// 生产路径（从 DB 拉起）和测试连接路径（临时表单）共享这份映射。
+// appID / appSecret 是已解密的 WeKnoraCloud 凭证，调用方负责传入。
+// InterfaceType 会根据 source / 模型参数自动回退到合理默认值。
+func ConfigFromModel(m *types.Model, appID, appSecret string) *Config {
+	if m == nil {
+		return nil
+	}
+	ifType := m.Parameters.InterfaceType
+	if ifType == "" {
+		if m.Source == types.ModelSourceLocal {
+			ifType = "ollama"
+		} else {
+			ifType = "openai"
+		}
+	}
+	return &Config{
+		ModelID:        m.ID,
+		APIKey:         m.Parameters.APIKey,
+		BaseURL:        m.Parameters.BaseURL,
+		ModelName:      m.Name,
+		Source:         m.Source,
+		InterfaceType:  ifType,
+		Provider:       m.Parameters.Provider,
+		MaxConcurrency: m.Parameters.MaxConcurrency,
+		Extra:          stringMapToAnyMap(m.Parameters.ExtraConfig),
+		CustomHeaders:  m.Parameters.CustomHeaders,
+		AppID:          appID,
+		AppSecret:      appSecret,
+	}
+}
+
+func stringMapToAnyMap(in map[string]string) map[string]any {
+	if in == nil {
+		return nil
+	}
+	out := make(map[string]any, len(in))
+	for k, v := range in {
+		out[k] = v
+	}
+	return out
+}
+
+// NewVLM creates a VLM instance based on the provided configuration.
+func NewVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
+	v, err := newVLM(config, ollamaService)
+	if err != nil {
+		return v, err
+	}
+	if logger.LLMDebugEnabled() {
+		v = &debugVLM{inner: v}
+	}
+	v, err = wrapVLMLangfuse(v, nil)
+	// Outermost: hold the per-model concurrency slot only around the real
+	// provider round-trip, so the wait is excluded from debug/langfuse timing.
+	return wrapVLMConcurrency(v, config.MaxConcurrency, err)
+}
+
+func newVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
+	ifType := strings.ToLower(config.InterfaceType)
+
+	if ifType == "ollama" || config.Source == types.ModelSourceLocal {
+		return NewOllamaVLM(config, ollamaService)
+	}
+
+	providerName := provider.ProviderName(config.Provider)
+	if providerName == "" {
+		providerName = provider.DetectProvider(config.BaseURL)
+	}
+	if providerName == provider.ProviderWeKnoraCloud {
+		return NewWeKnoraCloudVLM(config)
+	}
+
+	return NewRemoteAPIVLM(config)
+}
+
+// NewVLMFromLegacyConfig creates a VLM from a legacy VLMConfig (inline BaseURL/APIKey/ModelName).
+func NewVLMFromLegacyConfig(vlmCfg types.VLMConfig, ollamaService *ollama.OllamaService) (VLM, error) {
+	if !vlmCfg.IsEnabled() {
+		return nil, fmt.Errorf("VLM config is not enabled")
+	}
+
+	ifType := vlmCfg.InterfaceType
+	if ifType == "" {
+		ifType = "openai"
+	}
+
+	source := types.ModelSourceRemote
+	if strings.EqualFold(ifType, "ollama") {
+		source = types.ModelSourceLocal
+	}
+
+	return NewVLM(&Config{
+		Source:        source,
+		BaseURL:       vlmCfg.BaseURL,
+		ModelName:     vlmCfg.ModelName,
+		APIKey:        vlmCfg.APIKey,
+		InterfaceType: ifType,
+	}, ollamaService)
+}

internal/models/vlm/weknoracloud.go

@@ -0,0 +1,188 @@
+package vlm
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/Tencent/WeKnora/internal/logger"
+	"github.com/Tencent/WeKnora/internal/models/utils"
+	"github.com/google/uuid"
+)
+
+const weKnoraCloudVLMPath = "/api/v1/chat/completions"
+
+// WeKnoraCloudVLM implements VLM via the WeKnoraCloud API.
+type WeKnoraCloudVLM struct {
+	modelName       string
+	remoteModelName string
+	modelID         string
+	appID           string
+	apiKey          string
+	baseURL         string
+	client          *http.Client
+}
+
+// NewWeKnoraCloudVLM creates a WeKnoraCloud-backed VLM instance.
+func NewWeKnoraCloudVLM(config *Config) (*WeKnoraCloudVLM, error) {
+	if config.AppID == "" {
+		return nil, fmt.Errorf("WeKnoraCloud VLM: AppID is required")
+	}
+	if config.AppSecret == "" {
+		return nil, fmt.Errorf("WeKnoraCloud VLM: AppSecret is required")
+	}
+	baseURL := strings.TrimRight(config.BaseURL, "/")
+	if err := validateVLMBaseURL(baseURL); err != nil {
+		return nil, err
+	}
+	remoteModelName := ""
+	if config.Extra != nil {
+		if v, ok := config.Extra["remote_model_name"]; ok {
+			if vs, ok := v.(string); ok {
+				remoteModelName = strings.TrimSpace(vs)
+			}
+		}
+	}
+	return &WeKnoraCloudVLM{
+		modelName:       config.ModelName,
+		remoteModelName: remoteModelName,
+		modelID:         config.ModelID,
+		appID:           config.AppID,
+		apiKey:          config.AppSecret,
+		baseURL:         baseURL,
+		client:          newVLMHTTPClient(vlmHTTPTimeout()),
+	}, nil
+}
+
+type weKnoraCloudVLMContentPart struct {
+	Type     string                   `json:"type"`
+	Text     string                   `json:"text,omitempty"`
+	ImageURL *weKnoraCloudVLMImageURL `json:"image_url,omitempty"`
+}
+
+type weKnoraCloudVLMImageURL struct {
+	URL string `json:"url"`
+}
+
+type weKnoraCloudVLMMessage struct {
+	Role    string      `json:"role"`
+	Content interface{} `json:"content"`
+}
+
+type weKnoraCloudVLMRequest struct {
+	Model       string                   `json:"model"`
+	Messages    []weKnoraCloudVLMMessage `json:"messages"`
+	MaxTokens   int                      `json:"max_tokens,omitempty"`
+	Temperature float64                  `json:"temperature,omitempty"`
+	Stream      bool                     `json:"stream"`
+}
+
+type weKnoraCloudVLMResponse struct {
+	Choices []struct {
+		Message struct {
+			Content string `json:"content"`
+		} `json:"message"`
+	} `json:"choices"`
+}
+
+// Predict sends images with a text prompt to the WeKnoraCloud API.
+func (v *WeKnoraCloudVLM) Predict(ctx context.Context, imgBytesList [][]byte, prompt string) (string, error) {
+	var parts []weKnoraCloudVLMContentPart
+
+	parts = append(parts, weKnoraCloudVLMContentPart{
+		Type: "text",
+		Text: prompt,
+	})
+
+	for _, imgBytes := range imgBytesList {
+		if len(imgBytes) > 0 {
+			mimeType := detectImageMIME(imgBytes)
+			b64 := base64.StdEncoding.EncodeToString(imgBytes)
+			dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType, b64)
+			parts = append(parts, weKnoraCloudVLMContentPart{
+				Type: "image_url",
+				ImageURL: &weKnoraCloudVLMImageURL{
+					URL: dataURI,
+				},
+			})
+		}
+	}
+
+	reqBody := weKnoraCloudVLMRequest{
+		Model: v.effectiveModelName(),
+		Messages: []weKnoraCloudVLMMessage{
+			{
+				Role:    "user",
+				Content: parts,
+			},
+		},
+		MaxTokens:   defaultMaxToks,
+		Temperature: float64(defaultTemp),
+		Stream:      false,
+	}
+
+	bodyBytes, err := json.Marshal(reqBody)
+	if err != nil {
+		return "", fmt.Errorf("weknoracloud VLM: marshal: %w", err)
+	}
+
+	requestID := uuid.New().String()
+	headers := utils.Sign(v.appID, v.apiKey, requestID, string(bodyBytes))
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, v.baseURL+weKnoraCloudVLMPath, bytes.NewReader(bodyBytes))
+	if err != nil {
+		return "", fmt.Errorf("weknoracloud VLM: create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	for k, hv := range headers {
+		req.Header.Set(k, hv)
+	}
+
+	totalImageSize := 0
+	for _, img := range imgBytesList {
+		totalImageSize += len(img)
+	}
+	logger.Infof(ctx, "[VLM] Calling WeKnoraCloud API, model=%s, baseURL=%s, numImages=%d, totalImageSize=%d",
+		v.effectiveModelName(), v.baseURL, len(imgBytesList), totalImageSize)
+
+	resp, err := v.client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("weknoracloud VLM: do request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	respBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("weknoracloud VLM: read response: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("weknoracloud VLM: status %d: %s", resp.StatusCode, string(respBytes))
+	}
+
+	var vlmResp weKnoraCloudVLMResponse
+	if err := json.Unmarshal(respBytes, &vlmResp); err != nil {
+		return "", fmt.Errorf("weknoracloud VLM: unmarshal: %w", err)
+	}
+	if len(vlmResp.Choices) == 0 {
+		return "", fmt.Errorf("weknoracloud VLM: no choices in response")
+	}
+
+	content := vlmResp.Choices[0].Message.Content
+	logger.Infof(ctx, "[VLM] WeKnoraCloud response received, len=%d", len(content))
+	return content, nil
+}
+
+func (v *WeKnoraCloudVLM) effectiveModelName() string {
+	if v.remoteModelName != "" {
+		return v.remoteModelName
+	}
+	return v.modelName
+}
+
+func (v *WeKnoraCloudVLM) GetModelName() string { return v.modelName }
+func (v *WeKnoraCloudVLM) GetModelID() string   { return v.modelID }