WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/storageurl/mode.go

@@ -0,0 +1,156 @@
+package storageurl
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/Tencent/WeKnora/internal/logger"
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// Mode selects how stored files are referenced in an API response.
+type Mode string
+
+const (
+	// ModeHandle returns the internal `resource://<handle>` reference. Clients
+	// must fetch the bytes through the authenticated `/files` proxy. This is the
+	// default and the only mode that never depends on external reachability.
+	ModeHandle Mode = "handle"
+	// ModePublic returns a time-limited HTTP(S) URL the client can load
+	// directly, so third-party apps can render images without a second
+	// authenticated call. References that cannot be turned into an HTTP URL are
+	// left as handles.
+	ModePublic Mode = "public"
+)
+
+const (
+	// QueryParam is the request query parameter that selects the mode per call.
+	QueryParam = "resource_urls"
+	// EnvVar sets the deployment-wide default mode for requests that omit
+	// QueryParam. It sits alongside APP_EXTERNAL_URL, which is what actually
+	// makes `resource://` handles resolvable to a public `/r/<token>` URL.
+	EnvVar = "RESOURCE_URL_MODE"
+)
+
+// ErrPublicModeForbidden is returned when the caller asked for ModePublic but
+// its credentials must not receive anonymous, time-limited file URLs. Callers
+// map it to 403 rather than 400: the request is well-formed, the scope is not.
+var ErrPublicModeForbidden = errors.New(
+	"resource_urls=public is not available for a knowledge-base-restricted API key")
+
+// forcedHandleKey marks a request that must never receive public URLs, whatever
+// the query parameter or the deployment default says.
+type forcedHandleKey struct{}
+
+// WithForcedHandleMode pins a request to ModeHandle. Anonymous surfaces use it:
+// an embed visitor is authorized only by the channel's session handle, so it
+// must keep fetching images through the channel-scoped `/embed/…/files` proxy
+// instead of receiving a shareable, credential-free URL.
+func WithForcedHandleMode(ctx context.Context) context.Context {
+	return context.WithValue(ctx, forcedHandleKey{}, true)
+}
+
+// HandleModeForced reports whether ctx was pinned by WithForcedHandleMode.
+func HandleModeForced(ctx context.Context) bool {
+	if ctx == nil {
+		return false
+	}
+	forced, _ := ctx.Value(forcedHandleKey{}).(bool)
+	return forced
+}
+
+// ParseMode validates a mode supplied by a client or by configuration. An empty
+// value yields ModeHandle so an unset parameter or setting keeps the default.
+func ParseMode(raw string) (Mode, error) {
+	switch Mode(strings.ToLower(strings.TrimSpace(raw))) {
+	case "":
+		return ModeHandle, nil
+	case ModeHandle:
+		return ModeHandle, nil
+	case ModePublic:
+		return ModePublic, nil
+	default:
+		return ModeHandle, fmt.Errorf(
+			"invalid %s value %q: expected %q or %q", QueryParam, raw, ModeHandle, ModePublic)
+	}
+}
+
+// badDefault keeps a misconfigured EnvVar to a single log line per distinct bad
+// value instead of one per request, while still warning again if an operator
+// rolls out a second typo. The value itself is re-read every time so a fix takes
+// effect without a restart.
+var badDefault struct {
+	sync.Mutex
+	warned map[string]bool
+}
+
+func warnBadDefaultOnce(ctx context.Context, raw string, err error) {
+	badDefault.Lock()
+	defer badDefault.Unlock()
+	if badDefault.warned[raw] {
+		return
+	}
+	if badDefault.warned == nil {
+		badDefault.warned = make(map[string]bool)
+	}
+	badDefault.warned[raw] = true
+	logger.Warnf(ctx, "ignoring %s: %v; falling back to %q", EnvVar, err, ModeHandle)
+}
+
+// DefaultMode returns the deployment-wide default from EnvVar. An unset or
+// unparseable value yields ModeHandle, so a typo degrades to the safe default
+// rather than failing every request.
+func DefaultMode(ctx context.Context) Mode {
+	raw := strings.TrimSpace(os.Getenv(EnvVar))
+	if raw == "" {
+		return ModeHandle
+	}
+	mode, err := ParseMode(raw)
+	if err != nil {
+		warnBadDefaultOnce(ctx, raw, err)
+		return ModeHandle
+	}
+	return mode
+}
+
+// ResolveMode combines the per-request query value with the deployment default,
+// then applies the two limits that no caller may opt out of.
+//
+// An explicit query value wins over the deployment default; an invalid one is
+// returned as an error so integrators find their typo instead of silently
+// receiving handles.
+//
+// Limits:
+//   - A ctx pinned by WithForcedHandleMode (anonymous embed traffic) is silently
+//     downgraded rather than rejected, so an embed client that happens to send
+//     the parameter keeps working — it just never gets a public URL.
+//   - A knowledge-base-restricted API key is rejected with
+//     ErrPublicModeForbidden. Such a key is already denied the `/files` proxy,
+//     so handing it anonymous file URLs would widen its scope from "chunk text"
+//     to "file bytes" through the back door.
+func ResolveMode(ctx context.Context, queryValue string) (Mode, error) {
+	if HandleModeForced(ctx) {
+		return ModeHandle, nil
+	}
+	mode, err := requestedMode(ctx, queryValue)
+	if err != nil {
+		return ModeHandle, err
+	}
+	if mode == ModePublic {
+		if scope, ok := types.TenantAPIKeyScopeFromContext(ctx); ok && scope.IsKnowledgeBaseRestricted() {
+			return ModeHandle, ErrPublicModeForbidden
+		}
+	}
+	return mode, nil
+}
+
+func requestedMode(ctx context.Context, queryValue string) (Mode, error) {
+	if strings.TrimSpace(queryValue) != "" {
+		return ParseMode(queryValue)
+	}
+	return DefaultMode(ctx), nil
+}

internal/storageurl/request.go

@@ -0,0 +1,200 @@
+package storageurl
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// NewRequestRewriter builds the Rewriter for one API request or response stream.
+//
+// ModeHandle yields a disabled Rewriter, so the default path resolves nothing
+// and costs no access-grant rows. The tenant is taken from ctx because a
+// reference may live on a tenant-configured storage backend rather than the
+// process-wide default.
+//
+// No extra authorization gate applies here, unlike the `/files` proxy which
+// rejects KB-restricted API keys. That gate exists because `/files` takes an
+// arbitrary caller-supplied path it cannot bind to a KB allow-list. Here the
+// references come from a response the caller is already authorized to receive,
+// so the server — not the client — chooses which resources get a URL.
+func NewRequestRewriter(
+	ctx context.Context,
+	mode Mode,
+	defaultSvc interfaces.FileService,
+	storageResolver interfaces.StorageBackendResolver,
+) *Rewriter {
+	if mode != ModePublic {
+		return NewRewriter(nil, "API")
+	}
+	tenant, _ := types.TenantInfoFromContext(ctx)
+	var resolvers []interfaces.StorageBackendResolver
+	if storageResolver != nil {
+		resolvers = append(resolvers, storageResolver)
+	}
+	resolver := NewFileServiceResolver(tenant, defaultSvc, resolvers...).WithContext(ctx)
+	return NewRewriter(resolver, "API")
+}
+
+// RewriteMessagesResponse returns rewritten message copies so callers can keep
+// the originals untouched (for example objects returned from a service cache).
+func (w *Rewriter) RewriteMessagesResponse(ctx context.Context, messages []*types.Message) []*types.Message {
+	if !w.Enabled() || len(messages) == 0 {
+		return messages
+	}
+	out := cloneMessages(messages)
+	w.RewriteMessages(ctx, out)
+	return out
+}
+
+// cloneMessages deep-copies messages via JSON so nested slices and agent steps
+// are independent of the source slice.
+func cloneMessages(messages []*types.Message) []*types.Message {
+	data, err := json.Marshal(messages)
+	if err != nil {
+		return messages
+	}
+	var out []*types.Message
+	if err := json.Unmarshal(data, &out); err != nil {
+		return messages
+	}
+	return out
+}
+
+// RewriteMessages replaces storage references in a message history response so
+// clients receive loadable image URLs. Messages are mutated in place.
+func (w *Rewriter) RewriteMessages(ctx context.Context, messages []*types.Message) {
+	if !w.Enabled() {
+		return
+	}
+	for _, message := range messages {
+		if message == nil {
+			continue
+		}
+		message.Content = w.String(ctx, message.Content)
+		for i := range message.Images {
+			message.Images[i].URL = w.Ref(ctx, message.Images[i].URL)
+			message.Images[i].Caption = w.String(ctx, message.Images[i].Caption)
+		}
+		message.KnowledgeReferences = w.CopyReferences(ctx, message.KnowledgeReferences)
+		w.rewriteAgentSteps(ctx, message.AgentSteps)
+	}
+}
+
+// CopyReferences returns rewritten copies of retrieval results, covering both the
+// chunk text and the structured image_info payload.
+//
+// Copies rather than in-place edits because an SSE references payload shares its
+// *SearchResult pointers with the stream replay buffer and the assistant message
+// being persisted; rewriting those in place would corrupt both.
+func (w *Rewriter) CopyReferences(ctx context.Context, refs []*types.SearchResult) []*types.SearchResult {
+	if !w.Enabled() || refs == nil {
+		return refs
+	}
+	out := make([]*types.SearchResult, len(refs))
+	for i, ref := range refs {
+		if ref == nil {
+			continue
+		}
+		rewritten := *ref
+		rewritten.Content = w.String(ctx, ref.Content)
+		rewritten.MatchedContent = w.String(ctx, ref.MatchedContent)
+		rewritten.ImageInfo = w.String(ctx, ref.ImageInfo)
+		out[i] = &rewritten
+	}
+	return out
+}
+
+// CopyData returns a rewritten copy of an SSE metadata map, or data itself when
+// it holds no storage reference. Agent tool results put renderable Markdown into
+// this map, and its shape is tool-defined, so every string leaf is rewritten.
+func (w *Rewriter) CopyData(ctx context.Context, data map[string]interface{}) map[string]interface{} {
+	if !w.Enabled() || data == nil {
+		return data
+	}
+	rewritten, changed := w.copyValue(ctx, data, 0)
+	if !changed {
+		return data
+	}
+	out, _ := rewritten.(map[string]interface{})
+	return out
+}
+
+// maxDataDepth bounds recursion into tool-defined metadata. Renderable content
+// sits within a couple of levels; the cap only guards against a pathologically
+// nested payload.
+const maxDataDepth = 8
+
+func (w *Rewriter) copyValue(ctx context.Context, value interface{}, depth int) (interface{}, bool) {
+	if depth > maxDataDepth {
+		return value, false
+	}
+	switch typed := value.(type) {
+	case string:
+		out := w.String(ctx, typed)
+		return out, out != typed
+	// The references event carries its results twice: once in
+	// StreamResponse.KnowledgeReferences and once in Data. With an in-memory
+	// stream manager these arrive as the typed slice rather than the decoded
+	// []interface{} a Redis round-trip produces, so both forms need a case or
+	// the Data copy leaks handles the caller asked to have resolved.
+	case types.References:
+		return types.References(w.CopyReferences(ctx, typed)), true
+	case []*types.SearchResult:
+		return w.CopyReferences(ctx, typed), true
+	case []string:
+		out := make([]string, len(typed))
+		changed := false
+		for i, item := range typed {
+			out[i] = w.String(ctx, item)
+			changed = changed || out[i] != item
+		}
+		return out, changed
+	case map[string]string:
+		out := make(map[string]string, len(typed))
+		changed := false
+		for key, item := range typed {
+			out[key] = w.String(ctx, item)
+			changed = changed || out[key] != item
+		}
+		return out, changed
+	case map[string]interface{}:
+		out := make(map[string]interface{}, len(typed))
+		changed := false
+		for key, item := range typed {
+			converted, itemChanged := w.copyValue(ctx, item, depth+1)
+			out[key] = converted
+			changed = changed || itemChanged
+		}
+		return out, changed
+	case []interface{}:
+		out := make([]interface{}, len(typed))
+		changed := false
+		for i, item := range typed {
+			converted, itemChanged := w.copyValue(ctx, item, depth+1)
+			out[i] = converted
+			changed = changed || itemChanged
+		}
+		return out, changed
+	default:
+		return value, false
+	}
+}
+
+// rewriteAgentSteps covers the reasoning trace, whose tool output embeds Markdown
+// images for retrieved figures and generated charts.
+func (w *Rewriter) rewriteAgentSteps(ctx context.Context, steps types.AgentSteps) {
+	for i := range steps {
+		steps[i].Thought = w.String(ctx, steps[i].Thought)
+		steps[i].ReasoningContent = w.String(ctx, steps[i].ReasoningContent)
+		for j := range steps[i].ToolCalls {
+			call := &steps[i].ToolCalls[j]
+			call.Reflection = w.String(ctx, call.Reflection)
+			if call.Result != nil {
+				call.Result.Output = w.String(ctx, call.Result.Output)
+			}
+		}
+	}
+}

internal/storageurl/request_test.go

@@ -0,0 +1,263 @@
+package storageurl
+
+import (
+	"context"
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func publicRewriter(url string) *Rewriter {
+	return NewRewriter(stubResolver(url), "TEST")
+}
+
+func TestNewRequestRewriter_HandleModeIsDisabled(t *testing.T) {
+	w := NewRequestRewriter(context.Background(), ModeHandle, &stubFileService{}, nil)
+	assert.False(t, w.Enabled(), "the default mode must not resolve anything")
+}
+
+func TestNewRequestRewriter_PublicModeIsEnabled(t *testing.T) {
+	w := NewRequestRewriter(context.Background(), ModePublic, &stubFileService{}, nil)
+	require.True(t, w.Enabled())
+	assert.Equal(t, "https://cdn.example.com/resource://xifDo7NTSL300Lp1goVutw",
+		w.Ref(context.Background(), "resource://xifDo7NTSL300Lp1goVutw"))
+}
+
+func TestRewriteMessages(t *testing.T) {
+	w := publicRewriter("https://cdn.example.com/x.png")
+	ctx := context.Background()
+
+	messages := []*types.Message{
+		nil,
+		{
+			Content: "answer ![fig](resource://xifDo7NTSL300Lp1goVutw)",
+			Images: types.MessageImages{{
+				URL:     "resource://aaaabbbbccccddddeeeeff",
+				Caption: "shows ![inline](minio://bucket/10000/exports/a.png)",
+			}},
+			KnowledgeReferences: types.References{{
+				Content:   "chunk ![c](resource://xifDo7NTSL300Lp1goVutw)",
+				ImageInfo: `[{"url":"resource://xifDo7NTSL300Lp1goVutw"}]`,
+			}},
+			AgentSteps: types.AgentSteps{{
+				Thought: "looking at ![t](resource://xifDo7NTSL300Lp1goVutw)",
+				ToolCalls: []types.ToolCall{{
+					Reflection: "saw ![r](resource://xifDo7NTSL300Lp1goVutw)",
+					Result:     &types.ToolResult{Output: "chart ![o](resource://xifDo7NTSL300Lp1goVutw)"},
+				}},
+			}},
+		},
+	}
+
+	w.RewriteMessages(ctx, messages)
+
+	message := messages[1]
+	assert.Equal(t, "answer ![fig](https://cdn.example.com/x.png)", message.Content)
+	assert.Equal(t, "https://cdn.example.com/x.png", message.Images[0].URL)
+	assert.Equal(t, "shows ![inline](https://cdn.example.com/x.png)", message.Images[0].Caption)
+	assert.Equal(t, "chunk ![c](https://cdn.example.com/x.png)", message.KnowledgeReferences[0].Content)
+	assert.Equal(t, `[{"url":"https://cdn.example.com/x.png"}]`, message.KnowledgeReferences[0].ImageInfo)
+	assert.Equal(t, "looking at ![t](https://cdn.example.com/x.png)", message.AgentSteps[0].Thought)
+	assert.Equal(t, "saw ![r](https://cdn.example.com/x.png)", message.AgentSteps[0].ToolCalls[0].Reflection)
+	assert.Equal(t, "chart ![o](https://cdn.example.com/x.png)", message.AgentSteps[0].ToolCalls[0].Result.Output)
+}
+
+func TestRewriteMessages_DisabledLeavesHandles(t *testing.T) {
+	w := NewRewriter(nil, "TEST")
+	messages := []*types.Message{{Content: "![a](resource://xifDo7NTSL300Lp1goVutw)"}}
+	w.RewriteMessages(context.Background(), messages)
+	assert.Equal(t, "![a](resource://xifDo7NTSL300Lp1goVutw)", messages[0].Content)
+}
+
+func TestRewriteMessagesResponse_DoesNotMutateOriginals(t *testing.T) {
+	w := publicRewriter("https://cdn.example.com/x.png")
+	original := &types.Message{
+		Content: "![a](resource://xifDo7NTSL300Lp1goVutw)",
+		KnowledgeReferences: types.References{{
+			Content: "chunk ![c](resource://xifDo7NTSL300Lp1goVutw)",
+		}},
+	}
+	messages := []*types.Message{original}
+
+	out := w.RewriteMessagesResponse(context.Background(), messages)
+
+	require.Len(t, out, 1)
+	assert.NotSame(t, original, out[0])
+	assert.Equal(t, "![a](resource://xifDo7NTSL300Lp1goVutw)", original.Content)
+	assert.Equal(t, "![a](https://cdn.example.com/x.png)", out[0].Content)
+	assert.Equal(t, "chunk ![c](resource://xifDo7NTSL300Lp1goVutw)", original.KnowledgeReferences[0].Content)
+	assert.Equal(t, "chunk ![c](https://cdn.example.com/x.png)", out[0].KnowledgeReferences[0].Content)
+}
+
+// SSE references payloads share their *SearchResult pointers with the stream
+// replay buffer and the assistant message being persisted, so rewriting must not
+// mutate the originals.
+func TestCopyReferences_DoesNotMutateOriginals(t *testing.T) {
+	w := publicRewriter("https://cdn.example.com/x.png")
+	original := &types.SearchResult{
+		Content:        "chunk ![c](resource://xifDo7NTSL300Lp1goVutw)",
+		MatchedContent: "match ![m](resource://xifDo7NTSL300Lp1goVutw)",
+		ImageInfo:      `[{"url":"resource://xifDo7NTSL300Lp1goVutw"}]`,
+	}
+	refs := []*types.SearchResult{original, nil}
+
+	out := w.CopyReferences(context.Background(), refs)
+
+	require.Len(t, out, 2)
+	assert.NotSame(t, original, out[0])
+	assert.Equal(t, "chunk ![c](resource://xifDo7NTSL300Lp1goVutw)", original.Content,
+		"the replay buffer's copy must be untouched")
+	assert.Equal(t, "chunk ![c](https://cdn.example.com/x.png)", out[0].Content)
+	assert.Equal(t, "match ![m](https://cdn.example.com/x.png)", out[0].MatchedContent)
+	assert.Equal(t, `[{"url":"https://cdn.example.com/x.png"}]`, out[0].ImageInfo)
+	assert.Nil(t, out[1])
+}
+
+func TestCopyReferences_DisabledReturnsInput(t *testing.T) {
+	w := NewRewriter(nil, "TEST")
+	refs := []*types.SearchResult{{Content: "![a](resource://xifDo7NTSL300Lp1goVutw)"}}
+	assert.Equal(t, refs, w.CopyReferences(context.Background(), refs))
+}
+
+// Agent tool metadata is tool-defined, so every string leaf is rewritten — and
+// the source map, which the replay buffer also holds, must not be mutated.
+func TestCopyData_RewritesNestedStringsWithoutMutating(t *testing.T) {
+	w := publicRewriter("https://cdn.example.com/x.png")
+	data := map[string]interface{}{
+		"tool_name":   "chart_export",
+		"duration_ms": 42,
+		"output":      "![chart](resource://xifDo7NTSL300Lp1goVutw)",
+		"nested": map[string]interface{}{
+			"images": []interface{}{"resource://xifDo7NTSL300Lp1goVutw", "http://example.com/x.png"},
+		},
+	}
+
+	out := w.CopyData(context.Background(), data)
+
+	assert.Equal(t, "![chart](resource://xifDo7NTSL300Lp1goVutw)", data["output"],
+		"the replay buffer's map must be untouched")
+	assert.Equal(t, "![chart](https://cdn.example.com/x.png)", out["output"])
+	assert.Equal(t, "chart_export", out["tool_name"])
+	assert.Equal(t, 42, out["duration_ms"])
+
+	nested := out["nested"].(map[string]interface{})["images"].([]interface{})
+	assert.Equal(t, "https://cdn.example.com/x.png", nested[0])
+	assert.Equal(t, "http://example.com/x.png", nested[1])
+}
+
+// The references SSE event carries its results twice: in
+// StreamResponse.KnowledgeReferences and again in Data. An in-memory stream
+// manager keeps the typed slice, so CopyData must traverse it or the Data copy
+// leaks the handles the caller asked to have resolved.
+func TestCopyData_RewritesTypedReferenceSlices(t *testing.T) {
+	w := publicRewriter("https://cdn.example.com/x.png")
+	original := &types.SearchResult{Content: "figure ![f](resource://xifDo7NTSL300Lp1goVutw)"}
+	data := map[string]interface{}{
+		"references": types.References{original},
+		"tags":       []string{"resource://xifDo7NTSL300Lp1goVutw", "plain"},
+		"metadata":   map[string]string{"thumb": "resource://xifDo7NTSL300Lp1goVutw"},
+	}
+
+	out := w.CopyData(context.Background(), data)
+
+	refs := out["references"].(types.References)
+	assert.Equal(t, "figure ![f](https://cdn.example.com/x.png)", refs[0].Content)
+	assert.Equal(t, "figure ![f](resource://xifDo7NTSL300Lp1goVutw)", original.Content)
+	assert.Equal(t,
+		[]string{"https://cdn.example.com/x.png", "plain"},
+		out["tags"].([]string),
+	)
+	assert.Equal(t,
+		map[string]string{"thumb": "https://cdn.example.com/x.png"},
+		out["metadata"].(map[string]string),
+	)
+}
+
+// Copying every metadata map on every SSE event would be pure garbage, so an
+// unchanged map must be returned as-is.
+func TestCopyData_ReturnsInputWhenNothingChanges(t *testing.T) {
+	w := publicRewriter("https://cdn.example.com/x.png")
+	data := map[string]interface{}{"tool_name": "chart_export", "duration_ms": 42}
+	assert.Equal(t, data, w.CopyData(context.Background(), data))
+}
+
+func TestCopyData_NilAndDisabled(t *testing.T) {
+	assert.Nil(t, publicRewriter("https://x/y.png").CopyData(context.Background(), nil))
+	data := map[string]interface{}{"output": "![a](resource://xifDo7NTSL300Lp1goVutw)"}
+	assert.Equal(t, data, NewRewriter(nil, "TEST").CopyData(context.Background(), data))
+}
+
+func TestDefaultMode(t *testing.T) {
+	ctx := context.Background()
+	t.Setenv(EnvVar, "")
+	assert.Equal(t, ModeHandle, DefaultMode(ctx))
+
+	t.Setenv(EnvVar, "public")
+	assert.Equal(t, ModePublic, DefaultMode(ctx))
+
+	t.Setenv(EnvVar, "nonsense")
+	assert.Equal(t, ModeHandle, DefaultMode(ctx), "a typo must degrade to the safe default")
+}
+
+// Anonymous surfaces (embed channels) pin the mode: neither the query parameter
+// nor the deployment default may hand a visitor a credential-free URL. The
+// downgrade is silent so a client that forwards the parameter keeps working.
+func TestResolveMode_ForcedHandleModeWins(t *testing.T) {
+	t.Setenv(EnvVar, "public")
+	ctx := WithForcedHandleMode(context.Background())
+
+	for _, queryValue := range []string{"", "public", "handle", "nonsense"} {
+		mode, err := ResolveMode(ctx, queryValue)
+		require.NoError(t, err, "queryValue=%q", queryValue)
+		assert.Equal(t, ModeHandle, mode, "queryValue=%q", queryValue)
+	}
+}
+
+// A KB-restricted API key is denied the /files proxy, so it must not receive
+// anonymous file URLs through this parameter either.
+func TestResolveMode_RejectsPublicForKBRestrictedKey(t *testing.T) {
+	ctx := types.WithTenantAPIKeyScope(context.Background(), types.TenantAPIKeyScope{
+		KnowledgeBaseIDs: types.StringArray{"kb-1"},
+	})
+
+	_, err := ResolveMode(ctx, "public")
+	assert.ErrorIs(t, err, ErrPublicModeForbidden)
+
+	// The deployment default must not smuggle it in either.
+	t.Setenv(EnvVar, "public")
+	_, err = ResolveMode(ctx, "")
+	assert.ErrorIs(t, err, ErrPublicModeForbidden)
+
+	// The default mode stays available: only public URLs are off limits.
+	mode, err := ResolveMode(ctx, "handle")
+	require.NoError(t, err)
+	assert.Equal(t, ModeHandle, mode)
+}
+
+// A full-access or tenant-wide key is unaffected.
+func TestResolveMode_AllowsPublicForUnrestrictedKey(t *testing.T) {
+	ctx := types.WithTenantAPIKeyScope(context.Background(), types.TenantAPIKeyScope{
+		Capabilities: types.StringArray{string(types.APIKeyCapabilityRetrieve)},
+	})
+	mode, err := ResolveMode(ctx, "public")
+	require.NoError(t, err)
+	assert.Equal(t, ModePublic, mode)
+}
+
+func TestResolveMode_QueryWinsOverDeployment(t *testing.T) {
+	ctx := context.Background()
+	t.Setenv(EnvVar, "public")
+
+	mode, err := ResolveMode(ctx, "handle")
+	require.NoError(t, err)
+	assert.Equal(t, ModeHandle, mode, "an explicit query value must win")
+
+	mode, err = ResolveMode(ctx, "")
+	require.NoError(t, err)
+	assert.Equal(t, ModePublic, mode)
+
+	_, err = ResolveMode(ctx, "yes-please")
+	assert.Error(t, err, "an invalid query value is a client error")
+}

internal/storageurl/resolver.go

@@ -0,0 +1,130 @@
+package storageurl
+
+import (
+	"context"
+	"os"
+	"strings"
+
+	filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
+	"github.com/Tencent/WeKnora/internal/logger"
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// LocalStorageBaseDir is the on-disk root used when a reference resolves to the
+// local provider.
+func LocalStorageBaseDir() string {
+	baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
+	if baseDir == "" {
+		baseDir = "/data/files"
+	}
+	return baseDir
+}
+
+// FileServiceResolver resolves and caches one FileService per storage provider.
+// The cache is scoped to a single request or outbound message so a long answer
+// with many images does not re-create an SDK client per reference.
+//
+// Not safe for concurrent use; Rewriter drives it from one goroutine at a time.
+type FileServiceResolver struct {
+	tenant          *types.Tenant
+	defaultSvc      interfaces.FileService
+	storageResolver interfaces.StorageBackendResolver
+	ctx             context.Context
+	cache           map[string]interfaces.FileService
+}
+
+// NewFileServiceResolver builds a resolver for tenant. defaultSvc is the
+// process-wide FileService used for `resource://` handles and as the fallback
+// when tenant storage config is missing.
+func NewFileServiceResolver(
+	tenant *types.Tenant,
+	defaultSvc interfaces.FileService,
+	storageResolvers ...interfaces.StorageBackendResolver,
+) *FileServiceResolver {
+	resolver := &FileServiceResolver{
+		tenant:     tenant,
+		defaultSvc: defaultSvc,
+		ctx:        context.Background(),
+		cache:      make(map[string]interfaces.FileService),
+	}
+	if len(storageResolvers) > 0 {
+		resolver.storageResolver = storageResolvers[0]
+	}
+	return resolver
+}
+
+// WithContext sets the context used for backend lookups and their logs.
+func (r *FileServiceResolver) WithContext(ctx context.Context) *FileServiceResolver {
+	if ctx != nil {
+		r.ctx = ctx
+	}
+	return r
+}
+
+// ResolveFileService implements Resolver.
+func (r *FileServiceResolver) ResolveFileService(filePath string) interfaces.FileService {
+	if _, ok := types.ParseResourcePath(filePath); ok {
+		return r.defaultSvc
+	}
+	backendID, _, _ := types.ParseStorageBackendPath(filePath)
+	provider := types.ParseProviderScheme(filePath)
+	if provider == "" {
+		if r.tenant != nil && r.tenant.StorageEngineConfig != nil {
+			provider = strings.ToLower(strings.TrimSpace(r.tenant.StorageEngineConfig.DefaultProvider))
+		}
+		if provider == "" {
+			return nil
+		}
+	}
+	cacheKey := backendID + ":" + provider
+	if svc, ok := r.cache[cacheKey]; ok {
+		return svc
+	}
+	if r.storageResolver != nil && r.tenant != nil {
+		svc, _, err := r.storageResolver.ResolveFileService(r.ctx, r.tenant, backendID, provider, LocalStorageBaseDir())
+		if err == nil {
+			r.cache[cacheKey] = svc
+			return svc
+		}
+		logger.Warnf(r.ctx, "resolve storage backend failed: backend_id=%s provider=%s err=%v",
+			backendID, provider, err)
+	}
+	svc := BuildFileServiceForProvider(r.tenant, provider, r.defaultSvc)
+	if svc != nil {
+		r.cache[cacheKey] = svc
+	}
+	return svc
+}
+
+// BuildFileServiceForProvider selects the FileService for a storage provider.
+// The reference's own scheme wins over the tenant DefaultProvider. It falls back
+// to the process-wide default FileService (STORAGE_TYPE / env) when tenant
+// config is missing — mirrors ImageMultimodalService.resolveFileServiceForPayload
+// (issue #1282).
+func BuildFileServiceForProvider(
+	tenant *types.Tenant,
+	provider string,
+	defaultSvc interfaces.FileService,
+) interfaces.FileService {
+	baseDir := LocalStorageBaseDir()
+	var sec *types.StorageEngineConfig
+	if tenant != nil {
+		sec = tenant.StorageEngineConfig
+	}
+
+	svc, _, err := filesvc.NewFileServiceFromStorageConfig(provider, sec, baseDir)
+	if err == nil {
+		return svc
+	}
+	if provider == "local" {
+		externalURL := strings.TrimSpace(os.Getenv("APP_EXTERNAL_URL"))
+		return filesvc.NewLocalFileService(baseDir, externalURL)
+	}
+	if defaultSvc != nil {
+		return defaultSvc
+	}
+	return nil
+}
+
+var _ Resolver = (*FileServiceResolver)(nil)

internal/storageurl/storageurl.go

@@ -0,0 +1,161 @@
+// Package storageurl converts internal storage references into HTTP(S) URLs an
+// external client can load directly.
+//
+// WeKnora persists files behind two internal reference forms: the stable
+// `resource://<handle>` application identity and legacy/canonical provider
+// paths (`local://…`, `minio://…`, `storage://<backend-id>/cos://…`). Neither is
+// fetchable by a browser or a third-party app, which must otherwise call the
+// authenticated `/files` proxy for every image.
+//
+// This package is the single implementation of the "give me a loadable link"
+// translation. It is used by the IM channels (which have no way to attach
+// WeKnora credentials to an image fetch) and, opt-in, by the HTTP API so
+// integrators receive ready-to-render URLs.
+package storageurl
+
+import (
+	"context"
+	"regexp"
+	"strings"
+	"sync"
+
+	"github.com/Tencent/WeKnora/internal/logger"
+	"github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// Pattern matches every internal storage reference form: `resource://` handles,
+// legacy `provider://` paths, and canonical `storage://<backend-id>/provider://`
+// paths. The trailing character class stops at Markdown/HTML delimiters so a
+// reference inside `![alt](…)` or `src="…"` is matched without them.
+var Pattern = regexp.MustCompile(
+	`\b(?:resource://[0-9A-Za-z_-]+|` +
+		`(?:storage://[0-9A-Za-z_-]+/)?` +
+		`(?:local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]+)`,
+)
+
+// IsHTTPURL reports whether s is an http(s) URL — the only form an external
+// client can fetch; any provider scheme (oss://, local://, …) is not. Scheme
+// match is case-insensitive per RFC 3986 §3.1: a backend may emit an
+// operator-configured host (e.g. OBS_PROXY_DOMAIN) with an uppercase scheme.
+func IsHTTPURL(s string) bool {
+	return len(s) >= 7 && strings.EqualFold(s[:7], "http://") ||
+		len(s) >= 8 && strings.EqualFold(s[:8], "https://")
+}
+
+// Resolver maps one storage reference to the FileService that owns it. A
+// reference carries its own provider scheme and optional storage backend id, so
+// a single answer may span several backends.
+type Resolver interface {
+	// ResolveFileService returns nil when no backend can serve ref.
+	ResolveFileService(ref string) interfaces.FileService
+}
+
+// Rewriter replaces storage references with loadable HTTP URLs.
+//
+// Resolutions are memoised for the Rewriter's lifetime. Create one per request
+// or per outbound message: a `resource://` handle costs one access-grant row per
+// resolution, and a streamed answer repeats the same image across many chunks.
+//
+// Safe for concurrent use: mu covers both the memo and the resolver, which is
+// itself single-threaded. Resolution therefore serialises across goroutines,
+// which is what we want — two chunks naming the same image must not each pay for
+// a signature.
+//
+// Logging policy:
+//   - A successful rewrite logs the source reference at INFO. The signed URL
+//     is DEBUG-only so log aggregation cannot hand out anonymously readable
+//     links; operators can raise the log level when verifying reachability.
+//   - Failure or no-op logs at WARN. The no-op case usually means
+//     APP_EXTERNAL_URL is unset, the most common cause of "image broken in the
+//     IM channel / in my app" reports.
+type Rewriter struct {
+	resolver  Resolver
+	logPrefix string
+
+	mu   sync.Mutex
+	memo map[string]string
+}
+
+// NewRewriter returns a Rewriter that resolves references through r. logPrefix
+// tags log lines with the calling surface (for example "IM" or "API"). A nil
+// resolver yields a Rewriter that leaves content unchanged.
+func NewRewriter(r Resolver, logPrefix string) *Rewriter {
+	return &Rewriter{resolver: r, logPrefix: logPrefix, memo: make(map[string]string)}
+}
+
+// Enabled reports whether this Rewriter can actually rewrite anything.
+func (w *Rewriter) Enabled() bool {
+	return w != nil && w.resolver != nil
+}
+
+// String replaces every storage reference in content with an HTTP URL.
+// References that are already HTTP, that no backend claims, or that resolve to
+// a non-HTTP location are left untouched so the caller degrades to the
+// authenticated file proxy rather than emitting an unfetchable URL.
+func (w *Rewriter) String(ctx context.Context, content string) string {
+	if !w.Enabled() || content == "" {
+		return content
+	}
+	return Pattern.ReplaceAllStringFunc(content, func(ref string) string {
+		return w.ref(ctx, ref)
+	})
+}
+
+// Ref rewrites a value that is a bare storage reference rather than prose
+// containing one — for example `MessageImage.URL`.
+func (w *Rewriter) Ref(ctx context.Context, ref string) string {
+	if !w.Enabled() || ref == "" {
+		return ref
+	}
+	if !Pattern.MatchString(ref) {
+		return ref
+	}
+	return w.String(ctx, ref)
+}
+
+// ref resolves one reference. The lock is held across resolve because Resolver
+// implementations (FileServiceResolver in particular) keep an unsynchronised
+// per-provider cache, and because it collapses a concurrent duplicate into one
+// signature instead of two.
+func (w *Rewriter) ref(ctx context.Context, ref string) string {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if cached, ok := w.memo[ref]; ok {
+		return cached
+	}
+	resolved := w.resolve(ctx, ref)
+	w.memo[ref] = resolved
+	return resolved
+}
+
+func (w *Rewriter) resolve(ctx context.Context, ref string) string {
+	fileSvc := w.resolver.ResolveFileService(ref)
+	if fileSvc == nil {
+		logger.Warnf(ctx, "[%s] storage URL rewrite: no file service for src=%s", w.logPrefix, ref)
+		return ref
+	}
+	httpURL, err := fileSvc.GetFileURL(ctx, ref)
+	if err != nil {
+		logger.Warnf(ctx, "[%s] storage URL rewrite failed: src=%s err=%v", w.logPrefix, ref, err)
+		return ref
+	}
+	// A non-http(s) result cannot be fetched by the client — this covers both an
+	// unchanged no-op and a resource:// alias left as an internal storage://
+	// path (APP_EXTERNAL_URL unset / nginx not proxying /r/).
+	if !IsHTTPURL(httpURL) {
+		logger.Warnf(ctx,
+			"[%s] storage URL rewrite no-op (resolved to non-HTTP URL %q; for local/private storage set "+
+				"APP_EXTERNAL_URL and ensure nginx proxies /r/): src=%s",
+			w.logPrefix, httpURL, ref)
+		return ref
+	}
+	logger.Infof(ctx, "[%s] storage URL rewrite: src=%s", w.logPrefix, ref)
+	logger.Debugf(ctx, "[%s] storage URL rewrite dst=%s", w.logPrefix, httpURL)
+	return httpURL
+}
+
+// Rewrite is the one-shot form of Rewriter.String for callers that translate a
+// single self-contained message.
+func Rewrite(ctx context.Context, content string, r Resolver, logPrefix string) string {
+	return NewRewriter(r, logPrefix).String(ctx, content)
+}

internal/storageurl/storageurl_test.go

@@ -0,0 +1,184 @@
+package storageurl
+
+import (
+	"context"
+	"errors"
+	"io"
+	"mime/multipart"
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types/interfaces"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// stubFileService implements interfaces.FileService; only GetFileURL matters here.
+type stubFileService struct {
+	getFileURL func(ctx context.Context, filePath string) (string, error)
+	calls      int
+}
+
+func (s *stubFileService) CheckConnectivity(context.Context) error { return nil }
+
+func (s *stubFileService) SaveFile(context.Context, *multipart.FileHeader, uint64, string) (string, error) {
+	return "", nil
+}
+
+func (s *stubFileService) SaveBytes(context.Context, []byte, uint64, string, bool) (string, error) {
+	return "", nil
+}
+
+func (s *stubFileService) GetFile(context.Context, string) (io.ReadCloser, error) { return nil, nil }
+
+func (s *stubFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
+	s.calls++
+	if s.getFileURL != nil {
+		return s.getFileURL(ctx, filePath)
+	}
+	return "https://cdn.example.com/" + filePath, nil
+}
+
+func (s *stubFileService) DeleteFile(context.Context, string) error { return nil }
+
+func (s *stubFileService) CopyFile(context.Context, string, uint64, string) (string, error) {
+	return "", nil
+}
+
+// fixedResolver returns the same FileService for every reference.
+type fixedResolver struct{ svc interfaces.FileService }
+
+func (r fixedResolver) ResolveFileService(string) interfaces.FileService { return r.svc }
+
+func stubResolver(url string) Resolver {
+	return fixedResolver{svc: &stubFileService{
+		getFileURL: func(context.Context, string) (string, error) { return url, nil },
+	}}
+}
+
+func TestRewriter_RewritesEveryReferenceForm(t *testing.T) {
+	svc := &stubFileService{
+		getFileURL: func(context.Context, string) (string, error) {
+			return "https://cdn.example.com/signed.png", nil
+		},
+	}
+	w := NewRewriter(fixedResolver{svc: svc}, "TEST")
+
+	in := "handle ![a](resource://xifDo7NTSL300Lp1goVutw) " +
+		"legacy ![b](minio://bucket/10000/exports/b.png) " +
+		"scoped ![c](storage://backend-a/cos://bucket/ap/10000/exports/c.png)"
+	out := w.String(context.Background(), in)
+
+	assert.NotContains(t, out, "resource://")
+	assert.NotContains(t, out, "minio://")
+	assert.NotContains(t, out, "storage://")
+	assert.Equal(t, 3, svc.calls)
+}
+
+// An already-public URL in the answer must be left alone.
+func TestRewriter_LeavesHTTPURLsAlone(t *testing.T) {
+	w := NewRewriter(stubResolver("https://cdn.example.com/x.png"), "TEST")
+	in := "![a](https://example.com/a.png) and ![b](http://example.com/b.png)"
+	assert.Equal(t, in, w.String(context.Background(), in))
+}
+
+// Emitting an unfetchable URL is worse than leaving the handle: the client can
+// still fall back to the authenticated /files proxy for a handle.
+func TestRewriter_NonHTTPResultIsNoOp(t *testing.T) {
+	w := NewRewriter(stubResolver("storage://7cb970a6/oss://bucket/10000/exports/a.png"), "TEST")
+	in := "![img](resource://xifDo7NTSL300Lp1goVutw)"
+	assert.Equal(t, in, w.String(context.Background(), in))
+}
+
+func TestRewriter_ResolveFailureIsNoOp(t *testing.T) {
+	w := NewRewriter(fixedResolver{svc: &stubFileService{
+		getFileURL: func(context.Context, string) (string, error) {
+			return "", errors.New("backend unreachable")
+		},
+	}}, "TEST")
+	in := "![img](resource://xifDo7NTSL300Lp1goVutw)"
+	assert.Equal(t, in, w.String(context.Background(), in))
+}
+
+func TestRewriter_UnknownBackendIsNoOp(t *testing.T) {
+	w := NewRewriter(fixedResolver{svc: nil}, "TEST")
+	in := "![img](resource://xifDo7NTSL300Lp1goVutw)"
+	assert.Equal(t, in, w.String(context.Background(), in))
+}
+
+// Uppercase schemes are valid per RFC 3986 §3.1 (e.g. an OBS_PROXY_DOMAIN
+// configured as HTTPS://…) and must be substituted, not dropped.
+func TestRewriter_UppercaseSchemeIsSubstituted(t *testing.T) {
+	w := NewRewriter(stubResolver("HTTPS://cdn.example.com/x.png"), "TEST")
+	out := w.String(context.Background(), "![img](resource://xifDo7NTSL300Lp1goVutw)")
+	assert.Contains(t, out, "HTTPS://cdn.example.com/x.png")
+	assert.NotContains(t, out, "resource://")
+}
+
+// Each resource:// resolution writes an access-grant row, so a repeated image
+// must be resolved once per request.
+func TestRewriter_MemoisesRepeatedReferences(t *testing.T) {
+	svc := &stubFileService{}
+	w := NewRewriter(fixedResolver{svc: svc}, "TEST")
+	ctx := context.Background()
+
+	ref := "resource://xifDo7NTSL300Lp1goVutw"
+	first := w.String(ctx, "![a]("+ref+")")
+	second := w.String(ctx, "![b]("+ref+")")
+
+	assert.Equal(t, 1, svc.calls, "the same reference must resolve once per Rewriter")
+	assert.Equal(t, "https://cdn.example.com/"+ref, first[5:len(first)-1])
+	assert.Contains(t, second, "https://cdn.example.com/"+ref)
+}
+
+func TestRewriter_DisabledWithoutResolver(t *testing.T) {
+	w := NewRewriter(nil, "TEST")
+	in := "![img](resource://xifDo7NTSL300Lp1goVutw)"
+	assert.False(t, w.Enabled())
+	assert.Equal(t, in, w.String(context.Background(), in))
+	assert.Equal(t, in, w.Ref(context.Background(), in))
+}
+
+// Ref handles a bare reference such as MessageImage.URL, and must not touch a
+// value that is not a reference at all.
+func TestRewriter_Ref(t *testing.T) {
+	w := NewRewriter(stubResolver("https://cdn.example.com/x.png"), "TEST")
+	ctx := context.Background()
+
+	assert.Equal(t, "https://cdn.example.com/x.png",
+		w.Ref(ctx, "resource://xifDo7NTSL300Lp1goVutw"))
+	assert.Equal(t, "", w.Ref(ctx, ""))
+	assert.Equal(t, "data:image/png;base64,AAAA", w.Ref(ctx, "data:image/png;base64,AAAA"))
+}
+
+func TestIsHTTPURL(t *testing.T) {
+	for _, s := range []string{"http://a", "https://a", "HTTP://a", "HTTPS://a"} {
+		assert.True(t, IsHTTPURL(s), s)
+	}
+	for _, s := range []string{"", "ftp://a", "resource://abc", "local://1/a.png", "http:/"} {
+		assert.False(t, IsHTTPURL(s), s)
+	}
+}
+
+func TestParseMode(t *testing.T) {
+	tests := []struct {
+		in      string
+		want    Mode
+		wantErr bool
+	}{
+		{"", ModeHandle, false},
+		{"handle", ModeHandle, false},
+		{"public", ModePublic, false},
+		{"  PUBLIC ", ModePublic, false},
+		{"true", ModeHandle, true},
+		{"signed", ModeHandle, true},
+	}
+	for _, tt := range tests {
+		got, err := ParseMode(tt.in)
+		if tt.wantErr {
+			require.Error(t, err, "ParseMode(%q)", tt.in)
+		} else {
+			require.NoError(t, err, "ParseMode(%q)", tt.in)
+		}
+		assert.Equal(t, tt.want, got, "ParseMode(%q)", tt.in)
+	}
+}

internal/storageurl/stream.go

@@ -0,0 +1,203 @@
+package storageurl
+
+import (
+	"context"
+	"regexp"
+	"strings"
+	"sync"
+	"unicode/utf8"
+)
+
+// ── Incomplete-reference detection ──
+//
+// Content is delivered to clients in chunks (SSE answer deltas, or the IM
+// channel's 300ms flush batches). A storage reference may be split across two
+// chunks, so a rewrite that only sees one chunk would leave a broken fragment.
+// These helpers locate an incomplete pattern at the tail of a chunk so the
+// caller can hold it back until the next chunk completes it.
+
+// incompleteRefSuffixRe matches a storage reference that reaches the end of the
+// string — it may continue in the next chunk.
+var incompleteRefSuffixRe = regexp.MustCompile(
+	`\b(?:resource|storage|local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]*$`,
+)
+
+// FindIncompleteRef returns the byte offset of a potentially truncated storage
+// reference at the tail of s, or -1 if none.
+func FindIncompleteRef(s string) int {
+	loc := incompleteRefSuffixRe.FindStringIndex(s)
+	if loc == nil {
+		return -1
+	}
+	return loc[0]
+}
+
+// incompleteMarkdownImageSuffixRe matches a Markdown image whose destination URL
+// (the parenthesized part) is not yet closed — e.g. "![alt](minio://part" or
+// "![alt](". Holding back only from "minio://" would flush "![alt](" to the
+// client and break the image once the URL arrives in the next chunk.
+//
+// The destination must be whitespace-free, and FindIncompleteMarkdownImage caps
+// its length: a URL contains neither whitespace nor a newline, and even a
+// presigned URL stays well under the cap. Without those limits, prose that
+// merely mentions an unclosed "…](" (an answer explaining Markdown syntax, or a
+// code sample) would look forever-unclosed and stall the rest of the stream.
+var incompleteMarkdownImageSuffixRe = regexp.MustCompile(`!\[[^\]]*\]\([^)\s]*$`)
+
+// maxIncompleteImageBytes bounds how much trailing text may be treated as an
+// unfinished Markdown image. Beyond it the text is not plausibly a link, so it
+// is flushed instead of buffered.
+const maxIncompleteImageBytes = 2048
+
+// FindIncompleteMarkdownImage returns the byte offset of an unclosed
+// `![alt](url` suffix at the end of s, or -1 if none.
+func FindIncompleteMarkdownImage(s string) int {
+	// Prefer pairing a trailing reference fragment with the nearest preceding
+	// `![…](` so alt text may itself contain ']' (e.g. `![a[b]](minio://part`).
+	if urlIdx := FindIncompleteRef(s); urlIdx >= 0 {
+		if imgIdx := strings.LastIndex(s[:urlIdx], "!["); imgIdx >= 0 {
+			if strings.Contains(s[imgIdx:urlIdx], "](") {
+				return imgIdx
+			}
+		}
+	}
+	loc := incompleteMarkdownImageSuffixRe.FindStringIndex(s)
+	if loc == nil || len(s)-loc[0] > maxIncompleteImageBytes {
+		return -1
+	}
+	return loc[0]
+}
+
+// HoldbackCutoff returns the offset at which chunk stops being safe to flush,
+// or len(chunk) when the whole chunk can be emitted.
+func HoldbackCutoff(chunk string) int {
+	cutoff := len(chunk)
+	if idx := FindIncompleteMarkdownImage(chunk); idx >= 0 && idx < cutoff {
+		return idx
+	}
+	if idx := FindIncompleteRef(chunk); idx >= 0 && idx < cutoff {
+		return idx
+	}
+	return cutoff
+}
+
+// maxHeldBytes bounds the per-key holdback buffer. A storage reference plus its
+// Markdown alt text is far shorter than this; the cap only stops a pathological
+// stream (for example an unclosed "![" that never terminates) from buffering an
+// entire answer.
+const maxHeldBytes = 4096
+
+// StreamRewriter rewrites storage references in a stream of content deltas.
+//
+// Each logical stream is identified by a key (WeKnora uses the SSE event id, the
+// same key clients accumulate on). Push returns only the prefix that is safe to
+// emit now, retaining any tail that may be an incomplete reference until the
+// next Push for that key.
+//
+// Safe for concurrent use.
+type StreamRewriter struct {
+	rewriter *Rewriter
+
+	mu   sync.Mutex
+	held map[string]heldContent
+}
+
+// heldContent is a retained tail plus the metadata of the event it came from, so
+// a tail released after its stream ended can be re-emitted as an equivalent
+// event rather than a bare fragment.
+type heldContent struct {
+	content string
+	meta    interface{}
+}
+
+// Held is one released tail: the rewritten content and the metadata carried by
+// the last Push that contributed to it.
+type Held struct {
+	Content string
+	Meta    interface{}
+}
+
+// NewStreamRewriter wraps rewriter with per-stream holdback state.
+func NewStreamRewriter(rewriter *Rewriter) *StreamRewriter {
+	return &StreamRewriter{rewriter: rewriter, held: make(map[string]heldContent)}
+}
+
+// Enabled reports whether this StreamRewriter can rewrite anything.
+func (s *StreamRewriter) Enabled() bool {
+	return s != nil && s.rewriter.Enabled()
+}
+
+// Rewriter exposes the underlying Rewriter for stream fields that arrive whole
+// (references, metadata) and therefore need no holdback.
+func (s *StreamRewriter) Rewriter() *Rewriter {
+	if s == nil {
+		return nil
+	}
+	return s.rewriter
+}
+
+// Push feeds the next chunk of the stream identified by key and returns the
+// rewritten content that is ready to emit. Set flush on the stream's terminal
+// chunk to release any held tail. meta is retained opaquely alongside the tail
+// and handed back by FlushAll so a late release can carry the same metadata as
+// the event it was cut from.
+func (s *StreamRewriter) Push(ctx context.Context, key, chunk string, flush bool, meta interface{}) string {
+	if !s.Enabled() {
+		return chunk
+	}
+
+	s.mu.Lock()
+	pending := s.held[key].content + chunk
+	cutoff := len(pending)
+	if !flush {
+		cutoff = HoldbackCutoff(pending)
+		// Never buffer without bound: release the excess even though it may
+		// contain a partial reference, which is what an un-rewritten stream
+		// would have shown anyway.
+		if len(pending)-cutoff > maxHeldBytes {
+			cutoff = runeStart(pending, len(pending)-maxHeldBytes)
+		}
+	}
+	emit := pending[:cutoff]
+	if remainder := pending[cutoff:]; remainder == "" {
+		delete(s.held, key)
+	} else {
+		s.held[key] = heldContent{content: remainder, meta: meta}
+	}
+	s.mu.Unlock()
+
+	return s.rewriter.String(ctx, emit)
+}
+
+// runeStart moves idx back to the nearest UTF-8 sequence boundary so a byte-based
+// cut never splits a character in half. Pattern-derived cutoffs already land on
+// ASCII delimiters; this only matters for the maxHeldBytes safety valve.
+func runeStart(s string, idx int) int {
+	for idx > 0 && !utf8.RuneStart(s[idx]) {
+		idx--
+	}
+	return idx
+}
+
+// FlushAll releases every held tail, rewritten, keyed by stream. Callers use it
+// when a stream ends without a terminal chunk (for example a client
+// disconnect) so buffered content is not silently dropped.
+func (s *StreamRewriter) FlushAll(ctx context.Context) map[string]Held {
+	if !s.Enabled() {
+		return nil
+	}
+	s.mu.Lock()
+	if len(s.held) == 0 {
+		s.mu.Unlock()
+		return nil
+	}
+	pending := s.held
+	s.held = make(map[string]heldContent)
+	s.mu.Unlock()
+
+	out := make(map[string]Held, len(pending))
+	for key, held := range pending {
+		out[key] = Held{Content: s.rewriter.String(ctx, held.content), Meta: held.meta}
+	}
+	return out
+}

internal/storageurl/stream_test.go

@@ -0,0 +1,227 @@
+package storageurl
+
+import (
+	"context"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"unicode/utf8"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFindIncompleteRef(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want int // expected return; -1 means no match expected
+	}{
+		{
+			"complete URL terminated by )",
+			"![img](local://1/abc/img.png)",
+			// The URL `local://1/abc/img.png` ends with `)` which is a terminator,
+			// but the regex [^\s)\]>"]* matches up to `)` — the `)` is NOT included.
+			// So the URL portion is `local://1/abc/img.png` and `)` terminates it.
+			// The match does NOT reach end of string → should return -1.
+			-1,
+		},
+		{
+			"complete URL terminated by space",
+			"text local://1/abc/img.png more text",
+			-1,
+		},
+		{
+			"truncated URL at end",
+			"text ![img](local://1/abc/im",
+			12, // starts at `l` in `local://`
+		},
+		{
+			"just scheme at end",
+			"text minio://",
+			5,
+		},
+		{
+			"partial scoped URL at end",
+			"text storage://backend-a/co",
+			5,
+		},
+		{
+			"no storage URL",
+			"just plain text http://example.com",
+			-1,
+		},
+		{
+			"URL at very end",
+			"local://1/img.png",
+			0,
+		},
+		{
+			"truncated resource handle at end",
+			"see ![img](resource://xifDo7NTSL",
+			11,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, FindIncompleteRef(tt.in), "FindIncompleteRef(%q)", tt.in)
+		})
+	}
+}
+
+func TestFindIncompleteMarkdownImage(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want int
+	}{
+		{"complete image", "![img](local://1/a.png)", -1},
+		{"complete then text", "![img](local://1/a.png) trailing", -1},
+		{"truncated provider URL in image", `![知识助理"知识库"管理视图界面](minio://wizard-test/10000/exports/c91cf852`, 0},
+		{"open paren only", "text ![alt](", 5},
+		{"bare provider suffix without markdown", "text minio://wizard-test/10000/exp", -1},
+		{"two images complete", "![a](local://1/a.png) ![b](local://1/b.png)", -1},
+		{"first complete second incomplete", "![a](local://1/a.png) ![b](minio://part", 22},
+		{"bracket inside alt text", "![a[b]](minio://wizard-test/10000/part", 0},
+		{"destination with whitespace is prose, not a link", "![alt](see the figure below", -1},
+		{
+			"destination too long to be a link",
+			"![alt](" + strings.Repeat("x", maxIncompleteImageBytes),
+			-1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, FindIncompleteMarkdownImage(tt.in), "FindIncompleteMarkdownImage(%q)", tt.in)
+		})
+	}
+}
+
+func TestHoldbackCutoff(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want int // -1 means "expect len(in)", i.e. no holdback
+	}{
+		{"no holdback needed", "plain text with complete ![img](local://1/img.png) content", -1},
+		{"truncated URL inside markdown image", "text ![img](local://1/abc/im", 5},
+		{"bare truncated reference", "text local://1/abc/im", 5},
+		{"unopened image destination", "prefix ![alt](", 7},
+		{"empty chunk", "", -1},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			want := tt.want
+			if want == -1 {
+				want = len(tt.in)
+			}
+			assert.Equal(t, want, HoldbackCutoff(tt.in), "HoldbackCutoff(%q)", tt.in)
+		})
+	}
+}
+
+// A reference split across two deltas must be held back and rewritten once
+// complete, never emitted as a broken fragment.
+func TestStreamRewriter_HoldsSplitReference(t *testing.T) {
+	sr := NewStreamRewriter(NewRewriter(stubResolver("https://cdn.example.com/x.png"), "TEST"))
+	ctx := context.Background()
+
+	first := sr.Push(ctx, "answer-1", "here it is: ![img](resource://xifDo7", false, nil)
+	assert.Equal(t, "here it is: ", first, "the incomplete image must be held back")
+
+	second := sr.Push(ctx, "answer-1", "NTSL300Lp1goVutw) done", false, nil)
+	assert.Equal(t, "![img](https://cdn.example.com/x.png) done", second)
+
+	assert.Empty(t, sr.Push(ctx, "answer-1", "", true, nil))
+	assert.Empty(t, sr.FlushAll(ctx), "nothing should remain held")
+}
+
+// Streams are keyed independently so interleaved events do not corrupt each
+// other's holdback buffers.
+func TestStreamRewriter_KeysAreIndependent(t *testing.T) {
+	sr := NewStreamRewriter(NewRewriter(stubResolver("https://cdn.example.com/x.png"), "TEST"))
+	ctx := context.Background()
+
+	assert.Empty(t, sr.Push(ctx, "a", "![x](resource://aaaa", false, nil))
+	assert.Equal(t, "plain b", sr.Push(ctx, "b", "plain b", false, nil))
+	assert.Equal(t,
+		"![x](https://cdn.example.com/x.png)",
+		sr.Push(ctx, "a", "bbbbccccdddddd)", false, nil),
+	)
+}
+
+// A stream that ends without a terminal chunk must not silently drop the tail.
+func TestStreamRewriter_FlushAllReleasesHeldTail(t *testing.T) {
+	sr := NewStreamRewriter(NewRewriter(stubResolver("https://cdn.example.com/x.png"), "TEST"))
+	ctx := context.Background()
+
+	meta := map[string]interface{}{"event_id": "answer-1", "is_fallback": true}
+	assert.Equal(t, "tail ",
+		sr.Push(ctx, "answer-1", "tail ![img](resource://partial", false, meta))
+	assert.Equal(t,
+		map[string]Held{"answer-1": {
+			Content: "![img](https://cdn.example.com/x.png",
+			Meta:    meta,
+		}},
+		sr.FlushAll(ctx),
+		"the held tail must still reach the client, rewritten, with its metadata",
+	)
+}
+
+// Text that merely looks like an unfinished image must not stall the stream:
+// prose can contain a literal "](", and a URL never runs this long.
+func TestStreamRewriter_LongUnclosedImageIsNotHeld(t *testing.T) {
+	sr := NewStreamRewriter(NewRewriter(stubResolver("https://cdn.example.com/x.png"), "TEST"))
+	chunk := "![never closed](" + strings.Repeat("x", maxIncompleteImageBytes)
+	assert.Equal(t, chunk, sr.Push(context.Background(), "answer-1", chunk, false, nil))
+}
+
+// Holdback must be bounded so a stream that never terminates a reference cannot
+// buffer the whole answer, and the byte-based release must not split a rune.
+func TestStreamRewriter_HoldbackIsBounded(t *testing.T) {
+	sr := NewStreamRewriter(NewRewriter(stubResolver("https://cdn.example.com/x.png"), "TEST"))
+	ctx := context.Background()
+
+	var emitted strings.Builder
+	emitted.WriteString(sr.Push(ctx, "answer-1", "开头 resource://", false, nil))
+	for i := 0; i < 20; i++ {
+		emitted.WriteString(sr.Push(ctx, "answer-1", strings.Repeat("中", 1024), false, nil))
+	}
+	assert.Greater(t, emitted.Len(), 0, "bounded holdback must release the excess")
+	assert.LessOrEqual(t, len(sr.held["answer-1"].content), maxHeldBytes)
+	assert.True(t, utf8.ValidString(emitted.String()), "the release must not split a rune")
+	assert.True(t, utf8.ValidString(sr.held["answer-1"].content), "the retained tail must stay valid")
+}
+
+// A disabled rewriter is a pass-through: the default API mode must not add
+// latency or buffering.
+func TestStreamRewriter_DisabledIsPassThrough(t *testing.T) {
+	sr := NewStreamRewriter(NewRewriter(nil, "TEST"))
+	in := "![img](resource://xifDo7"
+	assert.Equal(t, in, sr.Push(context.Background(), "answer-1", in, false, nil))
+	assert.False(t, sr.Enabled())
+}
+
+// Two goroutines pushing different streams share one resolver, so resolution
+// must be serialised — this fails under -race if it is not.
+func TestStreamRewriter_ConcurrentPushIsSafe(t *testing.T) {
+	sr := NewStreamRewriter(NewRewriter(stubResolver("https://cdn.example.com/x.png"), "TEST"))
+	ctx := context.Background()
+
+	var wg sync.WaitGroup
+	for i := 0; i < 8; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			key := "answer-" + strconv.Itoa(i)
+			for j := 0; j < 20; j++ {
+				// A distinct reference per push so every call really reaches the
+				// resolver instead of hitting the memo.
+				ref := "minio://bucket/10000/" + strconv.Itoa(i) + "-" + strconv.Itoa(j) + ".png"
+				sr.Push(ctx, key, "![x]("+ref+") ", false, nil)
+			}
+		}(i)
+	}
+	wg.Wait()
+	assert.Empty(t, sr.FlushAll(ctx))
+}