WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/searchutil/chunkmerge.go

@@ -0,0 +1,266 @@
+package searchutil
+
+import (
+	"sort"
+	"strings"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// JoinChunkContent joins two current chunk bodies without relying on parser
+// offsets. Exact containment is collapsed, a real suffix/prefix overlap is
+// removed, and otherwise both bodies are retained with separator between
+// them. The conservative fallback intentionally prefers small duplication
+// over silently dropping edited content.
+func JoinChunkContent(acc, next, separator string) string {
+	if acc == "" {
+		return next
+	}
+	if next == "" {
+		return acc
+	}
+	if ContainsChunkContent(acc, next) {
+		return acc
+	}
+	if ContainsChunkContent(next, acc) {
+		return next
+	}
+
+	accRunes := []rune(acc)
+	nextRunes := []rune(next)
+	maxOverlap := minInt(len(accRunes), len(nextRunes))
+	// Editable chunks may be much larger than parser-produced chunks. Bound
+	// suffix matching so an adversarial 200 KB edit cannot turn retrieval into
+	// quadratic work. Parser overlap windows are normally far below this cap;
+	// larger unmatched overlap is safely retained as duplication.
+	if maxOverlap > defaultSearchSpan {
+		maxOverlap = defaultSearchSpan
+	}
+	for overlap := maxOverlap; overlap >= minOverlapRunes; overlap-- {
+		if runeSlicesEqual(accRunes[len(accRunes)-overlap:], nextRunes[:overlap]) {
+			return acc + string(nextRunes[overlap:])
+		}
+	}
+	return acc + separator + next
+}
+
+// ContainsChunkContent reports whether the complete current body is safely
+// represented by another body. Very short substrings are not treated as
+// containment because common words and punctuation would create false drops.
+func ContainsChunkContent(container, contained string) bool {
+	if container == "" || contained == "" {
+		return false
+	}
+	if container == contained {
+		return true
+	}
+	return len([]rune(contained)) >= minOverlapRunes && strings.Contains(container, contained)
+}
+
+func runeSlicesEqual(left, right []rune) bool {
+	if len(left) != len(right) {
+		return false
+	}
+	for i := range left {
+		if left[i] != right[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// 这里实现 chunk 内容的「重叠拼接」公共逻辑，供文档重建（reconstructContent）、
+// 知识图谱内容合并（graph mergeChunkContents）等路径复用。聊天检索链路允许
+// 用户编辑 Chunk，使用上面的 JoinChunkContent，避免依赖原文位置坐标。
+//
+// 历史上各处都用「按位置」的公式裁剪重叠（offset = len(content) - (EndAt -
+// lastEndAt) 之类），它默认 len([]rune(Content)) == EndAt-StartAt。但有两类
+// 数据会破坏这个不变式，导致拼接错位、丢字或重复：
+//  1. 父子分块器会给被拆开的表格「补写表头」，补进去的表头是零宽度的
+//     （start == end），位置坐标无法表达它，content 比 EndAt-StartAt 更长；
+//  2. content 里可能保留 HTML 实体（如 " / >），其字符数比原文区间长。
+//
+// 因此这里改为「按文本」匹配重叠：在下一段开头的窗口里查找已合并文本的后缀首次
+// 出现的位置，从该位置之后接上。位置信息（StartAt/EndAt）仅用于估算搜索窗口
+// 大小，不再用于裁剪。
+
+const (
+	// minOverlapRunes 是参与匹配的最短后缀长度。太短（如表格分隔行 |---|）
+	// 容易误匹配，因此忽略。
+	minOverlapRunes = 12
+	// defaultSearchSpan 是搜索窗口的下限，保证即使位置信息缺失/为 0 也能
+	// 检测到一定范围内的真实重叠。
+	defaultSearchSpan = 400
+)
+
+// AppendWithOverlap 把 next 追加到 acc 之后，并去除二者之间的重叠部分。
+//
+// positionOverlap 是由 StartAt/EndAt 估算的重叠量（lastEnd - curStart），仅用于
+// 界定搜索窗口大小；真正的重叠按文本匹配，能兼容补写表头与 HTML 实体长度偏差。
+// 若找不到文本重叠，则原样拼接（不裁剪），宁可保留也不破坏内容。
+//
+// positionOverlap <= 0 时两段位置上严格相邻或不相交，没有可去重的重叠；
+// 此时进入文本匹配会因 headSlack 下限 320 在 next 开头窗口内误命中
+// acc 后缀的真实内容重复（如同一句话在文档多次出现），把 next 开头
+// 整段误判为补写表头删掉，造成不可逆的内容丢失。直接拼接，补写表头
+// 重复交给调用方后处理。
+func AppendWithOverlap(acc, next string, positionOverlap int) string {
+	if acc == "" {
+		return next
+	}
+	if next == "" {
+		return acc
+	}
+	if positionOverlap <= 0 {
+		return acc + next
+	}
+
+	accRunes := []rune(acc)
+	nextRunes := []rune(next)
+
+	span := positionOverlap
+
+	maxK := minInt(len(accRunes), len(nextRunes))
+	if cap := maxInt(span*3, defaultSearchSpan); maxK > cap {
+		maxK = cap
+	}
+	// 重叠内容之前最多允许跳过多少前缀（即补写的表头等合成文本）。
+	headSlack := maxInt(span*2, 320)
+
+	for k := maxK; k >= minOverlapRunes; k-- {
+		needle := accRunes[len(accRunes)-k:]
+		if pos := indexRunes(nextRunes, needle, headSlack); pos >= 0 {
+			return acc + string(nextRunes[pos+k:])
+		}
+	}
+	return acc + next
+}
+
+// AppendWithExactOverlap 在调用方已确认位置坐标可信时，按坐标给出的精确重叠量
+// 拼接 acc 与 next：校验 acc 的末 overlap 个字符与 next 的前 overlap 个字符逐字
+// 符相等，相等则精确裁剪，overlap 为 0 时直接拼接。
+//
+// 与 AppendWithOverlap 的区别在于「不猜」：后者为兼容补写表头、HTML 实体等长度
+// 偏差，会在窗口内搜索最长后缀匹配，重复周期性文本（表格、日志）可能被误判成重
+// 叠而裁掉真实内容。坐标可信时重叠量是已知的，不需要搜索。
+//
+// 校验不通过返回 ok=false，由调用方决定是否回退到 AppendWithOverlap。
+func AppendWithExactOverlap(acc, next string, overlap int) (string, bool) {
+	if acc == "" {
+		return next, true
+	}
+	if next == "" {
+		return acc, true
+	}
+	if overlap < 0 {
+		return "", false
+	}
+	if overlap == 0 {
+		return acc + next, true
+	}
+
+	accRunes := []rune(acc)
+	nextRunes := []rune(next)
+	if overlap > len(accRunes) || overlap > len(nextRunes) {
+		return "", false
+	}
+	if !runeSlicesEqual(accRunes[len(accRunes)-overlap:], nextRunes[:overlap]) {
+		return "", false
+	}
+	return acc + string(nextRunes[overlap:]), true
+}
+
+// MergeTextChunks 按 StartAt（并列时按 ChunkIndex）排序后，用 AppendWithOverlap
+// 把多个 chunk 的内容重建为完整文本。gapSep 用于位置不相邻（有间隙）的两段之间
+// 的分隔符（如 "\n"），传空串则直接拼接。
+//
+// 调用方负责先做类型过滤（例如只保留文本 chunk）；本函数不感知 ChunkType。
+func MergeTextChunks(chunks []*types.Chunk, gapSep string) string {
+	if len(chunks) == 0 {
+		return ""
+	}
+
+	sorted := make([]*types.Chunk, len(chunks))
+	copy(sorted, chunks)
+	sort.SliceStable(sorted, func(i, j int) bool {
+		if sorted[i].StartAt == sorted[j].StartAt {
+			return sorted[i].ChunkIndex < sorted[j].ChunkIndex
+		}
+		return sorted[i].StartAt < sorted[j].StartAt
+	})
+
+	merged := ""
+	mergedEnd := -1
+	for _, c := range sorted {
+		if c == nil || c.Content == "" {
+			continue
+		}
+		if merged == "" {
+			merged = c.Content
+			if c.EndAt > 0 {
+				mergedEnd = c.EndAt
+			}
+			continue
+		}
+
+		// 间隙 / 位置信息缺失（EndAt==0）：作为独立段落拼接，不做重叠裁剪。
+		if c.StartAt > mergedEnd || c.EndAt == 0 {
+			if gapSep != "" {
+				merged += gapSep
+			}
+			merged += c.Content
+			if c.EndAt > 0 {
+				mergedEnd = c.EndAt
+			}
+			continue
+		}
+
+		// 部分重叠或首尾相接：按文本匹配去重叠后拼接。
+		if c.EndAt > mergedEnd {
+			merged = AppendWithOverlap(merged, c.Content, mergedEnd-c.StartAt)
+			mergedEnd = c.EndAt
+		}
+		// 否则被上一段完全覆盖，跳过。
+	}
+
+	return merged
+}
+
+// indexRunes 在 haystack 中查找 needle 首次出现的 rune 下标，且起始位置不超过
+// maxStart。找不到返回 -1。
+func indexRunes(haystack, needle []rune, maxStart int) int {
+	if len(needle) == 0 || len(needle) > len(haystack) {
+		return -1
+	}
+	limit := len(haystack) - len(needle)
+	if maxStart < limit {
+		limit = maxStart
+	}
+	for i := 0; i <= limit; i++ {
+		match := true
+		for j := 0; j < len(needle); j++ {
+			if haystack[i+j] != needle[j] {
+				match = false
+				break
+			}
+		}
+		if match {
+			return i
+		}
+	}
+	return -1
+}
+
+func minInt(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func maxInt(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}

internal/searchutil/chunkmerge_test.go

@@ -0,0 +1,181 @@
+package searchutil
+
+import (
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestAppendWithOverlap_ContiguousNoTrim(t *testing.T) {
+	// 首尾相接（无重叠），且 next 含 HTML 实体使其字符数 > EndAt-StartAt。
+	// 旧的按位置公式会多切掉开头，这里应当整段保留。
+	acc := "## 第二节\n\n"
+	next := "| 列A | 列B |\n| 值1 | 含实体"引号"的内容 |\n"
+	got := AppendWithOverlap(acc, next, 0)
+	want := acc + next
+	if got != want {
+		t.Fatalf("contiguous merge mismatch:\n got=%q\nwant=%q", got, want)
+	}
+}
+
+func TestAppendWithOverlap_PrependedTableHeaderSkipped(t *testing.T) {
+	// 模拟 chunker 给拆分表格补写表头：next 开头多了一份表头（零宽，位置不可见），
+	// 真正的重叠行在表头之后。按位置裁剪会错位，按文本匹配应正确去重。
+	header := "| 列1 | 列2 | 列3 |\n|:---|:---|:---|\n"
+	overlapRows := "| 第5行 | 内容5A | 内容5B |\n| 第6行 | 内容6A | 内容6B |\n"
+	accTail := "| 第4行 | 内容4A | 内容4B |\n" + overlapRows
+	acc := header + "| 第1行 | x | — |\n" + accTail
+
+	newRows := "| 第7行 | 内容7A | 内容7B |\n| 第8行 | 内容8A | 内容8B |\n"
+	next := header + overlapRows + newRows
+
+	// 位置重叠量大约是两行的长度（这里给个近似值即可，仅用于窗口估算）。
+	got := AppendWithOverlap(acc, next, len([]rune(overlapRows)))
+	want := acc + newRows
+	if got != want {
+		t.Fatalf("prepended-header merge mismatch:\n got=%q\nwant=%q", got, want)
+	}
+}
+
+func TestAppendWithOverlap_PlainOverlap(t *testing.T) {
+	acc := "abcdefghijklmnopqrstuvwxyz0123"
+	// next 与 acc 末尾 "klmnopqrstuvwxyz0123" 重叠，再接新内容
+	next := "klmnopqrstuvwxyz0123ABCDEFG"
+	got := AppendWithOverlap(acc, next, 20)
+	want := "abcdefghijklmnopqrstuvwxyz0123ABCDEFG"
+	if got != want {
+		t.Fatalf("plain overlap mismatch:\n got=%q\nwant=%q", got, want)
+	}
+}
+
+func TestAppendWithOverlap_NoOverlap(t *testing.T) {
+	acc := "hello world"
+	next := "completely different"
+	got := AppendWithOverlap(acc, next, 0)
+	want := acc + next
+	if got != want {
+		t.Fatalf("no overlap mismatch:\n got=%q\nwant=%q", got, want)
+	}
+}
+
+func TestAppendWithExactOverlap_TrimsKnownOverlap(t *testing.T) {
+	overlap := "shared boundary text"
+	got, ok := AppendWithExactOverlap("before "+overlap, overlap+" after", len([]rune(overlap)))
+	if !ok {
+		t.Fatal("exact overlap should be accepted")
+	}
+	want := "before " + overlap + " after"
+	if got != want {
+		t.Fatalf("exact overlap mismatch:\n got=%q\nwant=%q", got, want)
+	}
+}
+
+func TestAppendWithExactOverlap_ZeroOverlapConcatenatesRepeatedText(t *testing.T) {
+	// 首尾相接的两段都由同一周期性文本组成（表格行 / 日志行）。重叠量为 0 时
+	// 必须原样拼接，不能去搜索后缀匹配，否则会吃掉 next 开头的重复行。
+	row := "| cell | cell |\n"
+	acc := "前言\n" + row + row
+	next := row + row + row + "结尾\n"
+	got, ok := AppendWithExactOverlap(acc, next, 0)
+	if !ok {
+		t.Fatal("zero overlap should be accepted")
+	}
+	if got != acc+next {
+		t.Fatalf("zero overlap must concatenate verbatim:\n got=%q\nwant=%q", got, acc+next)
+	}
+}
+
+func TestAppendWithExactOverlap_RejectsMismatchedOverlap(t *testing.T) {
+	// 长度不变式成立但文本已经不同（HTML 实体、补写表头等），必须拒绝，
+	// 由调用方回退到按文本匹配。
+	if _, ok := AppendWithExactOverlap("abcdefghijkl", "XYZdefghijkl", 6); ok {
+		t.Fatal("mismatched overlap should be rejected")
+	}
+}
+
+func TestAppendWithExactOverlap_RejectsOverlapLongerThanBodies(t *testing.T) {
+	if _, ok := AppendWithExactOverlap("short", "shorter", 99); ok {
+		t.Fatal("overlap exceeding body length should be rejected")
+	}
+	if _, ok := AppendWithExactOverlap("short", "shorter", -1); ok {
+		t.Fatal("negative overlap should be rejected")
+	}
+}
+
+func TestJoinChunkContentUsesCurrentTextInsteadOfSourceOffsets(t *testing.T) {
+	first := "first edited body with no original overlap"
+	second := "second independently edited body"
+	got := JoinChunkContent(first, second, "\n\n")
+	want := first + "\n\n" + second
+	if got != want {
+		t.Fatalf("edited join mismatch:\n got=%q\nwant=%q", got, want)
+	}
+}
+
+func TestJoinChunkContentRemovesRealBoundaryOverlap(t *testing.T) {
+	overlap := "shared boundary text"
+	got := JoinChunkContent("before "+overlap, overlap+" after", "\n\n")
+	want := "before " + overlap + " after"
+	if got != want {
+		t.Fatalf("overlap join mismatch:\n got=%q\nwant=%q", got, want)
+	}
+}
+
+func TestJoinChunkContentCollapsesExactContainment(t *testing.T) {
+	outer := "prefix complete current body suffix"
+	if got := JoinChunkContent(outer, "complete current body", "\n\n"); got != outer {
+		t.Fatalf("contained body should be collapsed: %q", got)
+	}
+}
+
+func TestMergeTextChunks_OrdersFiltersAndStitches(t *testing.T) {
+	header := "| a | b |\n|:--|:--|\n"
+	chunks := []*types.Chunk{
+		{
+			Content: header + "| r1 | x |\n| r2 | y |\n", ChunkType: types.ChunkTypeText,
+			StartAt: 0, EndAt: 20, ChunkIndex: 0,
+		},
+		{
+			// 补写表头 + 与上一段 r2 重叠 + 新行 r3
+			Content: header + "| r2 | y |\n| r3 | z |\n", ChunkType: types.ChunkTypeText,
+			StartAt: 10, EndAt: 40, ChunkIndex: 1,
+		},
+	}
+	got := MergeTextChunks(chunks, "\n")
+	want := header + "| r1 | x |\n| r2 | y |\n" + "| r3 | z |\n"
+	if got != want {
+		t.Fatalf("MergeTextChunks mismatch:\n got=%q\nwant=%q", got, want)
+	}
+}
+
+func TestMergeTextChunks_GapSeparator(t *testing.T) {
+	chunks := []*types.Chunk{
+		{Content: "first", ChunkType: types.ChunkTypeText, StartAt: 0, EndAt: 5, ChunkIndex: 0},
+		{Content: "second", ChunkType: types.ChunkTypeText, StartAt: 100, EndAt: 106, ChunkIndex: 1},
+	}
+	got := MergeTextChunks(chunks, "\n")
+	want := "first\nsecond"
+	if got != want {
+		t.Fatalf("gap separator mismatch:\n got=%q\nwant=%q", got, want)
+	}
+}
+
+// TestAppendWithOverlap_ContiguousRealContentRepeat is a regression test:
+// two chunks are strictly contiguous (positionOverlap==0), but a boilerplate
+// sentence at the tail of acc reappears inside the head window of next as
+// real content (the same sentence is written multiple times in the document).
+// The old algorithm would enter text matching and, due to the headSlack
+// floor of 320, mistake the head of next for a prepended table header and
+// delete it, causing irreversible content loss. After the fix it should
+// concatenate directly without trimming.
+func TestAppendWithOverlap_ContiguousRealContentRepeat(t *testing.T) {
+	repeat := "The system shall maintain a complete audit trail of all transactions."
+	acc := "3.2 Logging Requirements\n\n" + repeat
+	next := "\n\n5.1 Security Controls\n\n* Role-based access\n* Encryption at rest\n\n5.2 Compliance\n\n" + repeat + " This satisfies SOC 2."
+	got := AppendWithOverlap(acc, next, 0)
+	want := acc + next
+	if got != want {
+		t.Fatalf("contiguous real-content repeat must not trim:\n got.len=%d\nwant.len=%d\n got.tail=%q\nwant.tail=%q",
+			len(got), len(want), got[len(acc)-50:], want[len(acc)-50:])
+	}
+}

internal/searchutil/conversion.go

@@ -0,0 +1,97 @@
+package searchutil
+
+import (
+	"fmt"
+	"time"
+	"unicode/utf8"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// ConvertWebResultOption configures ConvertWebSearchResults behavior.
+type ConvertWebResultOption func(*convertWebResultOptions)
+
+type convertWebResultOptions struct {
+	seqFunc func(idx int) int
+}
+
+// WithSeqFunc overrides the default sequence assignment for converted results.
+func WithSeqFunc(f func(idx int) int) ConvertWebResultOption {
+	return func(opts *convertWebResultOptions) {
+		opts.seqFunc = f
+	}
+}
+
+// ConvertWebSearchResults converts []*types.WebSearchResult into []*types.SearchResult.
+func ConvertWebSearchResults(
+	webResults []*types.WebSearchResult,
+	opts ...ConvertWebResultOption,
+) []*types.SearchResult {
+	options := convertWebResultOptions{
+		seqFunc: func(int) int { return 1 },
+	}
+	for _, opt := range opts {
+		opt(&options)
+	}
+
+	results := make([]*types.SearchResult, 0, len(webResults))
+
+	for i, webResult := range webResults {
+		if webResult == nil {
+			continue
+		}
+
+		chunkID := webResult.URL
+		if chunkID == "" {
+			chunkID = fmt.Sprintf("web_search_%d", i)
+		}
+
+		content := webResult.Title
+		appendContent := func(text string) {
+			if text == "" {
+				return
+			}
+			if content != "" {
+				content += "\n\n" + text
+			} else {
+				content = text
+			}
+		}
+
+		appendContent(webResult.Snippet)
+		appendContent(webResult.Content)
+
+		result := &types.SearchResult{
+			ID:             chunkID,
+			Content:        content,
+			KnowledgeID:    chunkID, // Use URL as KnowledgeID so each web result stays independent during merge
+			ChunkIndex:     0,
+			KnowledgeTitle: webResult.Title,
+			StartAt:        0,
+			EndAt:          utf8.RuneCountInString(content),
+			Seq:            options.seqFunc(i),
+			Score:          0.6,
+			MatchType:      types.MatchTypeWebSearch,
+			SubChunkID:     []string{},
+			Metadata: map[string]string{
+				"url":     webResult.URL,
+				"source":  webResult.Source,
+				"title":   webResult.Title,
+				"snippet": webResult.Snippet,
+			},
+			ChunkType:         string(types.ChunkTypeWebSearch),
+			ParentChunkID:     "",
+			ImageInfo:         "",
+			KnowledgeFilename: "",
+			KnowledgeSource:   "web_search",
+		}
+
+		if webResult.PublishedAt != nil {
+			result.Metadata["published_at"] = webResult.PublishedAt.Format(time.RFC3339)
+		}
+
+		results = append(results, result)
+	}
+
+	return results
+}

internal/searchutil/imageinfo.go

@@ -0,0 +1,583 @@
+package searchutil
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// MarkdownImageRegex matches Markdown image links: ![alt](url)
+var MarkdownImageRegex = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`)
+
+// HTMLImageSrcRegex matches an HTML <img> tag with a quoted src attribute.
+// Documents that rely on surrounding markup for layout embed their screenshots
+// this way, so image-derived text has to be matched back to these tags as well
+// as to Markdown image links.
+//
+// Submatches: 1 = attributes before src, 2 = the src value, 3 = attributes
+// after. Callers index the value through HTMLImageSrcURLGroup rather than
+// assuming a position.
+//
+// src must be preceded by whitespace so that data-src and other hyphenated
+// attribute names are not mistaken for it — matching those would capture a
+// lazy-loading placeholder and leave the real src unvisited. An unquoted src
+// and a srcset-only tag are deliberately out of scope; both are rare in
+// authored documents and matching them reliably needs an HTML parser.
+//
+// Both this package and the document parser share this one definition. Two
+// copies would be free to drift, and an image the parser stores but the
+// enricher cannot match back is exactly the kind of gap this exists to close.
+var HTMLImageSrcRegex = regexp.MustCompile(`(?i)<img\b([^>]*?)\ssrc\s*=\s*['"]([^'"]+)['"]([^>]*)>`)
+
+// HTMLImageSrcURLGroup is the submatch index of the src value in
+// HTMLImageSrcRegex.
+const HTMLImageSrcURLGroup = 2
+
+// CollectImageInfoByChunkIDs collects merged image_info JSON for each given
+// chunk ID by querying child chunks (image_ocr / image_caption). It supports
+// two-level resolution:
+//   - If chunkIDs are text chunks, their direct children are image chunks → one query.
+//   - If chunkIDs are parent_text chunks, their children are text chunks
+//     whose children are image chunks → two queries.
+//
+// Disabled children are skipped at both levels. Removing an image from a chunk
+// disables its image_ocr / image_caption children (syncEditedChunkImages), and
+// disabling a text chunk disables them too; honoring that flag here is what
+// keeps a deleted image out of retrieval, summaries and model context.
+//
+// Returns a map of input chunkID → merged image_info JSON string.
+func CollectImageInfoByChunkIDs(
+	ctx context.Context,
+	chunkRepo interfaces.ChunkRepository,
+	tenantID uint64,
+	chunkIDs []string,
+) map[string]string {
+	if len(chunkIDs) == 0 {
+		return nil
+	}
+
+	children, err := chunkRepo.ListChunksByParentIDs(ctx, tenantID, chunkIDs)
+	if err != nil || len(children) == 0 {
+		return nil
+	}
+
+	type imageAgg struct {
+		byURL map[string]types.ImageInfo
+	}
+	aggMap := make(map[string]*imageAgg)
+
+	addInfo := func(targetID string, child *types.Chunk) {
+		if child.ImageInfo == "" {
+			return
+		}
+		var infos []types.ImageInfo
+		if err := json.Unmarshal([]byte(child.ImageInfo), &infos); err != nil || len(infos) == 0 {
+			return
+		}
+		agg, ok := aggMap[targetID]
+		if !ok {
+			agg = &imageAgg{byURL: make(map[string]types.ImageInfo)}
+			aggMap[targetID] = agg
+		}
+		for _, info := range infos {
+			key := info.URL
+			if key == "" {
+				key = info.OriginalURL
+			}
+			if key == "" {
+				continue
+			}
+			existing, exists := agg.byURL[key]
+			if !exists {
+				agg.byURL[key] = info
+			} else {
+				if info.OCRText != "" {
+					existing.OCRText = info.OCRText
+				}
+				if info.Caption != "" {
+					existing.Caption = info.Caption
+				}
+				agg.byURL[key] = existing
+			}
+		}
+	}
+
+	var textChildIDs []string
+	textToParent := make(map[string]string)
+
+	for _, child := range children {
+		if !child.IsEnabled {
+			continue
+		}
+		switch child.ChunkType {
+		case types.ChunkTypeImageOCR, types.ChunkTypeImageCaption:
+			addInfo(child.ParentChunkID, child)
+		case types.ChunkTypeText:
+			textChildIDs = append(textChildIDs, child.ID)
+			textToParent[child.ID] = child.ParentChunkID
+		}
+	}
+
+	if len(textChildIDs) > 0 {
+		grandChildren, err := chunkRepo.ListChunksByParentIDs(ctx, tenantID, textChildIDs)
+		if err == nil {
+			for _, gc := range grandChildren {
+				if !gc.IsEnabled {
+					continue
+				}
+				if gc.ChunkType != types.ChunkTypeImageOCR && gc.ChunkType != types.ChunkTypeImageCaption {
+					continue
+				}
+				if parentTextID, ok := textToParent[gc.ParentChunkID]; ok {
+					addInfo(parentTextID, gc)
+				}
+			}
+		}
+	}
+
+	out := make(map[string]string, len(aggMap))
+	for id, agg := range aggMap {
+		if len(agg.byURL) == 0 {
+			continue
+		}
+		merged := make([]types.ImageInfo, 0, len(agg.byURL))
+		for _, info := range agg.byURL {
+			merged = append(merged, info)
+		}
+		data, err := json.Marshal(merged)
+		if err != nil {
+			continue
+		}
+		out[id] = string(data)
+	}
+	return out
+}
+
+// EnrichSearchResultsImageInfo fills in ImageInfo for SearchResults that have
+// none by batch-querying child image chunks.
+func EnrichSearchResultsImageInfo(
+	ctx context.Context,
+	chunkRepo interfaces.ChunkRepository,
+	tenantID uint64,
+	results []*types.SearchResult,
+) {
+	var chunkIDs []string
+	seen := make(map[string]bool)
+	for _, r := range results {
+		if r.ImageInfo != "" {
+			continue
+		}
+		if !seen[r.ID] {
+			seen[r.ID] = true
+			chunkIDs = append(chunkIDs, r.ID)
+		}
+	}
+	if len(chunkIDs) == 0 {
+		return
+	}
+
+	infoMap := CollectImageInfoByChunkIDs(ctx, chunkRepo, tenantID, chunkIDs)
+	if len(infoMap) == 0 {
+		return
+	}
+
+	for _, r := range results {
+		if r.ImageInfo != "" {
+			continue
+		}
+		if merged, ok := infoMap[r.ID]; ok {
+			r.ImageInfo = merged
+		}
+	}
+}
+
+// MergeImageInfoJSON combines per-chunk image_info JSON strings (from
+// CollectImageInfoByChunkIDs) into a single JSON array, deduplicating by URL.
+func MergeImageInfoJSON(perChunk map[string]string) string {
+	if len(perChunk) == 0 {
+		return ""
+	}
+	seen := make(map[string]bool)
+	var all []types.ImageInfo
+	for _, raw := range perChunk {
+		var infos []types.ImageInfo
+		if err := json.Unmarshal([]byte(raw), &infos); err != nil {
+			continue
+		}
+		for _, info := range infos {
+			key := info.URL
+			if key == "" {
+				key = info.OriginalURL
+			}
+			if key != "" && !seen[key] {
+				seen[key] = true
+				all = append(all, info)
+			}
+		}
+	}
+	if len(all) == 0 {
+		return ""
+	}
+	data, err := json.Marshal(all)
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}
+
+// EnrichContentWithImageInfo embeds image info as XML tags into text content.
+// Inline Markdown image links get wrapped in <image> with <image_caption> / <image_ocr>;
+// images not found in content are appended as <image> blocks.
+func EnrichContentWithImageInfo(content string, imageInfoJSON string) string {
+	var imageInfos []types.ImageInfo
+	if err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos); err != nil {
+		return content
+	}
+	if len(imageInfos) == 0 {
+		return content
+	}
+
+	imageInfoMap := make(map[string]*types.ImageInfo)
+	for i := range imageInfos {
+		if imageInfos[i].URL != "" {
+			imageInfoMap[imageInfos[i].URL] = &imageInfos[i]
+		}
+		if imageInfos[i].OriginalURL != "" {
+			imageInfoMap[imageInfos[i].OriginalURL] = &imageInfos[i]
+		}
+	}
+
+	matches := MarkdownImageRegex.FindAllStringSubmatch(content, -1)
+	processedURLs := make(map[string]bool)
+
+	for _, match := range matches {
+		if len(match) < 3 {
+			continue
+		}
+		imgURL := match[2]
+		processedURLs[imgURL] = true
+
+		imgInfo, found := imageInfoMap[imgURL]
+		var b strings.Builder
+		b.WriteString(fmt.Sprintf("<image url=\"%s\">\n", imgURL))
+		b.WriteString(fmt.Sprintf("<image_original>%s</image_original>\n", match[0]))
+		if found && imgInfo != nil {
+			b.WriteString(BuildImageInfoXML(imgInfo))
+		}
+		b.WriteString("</image>")
+		content = strings.Replace(content, match[0], b.String(), 1)
+	}
+
+	var extras []string
+	for _, imgInfo := range imageInfos {
+		if processedURLs[imgInfo.URL] || processedURLs[imgInfo.OriginalURL] {
+			continue
+		}
+		url := imgInfo.URL
+		if url == "" {
+			url = imgInfo.OriginalURL
+		}
+		if block := BuildImageInfoXMLWithURL(url, &imgInfo); block != "" {
+			extras = append(extras, block)
+		}
+	}
+	if len(extras) > 0 {
+		if content != "" {
+			content += "\n"
+		}
+		content += strings.Join(extras, "\n")
+	}
+	return content
+}
+
+// EnrichContentWithImageInfoForChat enriches matching Markdown images with
+// caption / OCR text while keeping the image itself as Markdown. Chat context is
+// deliberately answer-ready: if a model copies a relevant image from its
+// context, the copied content should still render instead of leaking the
+// internal <image> XML protocol into the answer.
+//
+// Only images with a matching image_info entry are enriched. This avoids adding
+// every parent thumbnail when only a few pages were retrieved, and skips orphan
+// image_info extras.
+func EnrichContentWithImageInfoForChat(content string, imageInfoJSON string) string {
+	var imageInfos []types.ImageInfo
+	if err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos); err != nil {
+		return content
+	}
+	if len(imageInfos) == 0 {
+		return content
+	}
+
+	imageInfoMap := make(map[string]*types.ImageInfo)
+	for i := range imageInfos {
+		if imageInfos[i].URL != "" {
+			imageInfoMap[imageInfos[i].URL] = &imageInfos[i]
+		}
+		if imageInfos[i].OriginalURL != "" {
+			imageInfoMap[imageInfos[i].OriginalURL] = &imageInfos[i]
+		}
+	}
+
+	// Screenshots embedded as HTML carry their image_info the same way as
+	// Markdown links: without covering both, a document that keeps its images in
+	// <img> tags reaches the model as bare markup with the caption and OCR text
+	// stripped out, even though the analysis ran and the entry is right here.
+	//
+	// Both syntaxes are located against the ORIGINAL content and spliced in one
+	// right-to-left pass. Running one regex over the other's output would let a
+	// metadata block that itself quotes an image reference be rescanned, cutting
+	// the OCR sentence in half and injecting the same block twice.
+	type injection struct {
+		at   int
+		text string
+	}
+	var injections []injection
+
+	// trim is applied to HTML only. An attribute value may be padded, while a
+	// Markdown target is looked up exactly as written so that the keys this
+	// function matches on stay the ones it has always matched on.
+	appendFor := func(locs [][]int, urlGroup int, trim bool) {
+		for _, loc := range locs {
+			start, end := loc[2*urlGroup], loc[2*urlGroup+1]
+			if start < 0 {
+				continue
+			}
+			key := content[start:end]
+			if trim {
+				key = strings.TrimSpace(key)
+			}
+			imgInfo, found := imageInfoMap[key]
+			if !found || imgInfo == nil {
+				continue
+			}
+			metadata := buildImageInfoMarkdownMetadata(imgInfo)
+			if metadata == "" {
+				continue
+			}
+			injections = append(injections, injection{at: loc[1], text: "\n\n" + metadata})
+		}
+	}
+	appendFor(MarkdownImageRegex.FindAllStringSubmatchIndex(content, -1), 2, false)
+	appendFor(HTMLImageSrcRegex.FindAllStringSubmatchIndex(content, -1), HTMLImageSrcURLGroup, true)
+
+	sort.Slice(injections, func(i, j int) bool { return injections[i].at > injections[j].at })
+	for _, inj := range injections {
+		content = content[:inj.at] + inj.text + content[inj.at:]
+	}
+	return content
+}
+
+// buildImageInfoMarkdownMetadata keeps image-derived text explicit for the LLM
+// without introducing a second, user-visible markup protocol. Blockquotes keep
+// multiline OCR attached to its image and remain harmless if copied verbatim.
+func buildImageInfoMarkdownMetadata(img *types.ImageInfo) string {
+	if img == nil {
+		return ""
+	}
+
+	var lines []string
+	if caption := strings.TrimSpace(img.Caption); caption != "" {
+		lines = append(lines, "**Image caption:** "+caption)
+	}
+	if ocr := strings.TrimSpace(img.OCRText); ocr != "" {
+		lines = append(lines, "**Image text (OCR):** "+ocr)
+	}
+	if len(lines) == 0 {
+		return ""
+	}
+
+	return "> " + strings.ReplaceAll(strings.Join(lines, "\n\n"), "\n", "\n> ")
+}
+
+// BuildImageInfoMarkdownWithURL formats one image as answer-ready Markdown for
+// LLM-facing chat/tool context. The URL is intentionally preserved verbatim;
+// resource and provider URLs are opaque handles resolved by the frontend.
+func BuildImageInfoMarkdownWithURL(url string, img *types.ImageInfo) string {
+	if img == nil {
+		return ""
+	}
+	url = strings.TrimSpace(url)
+	metadata := buildImageInfoMarkdownMetadata(img)
+	if url == "" {
+		return metadata
+	}
+
+	alt := strings.Join(strings.Fields(img.Caption), " ")
+	if alt == "" {
+		alt = "image"
+	}
+	alt = strings.NewReplacer(
+		`\`, `\\`,
+		`[`, `\[`,
+		`]`, `\]`,
+	).Replace(alt)
+
+	image := fmt.Sprintf("![%s](%s)", alt, url)
+	if metadata == "" {
+		return image
+	}
+	return image + "\n\n" + metadata
+}
+
+// BuildImageInfoXML returns XML-tagged caption / ocr for one image.
+func BuildImageInfoXML(img *types.ImageInfo) string {
+	var b strings.Builder
+	if img.Caption != "" {
+		b.WriteString(fmt.Sprintf("<image_caption>%s</image_caption>\n", img.Caption))
+	}
+	if img.OCRText != "" {
+		b.WriteString(fmt.Sprintf("<image_ocr>%s</image_ocr>\n", img.OCRText))
+	}
+	return b.String()
+}
+
+// BuildImageInfoXMLWithURL wraps image info in an <image> element carrying the URL.
+func BuildImageInfoXMLWithURL(url string, img *types.ImageInfo) string {
+	inner := BuildImageInfoXML(img)
+	if inner == "" {
+		return ""
+	}
+	return fmt.Sprintf("<image url=\"%s\">\n%s</image>", url, inner)
+}
+
+// EnrichContentCaptionOnly is like EnrichContentWithImageInfo but only
+// includes image captions (no OCR text). Original content (including Markdown
+// image links) is preserved. Useful for summary generation where OCR would
+// add too much noise.
+func EnrichContentCaptionOnly(content string, imageInfoJSON string) string {
+	var imageInfos []types.ImageInfo
+	if err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos); err != nil {
+		return content
+	}
+	if len(imageInfos) == 0 {
+		return content
+	}
+
+	imageInfoMap := make(map[string]*types.ImageInfo)
+	for i := range imageInfos {
+		if imageInfos[i].URL != "" {
+			imageInfoMap[imageInfos[i].URL] = &imageInfos[i]
+		}
+		if imageInfos[i].OriginalURL != "" {
+			imageInfoMap[imageInfos[i].OriginalURL] = &imageInfos[i]
+		}
+	}
+
+	matches := MarkdownImageRegex.FindAllStringSubmatch(content, -1)
+	processedURLs := make(map[string]bool)
+
+	for _, match := range matches {
+		if len(match) < 3 {
+			continue
+		}
+		imgURL := match[2]
+		processedURLs[imgURL] = true
+
+		imgInfo, found := imageInfoMap[imgURL]
+		if found && imgInfo != nil && imgInfo.Caption != "" {
+			replacement := match[0] + "\n" + fmt.Sprintf("<image_caption>%s</image_caption>", imgInfo.Caption)
+			content = strings.Replace(content, match[0], replacement, 1)
+		}
+	}
+
+	var extras []string
+	for _, imgInfo := range imageInfos {
+		if processedURLs[imgInfo.URL] || processedURLs[imgInfo.OriginalURL] {
+			continue
+		}
+		if imgInfo.Caption != "" {
+			extras = append(extras, fmt.Sprintf("<image_caption>%s</image_caption>", imgInfo.Caption))
+		}
+	}
+	if len(extras) > 0 {
+		if content != "" {
+			content += "\n"
+		}
+		content += strings.Join(extras, "\n")
+	}
+	return content
+}
+
+// EnrichContentCaptionAndOCR is like EnrichContentCaptionOnly but ALSO
+// embeds OCR text alongside captions. URL and <image_original> wrapper
+// blocks are deliberately omitted (unlike EnrichContentWithImageInfo) —
+// the summary LLM only needs the human-readable text, not opaque export
+// hashes. Used as a fallback for image-dominated documents where caption
+// alone carries too little signal.
+func EnrichContentCaptionAndOCR(content string, imageInfoJSON string) string {
+	var imageInfos []types.ImageInfo
+	if err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos); err != nil {
+		return content
+	}
+	if len(imageInfos) == 0 {
+		return content
+	}
+
+	imageInfoMap := make(map[string]*types.ImageInfo)
+	for i := range imageInfos {
+		if imageInfos[i].URL != "" {
+			imageInfoMap[imageInfos[i].URL] = &imageInfos[i]
+		}
+		if imageInfos[i].OriginalURL != "" {
+			imageInfoMap[imageInfos[i].OriginalURL] = &imageInfos[i]
+		}
+	}
+
+	matches := MarkdownImageRegex.FindAllStringSubmatch(content, -1)
+	processedURLs := make(map[string]bool)
+
+	for _, match := range matches {
+		if len(match) < 3 {
+			continue
+		}
+		imgURL := match[2]
+		processedURLs[imgURL] = true
+
+		imgInfo, found := imageInfoMap[imgURL]
+		if !found || imgInfo == nil {
+			continue
+		}
+		appended := buildCaptionOCRBlock(imgInfo)
+		if appended == "" {
+			continue
+		}
+		content = strings.Replace(content, match[0], match[0]+"\n"+appended, 1)
+	}
+
+	var extras []string
+	for _, imgInfo := range imageInfos {
+		if processedURLs[imgInfo.URL] || processedURLs[imgInfo.OriginalURL] {
+			continue
+		}
+		if block := buildCaptionOCRBlock(&imgInfo); block != "" {
+			extras = append(extras, block)
+		}
+	}
+	if len(extras) > 0 {
+		if content != "" {
+			content += "\n"
+		}
+		content += strings.Join(extras, "\n")
+	}
+	return content
+}
+
+// buildCaptionOCRBlock returns the inline caption + OCR snippet (no URL
+// wrapper) used by EnrichContentCaptionAndOCR. Empty string when the image
+// has neither caption nor OCR.
+func buildCaptionOCRBlock(img *types.ImageInfo) string {
+	var parts []string
+	if img.Caption != "" {
+		parts = append(parts, fmt.Sprintf("<image_caption>%s</image_caption>", img.Caption))
+	}
+	if img.OCRText != "" {
+		parts = append(parts, fmt.Sprintf("<image_ocr>%s</image_ocr>", img.OCRText))
+	}
+	return strings.Join(parts, "\n")
+}

internal/searchutil/imageinfo_disabled_test.go

@@ -0,0 +1,84 @@
+package searchutil
+
+import (
+	"context"
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// childrenByParentRepo serves chunk children from a static parent → children map.
+type childrenByParentRepo struct {
+	interfaces.ChunkRepository
+	byParent map[string][]*types.Chunk
+}
+
+func (r childrenByParentRepo) ListChunksByParentIDs(
+	_ context.Context, _ uint64, parentIDs []string,
+) ([]*types.Chunk, error) {
+	var out []*types.Chunk
+	for _, id := range parentIDs {
+		out = append(out, r.byParent[id]...)
+	}
+	return out, nil
+}
+
+func imageChild(id, parentID string, chunkType types.ChunkType, url string, enabled bool) *types.Chunk {
+	return &types.Chunk{
+		ID:            id,
+		ParentChunkID: parentID,
+		ChunkType:     chunkType,
+		IsEnabled:     enabled,
+		ImageInfo:     `[{"url":"` + url + `","caption":"a screenshot"}]`,
+	}
+}
+
+// Deleting an image from a chunk disables its image children. Their image_info
+// must not come back through enrichment, or the removed image reappears in
+// summaries, tool output and model context.
+func TestCollectImageInfoByChunkIDsSkipsDisabledImageChildren(t *testing.T) {
+	repo := childrenByParentRepo{byParent: map[string][]*types.Chunk{
+		"text-1": {
+			imageChild("ocr-1", "text-1", types.ChunkTypeImageOCR, "resource://removed", false),
+			imageChild("cap-1", "text-1", types.ChunkTypeImageCaption, "resource://removed", false),
+		},
+		"text-2": {
+			imageChild("ocr-2", "text-2", types.ChunkTypeImageOCR, "resource://kept", true),
+		},
+	}}
+
+	got := CollectImageInfoByChunkIDs(context.Background(), repo, 1, []string{"text-1", "text-2"})
+
+	if _, ok := got["text-1"]; ok {
+		t.Fatalf("disabled image children leaked image_info: %q", got["text-1"])
+	}
+	if got["text-2"] == "" {
+		t.Fatal("enabled image child lost its image_info")
+	}
+}
+
+// The parent_text → text → image path must apply the same rule at both levels.
+func TestCollectImageInfoByChunkIDsSkipsDisabledGrandchildrenAndTextChildren(t *testing.T) {
+	repo := childrenByParentRepo{byParent: map[string][]*types.Chunk{
+		"parent-1": {
+			{ID: "text-1", ParentChunkID: "parent-1", ChunkType: types.ChunkTypeText, IsEnabled: true},
+		},
+		"parent-2": {
+			// Disabling a text chunk hides everything under it.
+			{ID: "text-2", ParentChunkID: "parent-2", ChunkType: types.ChunkTypeText, IsEnabled: false},
+		},
+		"text-1": {
+			imageChild("ocr-1", "text-1", types.ChunkTypeImageOCR, "resource://removed", false),
+		},
+		"text-2": {
+			imageChild("ocr-2", "text-2", types.ChunkTypeImageOCR, "resource://hidden", true),
+		},
+	}}
+
+	got := CollectImageInfoByChunkIDs(context.Background(), repo, 1, []string{"parent-1", "parent-2"})
+
+	if len(got) != 0 {
+		t.Fatalf("expected no image_info, got %v", got)
+	}
+}

internal/searchutil/imageinfo_html_test.go

@@ -0,0 +1,193 @@
+package searchutil
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func imageInfoJSON(t *testing.T, infos ...types.ImageInfo) string {
+	t.Helper()
+	data, err := json.Marshal(infos)
+	if err != nil {
+		t.Fatalf("marshal image info: %v", err)
+	}
+	return string(data)
+}
+
+func TestImageURLsInContent_CollectsHTMLSrc(t *testing.T) {
+	cases := []struct {
+		name    string
+		content string
+		want    string
+	}{
+		{"markdown", "![a](local://images/a.png)", "local://images/a.png"},
+		{"html double quotes", `<img src="local://images/b.png">`, "local://images/b.png"},
+		{"html single quotes", `<img src='local://images/c.png'>`, "local://images/c.png"},
+		{"html with attributes", `<img class="x" src="local://images/d.png" alt="y">`, "local://images/d.png"},
+		{"html uppercase", `<IMG SRC="local://images/e.png">`, "local://images/e.png"},
+		{"html padded", `<img src="  local://images/f.png  ">`, "local://images/f.png"},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			urls := ImageURLsInContent(tc.content)
+			if !urls[tc.want] {
+				t.Errorf("expected %q to be collected from %q, got %v", tc.want, tc.content, urls)
+			}
+		})
+	}
+}
+
+func TestFilterImageInfoByContentURLs_KeepsHTMLReferencedEntry(t *testing.T) {
+	info := types.ImageInfo{
+		URL:     "local://images/shot.png",
+		Caption: "the login screen",
+		OCRText: "Sign in with SSO",
+	}
+
+	// A passage whose images are HTML tags used to look image-free, which made
+	// the filter drop every entry it was given.
+	got := FilterImageInfoByContentURLs(
+		`<p><img src="local://images/shot.png" alt="login"></p>`,
+		imageInfoJSON(t, info),
+	)
+	if got == "" {
+		t.Fatal("expected the entry to be kept for an HTML-only passage")
+	}
+
+	var kept []types.ImageInfo
+	if err := json.Unmarshal([]byte(got), &kept); err != nil {
+		t.Fatalf("unmarshal: %v", err)
+	}
+	if len(kept) != 1 || kept[0].URL != info.URL {
+		t.Errorf("unexpected filter result: %+v", kept)
+	}
+}
+
+func TestFilterImageInfoByContentURLs_StillDropsUnreferenced(t *testing.T) {
+	got := FilterImageInfoByContentURLs(
+		`<p><img src="local://images/present.png"></p>`,
+		imageInfoJSON(t, types.ImageInfo{URL: "local://images/absent.png", OCRText: "elsewhere"}),
+	)
+	if got != "" {
+		t.Errorf("expected an unreferenced entry to be dropped, got %s", got)
+	}
+}
+
+func TestEnrichContentWithImageInfoForChat_HTMLTag(t *testing.T) {
+	info := types.ImageInfo{
+		URL:     "local://images/shot.png",
+		Caption: "the login screen",
+		OCRText: "Sign in with SSO",
+	}
+	content := `<p><img src="local://images/shot.png" alt="login"></p>`
+
+	got := EnrichContentWithImageInfoForChat(content, imageInfoJSON(t, info))
+
+	if !strings.Contains(got, "Sign in with SSO") {
+		t.Errorf("OCR text was not injected for an HTML image: %s", got)
+	}
+	if !strings.Contains(got, "the login screen") {
+		t.Errorf("caption was not injected for an HTML image: %s", got)
+	}
+	if !strings.Contains(got, `<img src="local://images/shot.png"`) {
+		t.Errorf("the original tag should be preserved: %s", got)
+	}
+}
+
+func TestEnrichContentWithImageInfoForChat_MarkdownUnchanged(t *testing.T) {
+	info := types.ImageInfo{
+		URL:     "local://images/shot.png",
+		OCRText: "Sign in with SSO",
+	}
+	content := "![login](local://images/shot.png)"
+
+	got := EnrichContentWithImageInfoForChat(content, imageInfoJSON(t, info))
+
+	if !strings.Contains(got, "![login](local://images/shot.png)") {
+		t.Errorf("markdown image should be preserved: %s", got)
+	}
+	if !strings.Contains(got, "Sign in with SSO") {
+		t.Errorf("markdown enrichment regressed: %s", got)
+	}
+}
+
+func TestFilterImageInfoByMatchRange_HTMLTagInsideWindow(t *testing.T) {
+	info := types.ImageInfo{URL: "local://images/shot.png", OCRText: "Sign in with SSO"}
+	parent := `intro text
+<img src="local://images/shot.png">
+tail text`
+
+	// Window covering the tag: the entry survives.
+	got := FilterImageInfoByMatchRange(parent, 0, 0, len([]rune(parent)), imageInfoJSON(t, info))
+	if got == "" {
+		t.Error("expected the entry to survive a window containing the whole tag")
+	}
+
+	// Window that cuts the tag in half: nothing matches, which is the known
+	// limitation — an HTML tag is several times longer than ![](url) and
+	// straddles a child window far more often.
+	if got := FilterImageInfoByMatchRange(parent, 0, 0, 20, imageInfoJSON(t, info)); got != "" {
+		t.Errorf("expected a truncated tag not to match, got %s", got)
+	}
+}
+
+func TestEnrichContentWithImageInfoForChat_DoesNotRescanInjectedMetadata(t *testing.T) {
+	// The OCR text of a screenshot that documents an <img> tag contains one.
+	// Scanning the injected metadata would splice a second block into the middle
+	// of the first.
+	info := types.ImageInfo{
+		URL:     "local://images/x.png",
+		OCRText: `use <img src="local://images/x.png"> to embed`,
+	}
+	content := "![a](local://images/x.png)"
+
+	got := EnrichContentWithImageInfoForChat(content, imageInfoJSON(t, info))
+
+	if n := strings.Count(got, "**Image text (OCR):**"); n != 1 {
+		t.Errorf("expected exactly 1 metadata block, got %d:\n%s", n, got)
+	}
+	if !strings.Contains(got, "to embed") {
+		t.Errorf("OCR sentence was truncated:\n%s", got)
+	}
+}
+
+func TestEnrichContentWithImageInfoForChat_BothSyntaxesOneBlockEach(t *testing.T) {
+	info := types.ImageInfo{URL: "local://images/x.png", Caption: "shot"}
+	content := "![a](local://images/x.png)\n\n<img src=\"local://images/x.png\">"
+
+	got := EnrichContentWithImageInfoForChat(content, imageInfoJSON(t, info))
+
+	if n := strings.Count(got, "**Image caption:** shot"); n != 2 {
+		t.Errorf("expected one block per reference (2), got %d:\n%s", n, got)
+	}
+}
+
+func TestHTMLImageSrcRegex_HyphenatedAttributeIsNotSrc(t *testing.T) {
+	urls := ImageURLsInContent(`<img data-src="local://images/placeholder.png" src="local://images/real.png">`)
+	if urls["local://images/placeholder.png"] {
+		t.Error("data-src must not be collected as src")
+	}
+	if !urls["local://images/real.png"] {
+		t.Errorf("src was not collected: %v", urls)
+	}
+}
+
+func TestEnrichContentWithImageInfoForChat_UnmatchedImageStaysBare(t *testing.T) {
+	// The chat variant deliberately enriches only images that appear in the
+	// passage, so an entry for some other page must not be appended.
+	info := types.ImageInfo{URL: "local://images/other.png", OCRText: "another page"}
+	content := `<p><img src="local://images/shot.png"></p>`
+
+	got := EnrichContentWithImageInfoForChat(content, imageInfoJSON(t, info))
+
+	if got != content {
+		t.Errorf("expected content untouched, got %s", got)
+	}
+	if strings.Contains(got, "another page") {
+		t.Errorf("an orphan entry leaked into the passage: %s", got)
+	}
+}

internal/searchutil/imageinfo_match.go

@@ -0,0 +1,210 @@
+package searchutil
+
+import (
+	"encoding/json"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// SliceContentByDocumentRange extracts the substring of content whose
+// document-level rune offsets fall within [rangeStart, rangeEnd).
+// contentStartAt is the document offset where content begins.
+func SliceContentByDocumentRange(content string, contentStartAt, rangeStart, rangeEnd int) string {
+	if content == "" {
+		return content
+	}
+	relStart := rangeStart - contentStartAt
+	relEnd := rangeEnd - contentStartAt
+	runes := []rune(content)
+	if relStart < 0 {
+		relStart = 0
+	}
+	if relEnd > len(runes) {
+		relEnd = len(runes)
+	}
+	if relStart >= relEnd {
+		return ""
+	}
+	return string(runes[relStart:relEnd])
+}
+
+// ImageURLsInContent returns the set of image URLs referenced by content,
+// covering both Markdown image links and HTML <img> tags with a quoted src.
+func ImageURLsInContent(content string) map[string]bool {
+	urls := make(map[string]bool)
+	for _, match := range MarkdownImageRegex.FindAllStringSubmatch(content, -1) {
+		if len(match) < 3 || match[2] == "" {
+			continue
+		}
+		urls[match[2]] = true
+	}
+	// Screenshots embedded as HTML count as present in the content too.
+	// Collecting only Markdown links would make the image_info filters treat an
+	// HTML-only passage as holding no images at all and drop every entry.
+	for _, match := range HTMLImageSrcRegex.FindAllStringSubmatch(content, -1) {
+		if len(match) <= HTMLImageSrcURLGroup {
+			continue
+		}
+		if src := strings.TrimSpace(match[HTMLImageSrcURLGroup]); src != "" {
+			urls[src] = true
+		}
+	}
+	return urls
+}
+
+// FilterImageInfoByContentURLs keeps only image_info entries whose URL or
+// OriginalURL is referenced by content, as either a Markdown image link or an
+// HTML <img> tag. Returns empty string when nothing matches or imageInfoJSON
+// is invalid.
+func FilterImageInfoByContentURLs(content string, imageInfoJSON string) string {
+	if imageInfoJSON == "" {
+		return ""
+	}
+	var infos []types.ImageInfo
+	if err := json.Unmarshal([]byte(imageInfoJSON), &infos); err != nil || len(infos) == 0 {
+		return ""
+	}
+	urls := ImageURLsInContent(content)
+	if len(urls) == 0 {
+		return ""
+	}
+	filtered := make([]types.ImageInfo, 0, len(infos))
+	for _, info := range infos {
+		if urls[info.URL] || urls[info.OriginalURL] {
+			filtered = append(filtered, info)
+		}
+	}
+	return marshalImageInfos(filtered)
+}
+
+// ImageURLsFromInfo returns every current and original URL represented by an
+// image_info payload.
+func ImageURLsFromInfo(imageInfoJSON string) map[string]bool {
+	urls := make(map[string]bool)
+	if imageInfoJSON == "" {
+		return urls
+	}
+	var infos []types.ImageInfo
+	if err := json.Unmarshal([]byte(imageInfoJSON), &infos); err != nil {
+		return urls
+	}
+	for _, info := range infos {
+		if info.URL != "" {
+			urls[info.URL] = true
+		}
+		if info.OriginalURL != "" {
+			urls[info.OriginalURL] = true
+		}
+	}
+	return urls
+}
+
+// PruneMarkdownImagesByImageInfo keeps only Markdown images represented by
+// the chunk-scoped image metadata. This is stable after text edits because it
+// matches durable image URLs instead of parser character offsets.
+func PruneMarkdownImagesByImageInfo(content, imageInfoJSON string) string {
+	allowed := make(map[string]bool)
+	if imageInfoJSON != "" {
+		var infos []types.ImageInfo
+		if err := json.Unmarshal([]byte(imageInfoJSON), &infos); err == nil {
+			for _, info := range infos {
+				if info.URL != "" {
+					allowed[info.URL] = true
+				}
+				if info.OriginalURL != "" {
+					allowed[info.OriginalURL] = true
+				}
+			}
+		}
+	}
+	filtered := MarkdownImageRegex.ReplaceAllStringFunc(content, func(image string) string {
+		match := MarkdownImageRegex.FindStringSubmatch(image)
+		if len(match) >= 3 && allowed[match[2]] {
+			return image
+		}
+		return ""
+	})
+	return collapseBlankLines(filtered)
+}
+
+// FilterImageInfoByMatchRange keeps image_info entries whose image reference
+// falls within the document rune range [matchStart, matchEnd).
+// Used when parent content is expanded for context but multimodal enrichment
+// should only cover the retrieved child window.
+func FilterImageInfoByMatchRange(
+	parentContent string,
+	parentStartAt, matchStart, matchEnd int,
+	imageInfoJSON string,
+) string {
+	if imageInfoJSON == "" {
+		return ""
+	}
+	var infos []types.ImageInfo
+	if err := json.Unmarshal([]byte(imageInfoJSON), &infos); err != nil || len(infos) == 0 {
+		return ""
+	}
+	window := SliceContentByDocumentRange(parentContent, parentStartAt, matchStart, matchEnd)
+	urls := ImageURLsInContent(window)
+	if len(urls) == 0 {
+		return ""
+	}
+	filtered := make([]types.ImageInfo, 0, len(infos))
+	for _, info := range infos {
+		if urls[info.URL] || urls[info.OriginalURL] {
+			filtered = append(filtered, info)
+		}
+	}
+	return marshalImageInfos(filtered)
+}
+
+func marshalImageInfos(infos []types.ImageInfo) string {
+	if len(infos) == 0 {
+		return ""
+	}
+	data, err := json.Marshal(infos)
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}
+
+// PruneMarkdownImagesOutsideRange removes Markdown image lines whose
+// document-level rune offsets fall outside [matchStart, matchEnd). Non-image
+// text is preserved so parent-child expansion still provides full textual
+// context while dropping irrelevant page thumbnails.
+func PruneMarkdownImagesOutsideRange(
+	content string,
+	contentStartAt, matchStart, matchEnd int,
+) string {
+	locs := MarkdownImageRegex.FindAllStringIndex(content, -1)
+	if len(locs) == 0 {
+		return content
+	}
+	var b strings.Builder
+	last := 0
+	for _, loc := range locs {
+		if loc[0] < last || loc[1] > len(content) {
+			continue
+		}
+		docStart := contentStartAt + utf8.RuneCountInString(content[:loc[0]])
+		docEnd := contentStartAt + utf8.RuneCountInString(content[:loc[1]])
+		inRange := docStart < matchEnd && docEnd > matchStart
+		if inRange {
+			b.WriteString(content[last:loc[1]])
+		} else {
+			b.WriteString(content[last:loc[0]])
+		}
+		last = loc[1]
+	}
+	b.WriteString(content[last:])
+	return collapseBlankLines(b.String())
+}
+
+func collapseBlankLines(s string) string {
+	for strings.Contains(s, "\n\n\n") {
+		s = strings.ReplaceAll(s, "\n\n\n", "\n\n")
+	}
+	return strings.TrimSpace(s)
+}

internal/searchutil/imageinfo_match_test.go

@@ -0,0 +1,174 @@
+package searchutil
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestSliceContentByDocumentRange(t *testing.T) {
+	parent := "aaaPAGE1bbbPAGE2ccc"
+	got := SliceContentByDocumentRange(parent, 100, 103, 108)
+	want := "PAGE1"
+	if got != want {
+		t.Fatalf("slice: got %q, want %q", got, want)
+	}
+}
+
+func TestFilterImageInfoByMatchRange(t *testing.T) {
+	parent := "![p1](u1)\n\n![p2](u2)\n\n![p3](u3)"
+	matchStart := len([]rune("![p1](u1)\n\n"))
+	matchEnd := matchStart + len([]rune("![p2](u2)"))
+	all := []types.ImageInfo{
+		{URL: "u1"}, {URL: "u2"}, {URL: "u3"},
+	}
+	raw, err := json.Marshal(all)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := FilterImageInfoByMatchRange(parent, 0, matchStart, matchEnd, string(raw))
+	var filtered []types.ImageInfo
+	if err := json.Unmarshal([]byte(got), &filtered); err != nil {
+		t.Fatal(err)
+	}
+	if len(filtered) != 1 || filtered[0].URL != "u2" {
+		t.Fatalf("filtered: %+v", filtered)
+	}
+}
+
+func TestFilterImageInfoByContentURLs(t *testing.T) {
+	content := "intro\n![page3](local://img3.jpg)\noutro"
+	all := []types.ImageInfo{
+		{URL: "local://img1.jpg", OCRText: "one"},
+		{URL: "local://img3.jpg", OCRText: "three"},
+	}
+	raw, err := json.Marshal(all)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := FilterImageInfoByContentURLs(content, string(raw))
+	var filtered []types.ImageInfo
+	if err := json.Unmarshal([]byte(got), &filtered); err != nil {
+		t.Fatalf("unmarshal filtered: %v", err)
+	}
+	if len(filtered) != 1 || filtered[0].URL != "local://img3.jpg" {
+		t.Fatalf("filtered: %+v", filtered)
+	}
+}
+
+func TestPruneMarkdownImagesOutsideRange(t *testing.T) {
+	parent := "![p1](u1)\n\n![p2](u2)\n\n![p3](u3)"
+	matchStart := len([]rune("![p1](u1)\n\n"))
+	matchEnd := matchStart + len([]rune("![p2](u2)"))
+	got := PruneMarkdownImagesOutsideRange(parent, 0, matchStart, matchEnd)
+	if got != "![p2](u2)" {
+		t.Fatalf("prune: got %q", got)
+	}
+}
+
+func TestPruneMarkdownImagesByImageInfoIgnoresShiftedOffsets(t *testing.T) {
+	content := "a manually inserted prefix that shifts every parser offset\n\n" +
+		"![p1](u1)\n\nbody\n\n![p2](u2)"
+	raw, err := json.Marshal([]types.ImageInfo{{URL: "u2"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := PruneMarkdownImagesByImageInfo(content, string(raw))
+	if strings.Contains(got, "u1") {
+		t.Fatalf("unscoped image remained: %q", got)
+	}
+	if !strings.Contains(got, "![p2](u2)") {
+		t.Fatalf("scoped image was removed: %q", got)
+	}
+}
+
+func TestEnrichContentWithImageInfoForChat_SkipsUnmatched(t *testing.T) {
+	content := "![p1](u1)\n\n![p2](u2)"
+	raw, _ := json.Marshal([]types.ImageInfo{{URL: "u2", OCRText: "two"}})
+	got := EnrichContentWithImageInfoForChat(content, string(raw))
+	if strings.Contains(got, "<image") {
+		t.Fatalf("chat context should not contain internal image XML: %s", got)
+	}
+	if !strings.Contains(got, "![p1](u1)") {
+		t.Fatalf("unmatched markdown should remain: %s", got)
+	}
+	if !strings.Contains(got, "![p2](u2)") {
+		t.Fatalf("matched markdown should remain renderable: %s", got)
+	}
+	if !strings.Contains(got, "> **Image text (OCR):** two") {
+		t.Fatalf("matched image should be enriched: %s", got)
+	}
+	if strings.Count(got, "![") != 2 {
+		t.Fatalf("chat enrich should not duplicate markdown images: %s", got)
+	}
+}
+
+func TestEnrichContentWithImageInfoForChat_UsesMarkdownForMultilineMetadata(t *testing.T) {
+	content := "before\n\n![flow](resource://AbCdEfGhIjKlMnOpQrStUv)\n\nafter"
+	raw, _ := json.Marshal([]types.ImageInfo{
+		{
+			URL:     "resource://AbCdEfGhIjKlMnOpQrStUv",
+			Caption: "目标说话人提取流程图",
+			OCRText: "输入\n目标说话人提取\n输出",
+		},
+	})
+
+	got := EnrichContentWithImageInfoForChat(content, string(raw))
+	for _, want := range []string{
+		"![flow](resource://AbCdEfGhIjKlMnOpQrStUv)",
+		"> **Image caption:** 目标说话人提取流程图",
+		"> **Image text (OCR):** 输入",
+		"> 目标说话人提取",
+		"> 输出",
+	} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("expected %q in enriched Markdown:\n%s", want, got)
+		}
+	}
+	if strings.Contains(got, "<image") {
+		t.Fatalf("chat context should be Markdown-only for image content: %s", got)
+	}
+}
+
+func TestEnrichContentWithImageInfoForChat_EnrichesRepeatedImagesOnceEach(t *testing.T) {
+	content := "![same](u1)\n\n![same](u1)"
+	raw, _ := json.Marshal([]types.ImageInfo{{URL: "u1", Caption: "same caption"}})
+
+	got := EnrichContentWithImageInfoForChat(content, string(raw))
+	if strings.Count(got, "![same](u1)") != 2 {
+		t.Fatalf("expected both Markdown images to remain: %s", got)
+	}
+	if strings.Count(got, "> **Image caption:** same caption") != 2 {
+		t.Fatalf("expected each image to be enriched exactly once: %s", got)
+	}
+}
+
+func TestBuildImageInfoMarkdownWithURL(t *testing.T) {
+	got := BuildImageInfoMarkdownWithURL(
+		"resource://AbCdEfGhIjKlMnOpQrStUv",
+		&types.ImageInfo{Caption: "流程图 [测试]", OCRText: "输入\n输出"},
+	)
+	for _, want := range []string{
+		`![流程图 \[测试\]](resource://AbCdEfGhIjKlMnOpQrStUv)`,
+		"> **Image caption:** 流程图 [测试]",
+		"> **Image text (OCR):** 输入",
+		"> 输出",
+	} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("expected %q in image Markdown:\n%s", want, got)
+		}
+	}
+	if strings.Contains(got, "<image") {
+		t.Fatalf("LLM-facing image context must not use image XML: %s", got)
+	}
+}
+
+func TestImageURLsInContent(t *testing.T) {
+	content := "![a](u1) x ![b](u2)"
+	urls := ImageURLsInContent(content)
+	if !urls["u1"] || !urls["u2"] || len(urls) != 2 {
+		t.Fatalf("urls: %#v", urls)
+	}
+}

internal/searchutil/normalize.go

@@ -0,0 +1,109 @@
+package searchutil
+
+import "sort"
+
+// KeywordScoreCallbacks allows callers to hook into normalization telemetry.
+type KeywordScoreCallbacks struct {
+	OnNoVariance func(count int, score float64)
+	OnNormalized func(count int, rawMin, rawMax, normalizeMin, normalizeMax float64)
+}
+
+// NormalizeKeywordScores normalizes keyword match scores in-place using robust percentile bounds.
+func NormalizeKeywordScores[T any](
+	results []T,
+	isKeyword func(T) bool,
+	getScore func(T) float64,
+	setScore func(T, float64),
+	callbacks KeywordScoreCallbacks,
+) {
+	keywordResults := make([]T, 0, len(results))
+	for _, result := range results {
+		if isKeyword(result) {
+			keywordResults = append(keywordResults, result)
+		}
+	}
+
+	if len(keywordResults) == 0 {
+		return
+	}
+
+	if len(keywordResults) == 1 {
+		setScore(keywordResults[0], 1.0)
+		return
+	}
+
+	minS := getScore(keywordResults[0])
+	maxS := minS
+	for _, r := range keywordResults[1:] {
+		score := getScore(r)
+		if score < minS {
+			minS = score
+		}
+		if score > maxS {
+			maxS = score
+		}
+	}
+
+	if maxS <= minS {
+		for _, r := range keywordResults {
+			setScore(r, 1.0)
+		}
+		if callbacks.OnNoVariance != nil {
+			callbacks.OnNoVariance(len(keywordResults), minS)
+		}
+		return
+	}
+
+	normalizeMin := minS
+	normalizeMax := maxS
+
+	if len(keywordResults) >= 10 {
+		scores := make([]float64, len(keywordResults))
+		for i, r := range keywordResults {
+			scores[i] = getScore(r)
+		}
+		sort.Float64s(scores)
+		p5Idx := len(scores) * 5 / 100
+		p95Idx := len(scores) * 95 / 100
+		if p5Idx < len(scores) {
+			normalizeMin = scores[p5Idx]
+		}
+		if p95Idx < len(scores) {
+			normalizeMax = scores[p95Idx]
+		}
+	}
+
+	rangeSize := normalizeMax - normalizeMin
+	if rangeSize > 0 {
+		for _, r := range keywordResults {
+			clamped := getScore(r)
+			if clamped < normalizeMin {
+				clamped = normalizeMin
+			} else if clamped > normalizeMax {
+				clamped = normalizeMax
+			}
+			ns := (clamped - normalizeMin) / rangeSize
+			if ns < 0 {
+				ns = 0
+			} else if ns > 1 {
+				ns = 1
+			}
+			setScore(r, ns)
+		}
+		if callbacks.OnNormalized != nil {
+			callbacks.OnNormalized(
+				len(keywordResults),
+				minS,
+				maxS,
+				normalizeMin,
+				normalizeMax,
+			)
+		}
+		return
+	}
+
+	// Fallback when percentile filtering collapses the range.
+	for _, r := range keywordResults {
+		setScore(r, 1.0)
+	}
+}

internal/searchutil/textutil.go

@@ -0,0 +1,162 @@
+package searchutil
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"strings"
+	"unicode"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// BuildContentSignature creates a normalized MD5 signature for content to detect duplicates.
+// It normalizes the content by lowercasing, trimming whitespace, and collapsing multiple spaces.
+func BuildContentSignature(content string) string {
+	c := strings.ToLower(strings.TrimSpace(content))
+	if c == "" {
+		return ""
+	}
+	// Normalize whitespace
+	c = strings.Join(strings.Fields(c), " ")
+	// Use MD5 hash of full content
+	hash := md5.Sum([]byte(c))
+	return hex.EncodeToString(hash[:])
+}
+
+// containsChinese checks whether text contains any CJK unified ideographs.
+func containsChinese(text string) bool {
+	for _, r := range text {
+		if unicode.Is(unicode.Han, r) {
+			return true
+		}
+	}
+	return false
+}
+
+// TokenizeSimple tokenizes text into a set of unique tokens.
+// For text containing Chinese characters, it uses jieba segmentation for accurate word boundaries.
+// For pure non-Chinese text, it falls back to whitespace-based splitting.
+// Returns a map where keys are lowercase tokens with rune length > 1.
+func TokenizeSimple(text string) map[string]struct{} {
+	text = strings.ToLower(strings.TrimSpace(text))
+	if text == "" {
+		return nil
+	}
+
+	var words []string
+	if containsChinese(text) {
+		// Use jieba for Chinese text segmentation (search mode for finer granularity)
+		words = types.Jieba.CutForSearch(text, true)
+	} else {
+		words = strings.Fields(text)
+	}
+
+	set := make(map[string]struct{}, len(words))
+	for _, w := range words {
+		w = strings.TrimSpace(w)
+		// Filter out single-rune tokens and pure punctuation/whitespace
+		if len([]rune(w)) > 1 && !isAllPunct(w) {
+			set[w] = struct{}{}
+		}
+	}
+	return set
+}
+
+// isAllPunct checks if a string consists entirely of punctuation or whitespace.
+func isAllPunct(s string) bool {
+	for _, r := range s {
+		if !unicode.IsPunct(r) && !unicode.IsSpace(r) && !unicode.IsSymbol(r) {
+			return false
+		}
+	}
+	return true
+}
+
+// Jaccard calculates Jaccard similarity between two token sets.
+// Returns a value between 0 and 1, where 1 means identical sets.
+func Jaccard(a, b map[string]struct{}) float64 {
+	if len(a) == 0 && len(b) == 0 {
+		return 0
+	}
+
+	// small set drives large set
+	if len(a) > len(b) {
+		return Jaccard(b, a)
+	}
+
+	// Calculate intersection
+	inter := 0
+	for k := range a {
+		if _, ok := b[k]; ok {
+			inter++
+		}
+	}
+
+	// Calculate union
+	union := len(a) + len(b) - inter
+	if union == 0 {
+		return 0
+	}
+
+	return float64(inter) / float64(union)
+}
+
+// NormalizeContent returns a lowercased, whitespace-collapsed version of s
+// suitable for containment and overlap checks.
+func NormalizeContent(s string) string {
+	c := strings.ToLower(strings.TrimSpace(s))
+	if c == "" {
+		return ""
+	}
+	return strings.Join(strings.Fields(c), " ")
+}
+
+// IsContentContained reports whether the normalized form of short is a
+// substring of the normalized form of long. Both inputs must already be
+// normalized via NormalizeContent.
+func IsContentContained(normalizedShort, normalizedLong string) bool {
+	if normalizedShort == "" || normalizedLong == "" {
+		return false
+	}
+	if len(normalizedShort) > len(normalizedLong) {
+		return false
+	}
+	return strings.Contains(normalizedLong, normalizedShort)
+}
+
+// ContentOverlapRatio estimates how much of a's content overlaps with b by
+// comparing their token sets (Jaccard-like but using overlap coefficient:
+// |intersection| / |smaller set|). Both inputs should be raw content strings.
+// Returns a value in [0, 1] where 1 means the smaller set is fully contained
+// in the larger set.
+func ContentOverlapRatio(a, b string) float64 {
+	tokA := TokenizeSimple(a)
+	tokB := TokenizeSimple(b)
+	if len(tokA) == 0 || len(tokB) == 0 {
+		return 0
+	}
+
+	small, large := tokA, tokB
+	if len(tokA) > len(tokB) {
+		small, large = tokB, tokA
+	}
+
+	inter := 0
+	for k := range small {
+		if _, ok := large[k]; ok {
+			inter++
+		}
+	}
+	return float64(inter) / float64(len(small))
+}
+
+// ClampFloat clamps a float value to the specified range [minV, maxV].
+func ClampFloat(v, minV, maxV float64) float64 {
+	if v < minV {
+		return minV
+	}
+	if v > maxV {
+		return maxV
+	}
+	return v
+}