WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/searchutil/imageinfo_match.go

@@ -0,0 +1,210 @@
+package searchutil
+
+import (
+	"encoding/json"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// SliceContentByDocumentRange extracts the substring of content whose
+// document-level rune offsets fall within [rangeStart, rangeEnd).
+// contentStartAt is the document offset where content begins.
+func SliceContentByDocumentRange(content string, contentStartAt, rangeStart, rangeEnd int) string {
+	if content == "" {
+		return content
+	}
+	relStart := rangeStart - contentStartAt
+	relEnd := rangeEnd - contentStartAt
+	runes := []rune(content)
+	if relStart < 0 {
+		relStart = 0
+	}
+	if relEnd > len(runes) {
+		relEnd = len(runes)
+	}
+	if relStart >= relEnd {
+		return ""
+	}
+	return string(runes[relStart:relEnd])
+}
+
+// ImageURLsInContent returns the set of image URLs referenced by content,
+// covering both Markdown image links and HTML <img> tags with a quoted src.
+func ImageURLsInContent(content string) map[string]bool {
+	urls := make(map[string]bool)
+	for _, match := range MarkdownImageRegex.FindAllStringSubmatch(content, -1) {
+		if len(match) < 3 || match[2] == "" {
+			continue
+		}
+		urls[match[2]] = true
+	}
+	// Screenshots embedded as HTML count as present in the content too.
+	// Collecting only Markdown links would make the image_info filters treat an
+	// HTML-only passage as holding no images at all and drop every entry.
+	for _, match := range HTMLImageSrcRegex.FindAllStringSubmatch(content, -1) {
+		if len(match) <= HTMLImageSrcURLGroup {
+			continue
+		}
+		if src := strings.TrimSpace(match[HTMLImageSrcURLGroup]); src != "" {
+			urls[src] = true
+		}
+	}
+	return urls
+}
+
+// FilterImageInfoByContentURLs keeps only image_info entries whose URL or
+// OriginalURL is referenced by content, as either a Markdown image link or an
+// HTML <img> tag. Returns empty string when nothing matches or imageInfoJSON
+// is invalid.
+func FilterImageInfoByContentURLs(content string, imageInfoJSON string) string {
+	if imageInfoJSON == "" {
+		return ""
+	}
+	var infos []types.ImageInfo
+	if err := json.Unmarshal([]byte(imageInfoJSON), &infos); err != nil || len(infos) == 0 {
+		return ""
+	}
+	urls := ImageURLsInContent(content)
+	if len(urls) == 0 {
+		return ""
+	}
+	filtered := make([]types.ImageInfo, 0, len(infos))
+	for _, info := range infos {
+		if urls[info.URL] || urls[info.OriginalURL] {
+			filtered = append(filtered, info)
+		}
+	}
+	return marshalImageInfos(filtered)
+}
+
+// ImageURLsFromInfo returns every current and original URL represented by an
+// image_info payload.
+func ImageURLsFromInfo(imageInfoJSON string) map[string]bool {
+	urls := make(map[string]bool)
+	if imageInfoJSON == "" {
+		return urls
+	}
+	var infos []types.ImageInfo
+	if err := json.Unmarshal([]byte(imageInfoJSON), &infos); err != nil {
+		return urls
+	}
+	for _, info := range infos {
+		if info.URL != "" {
+			urls[info.URL] = true
+		}
+		if info.OriginalURL != "" {
+			urls[info.OriginalURL] = true
+		}
+	}
+	return urls
+}
+
+// PruneMarkdownImagesByImageInfo keeps only Markdown images represented by
+// the chunk-scoped image metadata. This is stable after text edits because it
+// matches durable image URLs instead of parser character offsets.
+func PruneMarkdownImagesByImageInfo(content, imageInfoJSON string) string {
+	allowed := make(map[string]bool)
+	if imageInfoJSON != "" {
+		var infos []types.ImageInfo
+		if err := json.Unmarshal([]byte(imageInfoJSON), &infos); err == nil {
+			for _, info := range infos {
+				if info.URL != "" {
+					allowed[info.URL] = true
+				}
+				if info.OriginalURL != "" {
+					allowed[info.OriginalURL] = true
+				}
+			}
+		}
+	}
+	filtered := MarkdownImageRegex.ReplaceAllStringFunc(content, func(image string) string {
+		match := MarkdownImageRegex.FindStringSubmatch(image)
+		if len(match) >= 3 && allowed[match[2]] {
+			return image
+		}
+		return ""
+	})
+	return collapseBlankLines(filtered)
+}
+
+// FilterImageInfoByMatchRange keeps image_info entries whose image reference
+// falls within the document rune range [matchStart, matchEnd).
+// Used when parent content is expanded for context but multimodal enrichment
+// should only cover the retrieved child window.
+func FilterImageInfoByMatchRange(
+	parentContent string,
+	parentStartAt, matchStart, matchEnd int,
+	imageInfoJSON string,
+) string {
+	if imageInfoJSON == "" {
+		return ""
+	}
+	var infos []types.ImageInfo
+	if err := json.Unmarshal([]byte(imageInfoJSON), &infos); err != nil || len(infos) == 0 {
+		return ""
+	}
+	window := SliceContentByDocumentRange(parentContent, parentStartAt, matchStart, matchEnd)
+	urls := ImageURLsInContent(window)
+	if len(urls) == 0 {
+		return ""
+	}
+	filtered := make([]types.ImageInfo, 0, len(infos))
+	for _, info := range infos {
+		if urls[info.URL] || urls[info.OriginalURL] {
+			filtered = append(filtered, info)
+		}
+	}
+	return marshalImageInfos(filtered)
+}
+
+func marshalImageInfos(infos []types.ImageInfo) string {
+	if len(infos) == 0 {
+		return ""
+	}
+	data, err := json.Marshal(infos)
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}
+
+// PruneMarkdownImagesOutsideRange removes Markdown image lines whose
+// document-level rune offsets fall outside [matchStart, matchEnd). Non-image
+// text is preserved so parent-child expansion still provides full textual
+// context while dropping irrelevant page thumbnails.
+func PruneMarkdownImagesOutsideRange(
+	content string,
+	contentStartAt, matchStart, matchEnd int,
+) string {
+	locs := MarkdownImageRegex.FindAllStringIndex(content, -1)
+	if len(locs) == 0 {
+		return content
+	}
+	var b strings.Builder
+	last := 0
+	for _, loc := range locs {
+		if loc[0] < last || loc[1] > len(content) {
+			continue
+		}
+		docStart := contentStartAt + utf8.RuneCountInString(content[:loc[0]])
+		docEnd := contentStartAt + utf8.RuneCountInString(content[:loc[1]])
+		inRange := docStart < matchEnd && docEnd > matchStart
+		if inRange {
+			b.WriteString(content[last:loc[1]])
+		} else {
+			b.WriteString(content[last:loc[0]])
+		}
+		last = loc[1]
+	}
+	b.WriteString(content[last:])
+	return collapseBlankLines(b.String())
+}
+
+func collapseBlankLines(s string) string {
+	for strings.Contains(s, "\n\n\n") {
+		s = strings.ReplaceAll(s, "\n\n\n", "\n\n")
+	}
+	return strings.TrimSpace(s)
+}