WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/application/service/chat_pipeline/merge_overlap.go

@@ -0,0 +1,221 @@
+package chatpipeline
+
+import (
+	"context"
+	"encoding/json"
+	"sort"
+
+	"github.com/Tencent/WeKnora/internal/searchutil"
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// mergeSequentialChunks joins sequential current chunk bodies.
+// Trusted pairs use position-aware merging;
+// any pair involving edited, expanded, or stale content falls back to
+// text matching so current content is never discarded on parser coordinates.
+// Chunks MUST be pre-sorted by ChunkIndex.
+func (p *PluginMerge) mergeSequentialChunks(
+	ctx context.Context,
+	knowledgeID string,
+	chunks []*types.SearchResult,
+) []*types.SearchResult {
+	if len(chunks) == 0 {
+		return nil
+	}
+
+	type mergedGroup struct {
+		result    *types.SearchResult
+		lastIndex int
+	}
+	groups := []mergedGroup{{result: chunks[0], lastIndex: chunks[0].ChunkIndex}}
+	for i := 1; i < len(chunks); i++ {
+		current := chunks[i]
+		last := &groups[len(groups)-1]
+		lastChunk := last.result
+
+		switch classifyMerge(lastChunk, last.lastIndex, current) {
+		case mergeSeparate:
+			groups = append(groups, mergedGroup{result: current, lastIndex: current.ChunkIndex})
+			continue
+		case mergeExtend:
+			lastChunk.Content = appendTrustedContent(lastChunk.Content, current.Content, lastChunk.EndAt-current.StartAt)
+			lastChunk.EndAt = current.EndAt
+			recordMergedChild(ctx, knowledgeID, lastChunk, current, "image_merge")
+		case mergeSubsume:
+			recordMergedChild(ctx, knowledgeID, lastChunk, current, "image_merge_contained")
+		case mergeJoinDistinct:
+			lastChunk.Content = searchutil.JoinChunkContent(lastChunk.Content, current.Content, "\n\n")
+			recordMergedChild(ctx, knowledgeID, lastChunk, current, "image_merge_contained")
+		case mergeJoinText:
+			lastChunk.Content = searchutil.JoinChunkContent(lastChunk.Content, current.Content, "\n\n")
+			recordMergedChild(ctx, knowledgeID, lastChunk, current, "image_merge")
+		}
+
+		// Extend the merged group's span and keep the higher score.
+		if current.ChunkIndex > last.lastIndex {
+			last.lastIndex = current.ChunkIndex
+		}
+		if current.Score > lastChunk.Score {
+			lastChunk.Score = current.Score
+		}
+	}
+
+	merged := make([]*types.SearchResult, 0, len(groups))
+	for _, group := range groups {
+		merged = append(merged, group.result)
+	}
+
+	// Sort merged chunks by score (highest first)
+	sort.Slice(merged, func(i, j int) bool {
+		return merged[i].Score > merged[j].Score
+	})
+
+	return merged
+}
+
+// appendTrustedContent joins a trusted pair using the overlap the coordinates
+// already give us, and only falls back to text search when the overlap does not
+// match character for character (HTML entities or synthetic table headers can
+// keep the length invariant while changing the body).
+//
+// The fallback matters both ways: searching for the longest suffix match would
+// otherwise mistake a repeated table row or log line for the overlap and drop
+// real content, which is exactly what the position path is meant to avoid.
+func appendTrustedContent(acc, next string, positionOverlap int) string {
+	if merged, ok := searchutil.AppendWithExactOverlap(acc, next, positionOverlap); ok {
+		return merged
+	}
+	return searchutil.AppendWithOverlap(acc, next, positionOverlap)
+}
+
+// chunkTrusted reports whether a result's StartAt/EndAt can be trusted for
+// position-based merging: unedited, not pipeline-rewritten, valid range, and
+// splitter-consistent length (runeLen(Content) == EndAt-StartAt).
+func chunkTrusted(chunk *types.SearchResult) bool {
+	return chunk.ContentRevision == 0 &&
+		!chunk.ContentRewritten &&
+		chunk.EndAt > chunk.StartAt &&
+		runeLen(chunk.Content) == chunk.EndAt-chunk.StartAt
+}
+
+// mergeSituation labels how the current result relates to the group's leading
+// result in document order. The classifier decides it once; the merge loop
+// switches on it without nesting.
+type mergeSituation int
+
+const (
+	// mergeSeparate: current is not mergeable, it starts a new group
+	// (position gap, or untrusted pair that is neither text-contained nor
+	// index-sequential).
+	mergeSeparate mergeSituation = iota
+	// mergeExtend: trusted pair with partial overlap or adjacency; the
+	// non-overlapping suffix is appended with text-verified trimming.
+	mergeExtend
+	// mergeSubsume: trusted pair, range-contained and text-verified; current
+	// is recorded without touching the merged content.
+	mergeSubsume
+	// mergeJoinDistinct: trusted pair, range-contained but textually distinct;
+	// current is kept via a text join, breaking the length invariant so later
+	// pairs degrade to the text path.
+	mergeJoinDistinct
+	// mergeJoinText: untrusted pair that is text-contained or index-sequential;
+	// joined via pure text matching.
+	mergeJoinText
+)
+
+// classifyMerge labels the pair relationship so the merge loop can dispatch
+// with a flat switch. It needs the group's running lastIndex for the untrusted
+// sequentiality check.
+func classifyMerge(lastChunk *types.SearchResult, lastIndex int, current *types.SearchResult) mergeSituation {
+	if chunkTrusted(lastChunk) && chunkTrusted(current) && current.StartAt >= lastChunk.StartAt {
+		switch {
+		case current.StartAt > lastChunk.EndAt:
+			return mergeSeparate
+		case current.EndAt > lastChunk.EndAt:
+			return mergeExtend
+		case searchutil.ContainsChunkContent(lastChunk.Content, current.Content):
+			return mergeSubsume
+		default:
+			return mergeJoinDistinct
+		}
+	}
+
+	textContained := searchutil.ContainsChunkContent(lastChunk.Content, current.Content) ||
+		searchutil.ContainsChunkContent(current.Content, lastChunk.Content)
+	sequential := current.ChunkIndex == lastIndex+1
+	if !textContained && !sequential {
+		return mergeSeparate
+	}
+	return mergeJoinText
+}
+
+// recordMergedChild records a merged chunk on the group result: appends its ID
+// to SubChunkID (deduplicated) and merges its ImageInfo, warning on failure.
+// warnKey distinguishes merge contexts in pipeline diagnostics.
+func recordMergedChild(ctx context.Context, knowledgeID string, target, source *types.SearchResult, warnKey string) {
+	if !containsID(target.SubChunkID, source.ID) {
+		target.SubChunkID = append(target.SubChunkID, source.ID)
+	}
+	if err := mergeImageInfo(ctx, target, source); err != nil {
+		pipelineWarn(ctx, "Merge", warnKey, map[string]any{
+			"knowledge_id": knowledgeID,
+			"error":        err.Error(),
+		})
+	}
+}
+
+// mergeImageInfo merges ImageInfo from source into target, deduplicating by URL.
+func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *types.SearchResult) error {
+	if source.ImageInfo == "" {
+		return nil
+	}
+
+	var sourceImageInfos []types.ImageInfo
+	if err := json.Unmarshal([]byte(source.ImageInfo), &sourceImageInfos); err != nil {
+		pipelineWarn(ctx, "Merge", "image_unmarshal_source", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return err
+	}
+
+	if len(sourceImageInfos) == 0 {
+		return nil
+	}
+
+	var targetImageInfos []types.ImageInfo
+	if target.ImageInfo != "" {
+		if err := json.Unmarshal([]byte(target.ImageInfo), &targetImageInfos); err != nil {
+			pipelineWarn(ctx, "Merge", "image_unmarshal_target", map[string]interface{}{
+				"error": err.Error(),
+			})
+			target.ImageInfo = source.ImageInfo
+			return nil
+		}
+	}
+
+	targetImageInfos = append(targetImageInfos, sourceImageInfos...)
+
+	uniqueMap := make(map[string]bool)
+	uniqueImageInfos := make([]types.ImageInfo, 0, len(targetImageInfos))
+
+	for _, imgInfo := range targetImageInfos {
+		if imgInfo.URL != "" && !uniqueMap[imgInfo.URL] {
+			uniqueMap[imgInfo.URL] = true
+			uniqueImageInfos = append(uniqueImageInfos, imgInfo)
+		}
+	}
+
+	mergedImageInfoJSON, err := json.Marshal(uniqueImageInfos)
+	if err != nil {
+		pipelineWarn(ctx, "Merge", "image_marshal", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return err
+	}
+
+	target.ImageInfo = string(mergedImageInfoJSON)
+	pipelineInfo(ctx, "Merge", "image_merged", map[string]interface{}{
+		"image_refs": len(uniqueImageInfos),
+	})
+	return nil
+}