WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/infrastructure/chunker/splitter.go

@@ -0,0 +1,914 @@
+// Package chunker implements text splitting for document chunking.
+//
+// Ported from the Python docreader/splitter/splitter.py recursive text splitter.
+package chunker
+
+import (
+	"regexp"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
+)
+
+// Chunk represents a piece of split text with position tracking.
+//
+// Content holds exactly the text from the original document between Start
+// and End (rune offsets), so End-Start == utf8.RuneCountInString(Content).
+// This invariant is relied on by document-reconstruction code paths
+// (knowledge.go:2278+ for summary generation, UI highlighting, etc.).
+//
+// ContextHeader is a separately-tracked context string (e.g. a Markdown
+// heading breadcrumb) that should be prepended at embedding/retrieval time
+// but is NOT part of Content. Keeping the two apart preserves the
+// position invariant while still letting embedding pipelines see the
+// section context.
+type Chunk struct {
+	Content       string
+	ContextHeader string
+	Seq           int
+	Start         int
+	End           int
+}
+
+// EmbeddingContent returns the text that should be fed to the embedding
+// model — the ContextHeader prepended (when set) plus the chunk content.
+// Use this where Content alone would lose semantic context (Tier-1 chunks).
+//
+// Content is returned verbatim from the source document (the End-Start
+// rune-count invariant requires that), but for embedding we trim the
+// surrounding whitespace so leading/trailing newlines from boundary slices
+// don't dilute the embedded vector or waste tokens. Inner whitespace is
+// preserved.
+func (c Chunk) EmbeddingContent() string {
+	body := strings.TrimSpace(c.Content)
+	if c.ContextHeader == "" {
+		return body
+	}
+	return c.ContextHeader + "\n\n" + body
+}
+
+// ImageRef is an image reference found within a chunk's content.
+type ImageRef struct {
+	OriginalRef string
+	AltText     string
+	Start       int // offset within the chunk content
+	End         int
+}
+
+// SplitterConfig configures the text splitter. Strategy and TokenLimit are
+// honored by the strategy entry point in strategy.go; the legacy SplitText
+// path uses only ChunkSize/Overlap/Separators.
+type SplitterConfig struct {
+	ChunkSize    int
+	ChunkOverlap int
+	Separators   []string
+
+	// Strategy selects an adaptive tier. Empty = legacy (backwards-compatible).
+	// See strategy.go for valid values.
+	Strategy string
+	// TokenLimit caps chunk size in approximate tokens. 0 = use ChunkSize chars.
+	TokenLimit int
+	// Languages hints multilingual heuristic patterns. Empty = auto-detect.
+	Languages []string
+}
+
+// Default chunk sizing constants. Single source of truth for the entire
+// chunker package and (via knowledge.go::buildSplitterConfig) the
+// knowledge service. The frontend KnowledgeBaseEditorModal mirrors these
+// numbers in its initial form state — keep them in sync if you change
+// either value here.
+//
+// DefaultChunkSize = 512 chars: ~100–130 English tokens / ~300 Chinese
+// tokens. Validated as a strong baseline by the Vecta Feb-2026 benchmark
+// across 50 academic papers. Use 200–400 for FAQ-style atomic content,
+// 1000–2000 for narrative / argumentative documents.
+//
+// DefaultChunkOverlap = 80 chars (≈15% of DefaultChunkSize): community-
+// recommended sweet spot between recall (an answer split across a
+// boundary needs overlap to be retrievable) and storage cost. Use 0 for
+// strictly atomic data (FAQ, JSON records), 150–200 for long narratives
+// where reasoning crosses chunks.
+//
+// MIGRATION NOTE: Prior versions had three different overlap defaults
+// (Go DefaultConfig: 64, knowledge.go buildSplitterConfig: 50, Python
+// docreader: 100). All consolidated to 80 here.
+//
+// Existing knowledge bases that stored ChunkOverlap=0 in the DB pick
+// this 80 up on next re-index; their previously-indexed embeddings will
+// not match new ones bit-for-bit. Recall stays similar but search
+// ranking can shift slightly. To freeze the old behavior on a per-KB
+// basis, explicitly set ChunkingConfig.ChunkOverlap to 64 before
+// re-indexing.
+const (
+	DefaultChunkSize    = 512
+	DefaultChunkOverlap = 80
+)
+
+// DefaultConfig returns sensible defaults.
+func DefaultConfig() SplitterConfig {
+	return SplitterConfig{
+		ChunkSize:    DefaultChunkSize,
+		ChunkOverlap: DefaultChunkOverlap,
+		Separators:   []string{"\n\n", "\n", "。"},
+	}
+}
+
+// protectedPatterns are regex patterns for content that must not be split.
+var protectedPatterns = []*regexp.Regexp{
+	regexp.MustCompile(`(?s)\$\$.*?\$\$`),                                                               // LaTeX block math
+	regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`),                                                          // Markdown images
+	regexp.MustCompile(`\[[^\]]*\]\([^)]+\)`),                                                           // Markdown links
+	regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+\\s*(?:\\|\\s*:?-{3,}:?\\s*)+\\|[\\r\\n]+"), // Table header+separator
+	regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+"),                                          // Table rows
+	regexp.MustCompile("(?s)```(?:\\w+)?[\\r\\n].*?```"),                                                // Fenced code blocks
+	regexp.MustCompile("`[^`\\r\\n]+`"),                                                                 // Markdown inline code
+}
+
+type span struct {
+	start, end int
+}
+
+// protectedSpansRune converts byte-offset protected spans to rune offsets
+// in a single forward pass over text. Used by callers that work in rune
+// space (e.g. the heuristic splitter) to avoid choosing chunk boundaries
+// that cut through protected content. byteSpans must be sorted by start
+// (protectedSpans guarantees this).
+func protectedSpansRune(text string, byteSpans []span) []span {
+	if len(byteSpans) == 0 {
+		return nil
+	}
+	out := make([]span, 0, len(byteSpans))
+	runeIdx := 0
+	byteIdx := 0
+	for _, s := range byteSpans {
+		for byteIdx < s.start && byteIdx < len(text) {
+			_, size := utf8.DecodeRuneInString(text[byteIdx:])
+			byteIdx += size
+			runeIdx++
+		}
+		startRune := runeIdx
+		for byteIdx < s.end && byteIdx < len(text) {
+			_, size := utf8.DecodeRuneInString(text[byteIdx:])
+			byteIdx += size
+			runeIdx++
+		}
+		out = append(out, span{start: startRune, end: runeIdx})
+	}
+	return out
+}
+
+// protectedSpans finds all non-overlapping protected regions in text.
+func protectedSpans(text string) []span {
+	type match struct {
+		start, end int
+	}
+	var all []match
+	for _, pat := range protectedPatterns {
+		locs := pat.FindAllStringIndex(text, -1)
+		for _, loc := range locs {
+			if loc[1]-loc[0] > 0 {
+				all = append(all, match{loc[0], loc[1]})
+			}
+		}
+	}
+	if len(all) == 0 {
+		return nil
+	}
+
+	// Sort by start, then by length descending
+	for i := 1; i < len(all); i++ {
+		for j := i; j > 0; j-- {
+			if all[j].start < all[j-1].start ||
+				(all[j].start == all[j-1].start && (all[j].end-all[j].start) > (all[j-1].end-all[j-1].start)) {
+				all[j], all[j-1] = all[j-1], all[j]
+			} else {
+				break
+			}
+		}
+	}
+
+	// Remove overlaps
+	var result []span
+	lastEnd := 0
+	for _, m := range all {
+		if m.start >= lastEnd {
+			result = append(result, span{m.start, m.end})
+			lastEnd = m.end
+		}
+	}
+	return result
+}
+
+// splitUnit is a piece of text with its original position.
+type splitUnit struct {
+	text       string
+	start, end int
+}
+
+// splitBySeparators splits text by separators in priority order, recursively
+// applying the next separator to any piece that is still larger than
+// chunkSize. Mirrors the recursive priority semantics of the Python
+// reference splitter (docreader/splitter/splitter.py:_split): if `\n\n`
+// produces a piece that's still too big, `\n` (and subsequent separators)
+// are applied within that piece — not to the whole text.
+//
+// chunkSize == 0 disables the recursion guard; callers that don't care
+// about size budget (e.g. a final mergeUnits-style pass) pass 0.
+func splitBySeparators(text string, separators []string, chunkSize int) []string {
+	if text == "" || len(separators) == 0 {
+		return []string{text}
+	}
+	if chunkSize > 0 && runeLen(text) <= chunkSize {
+		return []string{text}
+	}
+
+	for i, sep := range separators {
+		if sep == "" {
+			continue
+		}
+		re := regexp.MustCompile("(" + regexp.QuoteMeta(sep) + ")")
+		splits := re.Split(text, -1)
+		matches := re.FindAllString(text, -1)
+		if len(matches) == 0 {
+			continue
+		}
+
+		var pieces []string
+		for j, s := range splits {
+			if s != "" {
+				pieces = append(pieces, s)
+			}
+			if j < len(matches) && matches[j] != "" {
+				pieces = append(pieces, matches[j])
+			}
+		}
+		if len(pieces) <= 1 {
+			continue
+		}
+
+		// Recursively split any piece that is still too large with the
+		// remaining (lower-priority) separators.
+		var out []string
+		remaining := separators[i+1:]
+		for _, p := range pieces {
+			if chunkSize > 0 && runeLen(p) > chunkSize && len(remaining) > 0 {
+				out = append(out, splitBySeparators(p, remaining, chunkSize)...)
+			} else {
+				out = append(out, p)
+			}
+		}
+		return out
+	}
+	return []string{text}
+}
+
+// runeLen returns the number of runes in s.
+func runeLen(s string) int {
+	return utf8.RuneCountInString(s)
+}
+
+// SplitText splits text into chunks with overlap, respecting protected patterns.
+func SplitText(text string, cfg SplitterConfig) []Chunk {
+	if text == "" {
+		return nil
+	}
+
+	chunkSize := cfg.ChunkSize
+	chunkOverlap := cfg.ChunkOverlap
+	separators := cfg.Separators
+
+	if chunkSize <= 0 {
+		chunkSize = 512
+	}
+	if chunkOverlap < 0 {
+		chunkOverlap = 0
+	}
+
+	// Step 1: Find protected spans
+	protected := protectedSpans(text)
+
+	// Step 2: Split non-protected regions by separators, keep protected as atomic units.
+	// chunkSize is forwarded so splitBySeparators can recursively apply lower-priority
+	// separators to oversize pieces (Python-parity recursive split).
+	units := buildUnitsWithProtection(text, protected, separators, chunkSize)
+
+	// Step 3: Merge units into chunks with overlap
+	return mergeUnits(units, chunkSize, chunkOverlap)
+}
+
+// buildUnitsWithProtection splits text into units, preserving protected spans as atomic.
+// Start/End positions in the returned units are rune offsets (not byte offsets),
+// because downstream merge logic indexes content via []rune slicing.
+// If a protected span exceeds maxProtectedSize, it will be forcibly split to prevent
+// creating chunks that are too large for downstream processing (e.g., embedding APIs).
+// chunkSize is forwarded to splitBySeparators so recursive splitting can keep pieces
+// under the budget when one separator alone leaves a piece oversize.
+func buildUnitsWithProtection(text string, protected []span, separators []string, chunkSize int) []splitUnit {
+	const maxProtectedSize = 7500 // Maximum size for a protected unit (留余量给标题等)
+
+	var units []splitUnit
+	bytePos := 0
+	runePos := 0
+
+	for _, p := range protected {
+		if p.start > bytePos {
+			pre := text[bytePos:p.start]
+			parts := splitBySeparators(pre, separators, chunkSize)
+			runeOffset := runePos
+			for _, part := range parts {
+				partRuneLen := runeLen(part)
+				units = append(units, splitUnit{
+					text:  part,
+					start: runeOffset,
+					end:   runeOffset + partRuneLen,
+				})
+				runeOffset += partRuneLen
+			}
+			runePos += runeLen(pre)
+		}
+
+		protText := text[p.start:p.end]
+		protRuneLen := runeLen(protText)
+
+		// If protected content is too large, forcibly split it
+		if protRuneLen > maxProtectedSize {
+			// Split into smaller chunks at line breaks or spaces
+			runes := []rune(protText)
+			offset := 0
+			for offset < len(runes) {
+				chunkEnd := offset + maxProtectedSize
+				if chunkEnd > len(runes) {
+					chunkEnd = len(runes)
+				} else {
+					// Try to break at a newline or space
+					for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- {
+						if runes[i] == '\n' || runes[i] == ' ' {
+							chunkEnd = i + 1
+							break
+						}
+					}
+				}
+
+				chunkText := string(runes[offset:chunkEnd])
+				chunkLen := chunkEnd - offset
+				units = append(units, splitUnit{
+					text:  chunkText,
+					start: runePos + offset,
+					end:   runePos + offset + chunkLen,
+				})
+				offset = chunkEnd
+			}
+		} else {
+			// Normal case: keep protected content as a single unit
+			units = append(units, splitUnit{
+				text:  protText,
+				start: runePos,
+				end:   runePos + protRuneLen,
+			})
+		}
+		runePos += protRuneLen
+		bytePos = p.end
+	}
+
+	if bytePos < len(text) {
+		remaining := text[bytePos:]
+		parts := splitBySeparators(remaining, separators, chunkSize)
+		runeOffset := runePos
+		for _, part := range parts {
+			partRuneLen := runeLen(part)
+			units = append(units, splitUnit{
+				text:  part,
+				start: runeOffset,
+				end:   runeOffset + partRuneLen,
+			})
+			runeOffset += partRuneLen
+		}
+	}
+
+	return units
+}
+
+// mergeUnits combines split units into chunks with overlap tracking.
+// Enforces an absolute maximum chunk size to prevent exceeding downstream limits (e.g., embedding APIs).
+// Active contextual headers (e.g., Markdown table headers) are prepended to new
+// chunks so that every chunk carries its own header context.
+func mergeUnits(units []splitUnit, chunkSize, chunkOverlap int) []Chunk {
+	if len(units) == 0 {
+		return nil
+	}
+
+	const absoluteMaxSize = 7500
+
+	ht := newHeaderTracker()
+
+	var chunks []Chunk
+	var current []splitUnit
+	curLen := 0
+
+	for _, u := range units {
+		uLen := runeLen(u.text)
+
+		// If this single unit exceeds absolute max, force split it further
+		if uLen > absoluteMaxSize {
+			// Flush current chunk if any
+			if len(current) > 0 {
+				chunks = append(chunks, buildChunk(current, len(chunks)))
+				current = nil
+				curLen = 0
+			}
+
+			// Update header state even for oversized units
+			ht.update(u.text)
+
+			// Split this oversized unit into smaller chunks
+			runes := []rune(u.text)
+			offset := 0
+			for offset < len(runes) {
+				chunkEnd := offset + absoluteMaxSize
+				if chunkEnd > len(runes) {
+					chunkEnd = len(runes)
+				} else {
+					for i := chunkEnd - 1; i > offset && i > chunkEnd-200; i-- {
+						if runes[i] == '\n' || runes[i] == ' ' {
+							chunkEnd = i + 1
+							break
+						}
+					}
+				}
+
+				chunkText := string(runes[offset:chunkEnd])
+				chunks = append(chunks, Chunk{
+					Content: chunkText,
+					Seq:     len(chunks),
+					Start:   u.start + offset,
+					End:     u.start + chunkEnd,
+				})
+				offset = chunkEnd
+			}
+			continue
+		}
+
+		// Update header tracking
+		ht.update(u.text)
+		// Flush at table boundary so the next table is not merged into a chunk
+		// that still carries the previous table's prepended header context.
+		if ht.headerEndedThisUnit && len(current) > 0 {
+			chunks = append(chunks, buildChunk(current, len(chunks)))
+			current = nil
+			curLen = 0
+		}
+		headers := ht.getHeaders()
+		headersLen := runeLen(headers)
+		if headersLen > chunkSize {
+			headers = ""
+			headersLen = 0
+		}
+
+		// If adding this unit (plus reserving space for headers in a potential
+		// next chunk) would exceed chunk size, flush the current chunk.
+		if curLen+uLen+headersLen > chunkSize && len(current) > 0 {
+			chunks = append(chunks, buildChunk(current, len(chunks)))
+
+			// Keep overlap from the end of current
+			current, curLen = computeOverlap(current, chunkOverlap, chunkSize, uLen)
+
+			// Shrink overlap further if needed to fit headers + next unit
+			if headers != "" && headersLen+uLen <= chunkSize {
+				for len(current) > 0 && curLen+uLen+headersLen > chunkSize {
+					curLen -= runeLen(current[0].text)
+					current = current[1:]
+				}
+
+				// Prepend headers if the column-name context is not already present
+				// in the overlap or the next unit being added.
+				overlapText := unitsText(current)
+				if !headerAlreadyPresent(headers, overlapText, u.text) &&
+					!headerColumnMismatch(headers, u.text) {
+					startPos := u.start
+					if len(current) > 0 {
+						startPos = current[0].start
+					}
+					hUnit := splitUnit{text: headers, start: startPos, end: startPos}
+					current = append([]splitUnit{hUnit}, current...)
+					curLen += headersLen
+				}
+			}
+		}
+
+		// Check if adding this unit would exceed absolute max
+		if curLen+uLen > absoluteMaxSize {
+			if len(current) > 0 {
+				chunks = append(chunks, buildChunk(current, len(chunks)))
+				current = nil
+				curLen = 0
+			}
+		}
+
+		current = append(current, u)
+		curLen += uLen
+	}
+
+	// Flush remaining
+	if len(current) > 0 {
+		chunks = append(chunks, buildChunk(current, len(chunks)))
+	}
+
+	return chunks
+}
+
+// unitsText concatenates the text of all units.
+func unitsText(units []splitUnit) string {
+	var sb strings.Builder
+	for _, u := range units {
+		sb.WriteString(u.text)
+	}
+	return sb.String()
+}
+
+// headerAlreadyPresent returns true if the column-name row from the header
+// is already present in the overlap or the next unit, preventing duplication.
+func headerAlreadyPresent(headers, overlapText, unitText string) bool {
+	// Fast path: full header already in overlap or unit
+	if strings.Contains(overlapText, headers) || strings.Contains(unitText, headers) {
+		return true
+	}
+
+	// Extract the column-name row (first meaningful non-separator line).
+	// For a rewritten header like "| col1 | col2 |\n| --- | --- |\n",
+	// the first line is the column names.
+	colRow := headerColumnRow(headers)
+	if colRow == "" {
+		return false
+	}
+
+	return strings.Contains(overlapText, colRow) || strings.Contains(unitText, colRow)
+}
+
+// headerColumnRow extracts the column-name line from a header string.
+// Returns empty string if the header has no meaningful column names.
+func headerColumnRow(header string) string {
+	for _, line := range strings.Split(header, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.Contains(line, "---") {
+			continue
+		}
+		// Skip lines that are only pipes/whitespace (empty header rows)
+		onlyPipes := true
+		for _, r := range line {
+			if r != '|' && r != ' ' && r != '\t' {
+				onlyPipes = false
+				break
+			}
+		}
+		if !onlyPipes {
+			return line
+		}
+	}
+	return ""
+}
+
+func buildChunk(units []splitUnit, seq int) Chunk {
+	var sb strings.Builder
+	for _, u := range units {
+		sb.WriteString(u.text)
+	}
+	return Chunk{
+		Content: sb.String(),
+		Seq:     seq,
+		Start:   units[0].start,
+		End:     units[len(units)-1].end,
+	}
+}
+
+// computeOverlap returns the semantic suffix to keep for overlap and its total
+// rune length.
+//
+// The configured overlap is a hard upper bound, not a raw character slice.
+// Boundary detection may inspect up to four additional runes immediately
+// before that tail window so a separator cut by the window boundary remains
+// visible. A candidate is eligible only when its last rune is at position -1
+// relative to the original window, or later, so retained content never exceeds
+// the overlap limit. Eligible boundaries use this priority:
+//
+//  1. paragraph break (\n\n / \r\n\r\n)
+//  2. line break (\n / \r\n)
+//  3. sentence end (。？！, ". ", "? ", "! ")
+//
+// Priority wins first; for boundaries with the same priority, the earliest
+// one in the window wins so the useful overlap is as large as possible. The
+// next chunk starts immediately after the selected separator. If the window
+// has no valid semantic boundary, no overlap is retained. This keeps overlap
+// deterministic and bounded while allowing a boundary inside a large split
+// unit (the previous implementation could only retain whole units and often
+// collapsed to zero overlap for ordinary paragraphs).
+const semanticOverlapLookbehind = 4 // rune length of the longest separator: \r\n\r\n
+
+func computeOverlap(current []splitUnit, chunkOverlap, chunkSize, nextLen int) ([]splitUnit, int) {
+	if chunkOverlap <= 0 {
+		return nil, 0
+	}
+
+	// Overlap is part of the next chunk's total size. When the incoming unit
+	// is already close to the chunk budget, shrink the search window rather
+	// than creating an oversized chunk.
+	maxOverlap := chunkOverlap
+	if remaining := chunkSize - nextLen; remaining < maxOverlap {
+		maxOverlap = remaining
+	}
+	if maxOverlap <= 0 {
+		return nil, 0
+	}
+
+	window := semanticOverlapWindow(current, maxOverlap+semanticOverlapLookbehind)
+	if len(window) == 0 {
+		return nil, 0
+	}
+
+	windowText := unitsText(window)
+	// Coordinates before originalWindowStart are the lookbehind region. With
+	// end-exclusive offsets, requiring boundaryEnd >= originalWindowStart is
+	// equivalent to requiring the separator's final rune to be at -1 or later.
+	originalWindowStart := runeLen(windowText) - maxOverlap
+	if originalWindowStart < 0 {
+		originalWindowStart = 0
+	}
+	boundaryEnd, ok := findSemanticOverlapBoundaryEndingAtOrAfter(windowText, originalWindowStart)
+	if !ok {
+		return nil, 0
+	}
+
+	overlap := trimUnitsPrefix(window, boundaryEnd)
+	overlapLen := 0
+	for _, u := range overlap {
+		overlapLen += runeLen(u.text)
+	}
+	if overlapLen <= 0 || overlapLen > maxOverlap || strings.TrimSpace(unitsText(overlap)) == "" {
+		return nil, 0
+	}
+	return overlap, overlapLen
+}
+
+// semanticOverlapWindow returns at most maxLen source-backed runes from the
+// tail of current. It may slice inside the first retained splitUnit so a
+// semantic boundary inside a large paragraph remains discoverable. Synthetic
+// zero-width units (for example repeated table headers) form a hard barrier:
+// crossing them would break the Start/End-to-Content source invariant.
+func semanticOverlapWindow(current []splitUnit, maxLen int) []splitUnit {
+	if maxLen <= 0 || len(current) == 0 {
+		return nil
+	}
+
+	remaining := maxLen
+	reversed := make([]splitUnit, 0, len(current))
+	for i := len(current) - 1; i >= 0 && remaining > 0; i-- {
+		u := current[i]
+		uLen := runeLen(u.text)
+		if uLen == 0 {
+			continue
+		}
+		// Header markers contain generated text but occupy no source range.
+		// Do not include or cross them when deriving an overlap suffix.
+		if u.start == u.end || u.end-u.start != uLen {
+			break
+		}
+
+		if uLen <= remaining {
+			reversed = append(reversed, u)
+			remaining -= uLen
+			continue
+		}
+
+		runes := []rune(u.text)
+		start := uLen - remaining
+		reversed = append(reversed, splitUnit{
+			text:  string(runes[start:]),
+			start: u.start + start,
+			end:   u.end,
+		})
+		remaining = 0
+	}
+
+	if len(reversed) == 0 {
+		return nil
+	}
+	window := make([]splitUnit, len(reversed))
+	for i := range reversed {
+		window[len(reversed)-1-i] = reversed[i]
+	}
+	return window
+}
+
+type semanticOverlapBoundary struct {
+	start    int
+	end      int
+	priority int
+}
+
+// findSemanticOverlapBoundary returns the rune offset immediately after the
+// selected separator. Boundaries inside protected Markdown/code/math regions
+// are ignored, and a boundary is invalid when only whitespace follows it.
+func findSemanticOverlapBoundary(text string) (int, bool) {
+	return findSemanticOverlapBoundaryEndingAtOrAfter(text, 0)
+}
+
+// findSemanticOverlapBoundaryEndingAtOrAfter applies the normal priority and
+// earliest-position rules only to candidates whose end-exclusive rune offset
+// is at least minEnd. Filtering before comparison prevents an earlier but
+// ineligible lookbehind separator from hiding a later eligible boundary.
+func findSemanticOverlapBoundaryEndingAtOrAfter(text string, minEnd int) (int, bool) {
+	runes := []rune(text)
+	if len(runes) == 0 {
+		return 0, false
+	}
+	if minEnd < 0 {
+		minEnd = 0
+	}
+	if minEnd > len(runes) {
+		return 0, false
+	}
+
+	protected := protectedSpansRune(text, protectedSpans(text))
+	insideProtected := func(pos int) bool {
+		for _, p := range protected {
+			if pos < p.start {
+				return false
+			}
+			if pos >= p.start && pos < p.end {
+				return true
+			}
+		}
+		return false
+	}
+	hasMeaningfulTail := func(end int) bool {
+		return end >= 0 && end < len(runes) && strings.TrimSpace(string(runes[end:])) != ""
+	}
+
+	var best semanticOverlapBoundary
+	found := false
+	consider := func(start, end, priority int) {
+		if start < 0 || end <= start || end < minEnd || end > len(runes) ||
+			insideProtected(start) || !hasMeaningfulTail(end) {
+			return
+		}
+		candidate := semanticOverlapBoundary{start: start, end: end, priority: priority}
+		if !found || candidate.priority < best.priority ||
+			(candidate.priority == best.priority && candidate.start < best.start) {
+			best = candidate
+			found = true
+		}
+	}
+
+	// Mark paragraph-break runes so their component newlines are not also
+	// emitted as lower-priority line-break candidates.
+	paragraphRune := make([]bool, len(runes))
+	for i := 0; i < len(runes); i++ {
+		switch {
+		case i+3 < len(runes) && runes[i] == '\r' && runes[i+1] == '\n' &&
+			runes[i+2] == '\r' && runes[i+3] == '\n':
+			consider(i, i+4, 1)
+			for j := i; j < i+4; j++ {
+				paragraphRune[j] = true
+			}
+			i += 3
+		case i+1 < len(runes) && runes[i] == '\n' && runes[i+1] == '\n':
+			consider(i, i+2, 1)
+			paragraphRune[i], paragraphRune[i+1] = true, true
+			i++
+		}
+	}
+
+	for i := 0; i < len(runes); i++ {
+		if paragraphRune[i] {
+			continue
+		}
+		if runes[i] == '\r' && i+1 < len(runes) && runes[i+1] == '\n' && !paragraphRune[i+1] {
+			consider(i, i+2, 2)
+			i++
+			continue
+		}
+		if runes[i] == '\n' {
+			consider(i, i+1, 2)
+		}
+	}
+
+	for i := 0; i < len(runes); i++ {
+		switch runes[i] {
+		case '。', '？', '！':
+			consider(i, i+1, 3)
+		case '.', '?', '!':
+			if i+1 < len(runes) && runes[i+1] == ' ' {
+				consider(i, i+2, 3)
+			}
+		}
+	}
+
+	if !found {
+		return 0, false
+	}
+	return best.end, true
+}
+
+// trimUnitsPrefix removes prefixLen source runes while preserving the
+// remaining units' original positions. prefixLen is relative to unitsText.
+func trimUnitsPrefix(units []splitUnit, prefixLen int) []splitUnit {
+	if prefixLen <= 0 {
+		out := make([]splitUnit, len(units))
+		copy(out, units)
+		return out
+	}
+
+	remaining := prefixLen
+	out := make([]splitUnit, 0, len(units))
+	for _, u := range units {
+		uLen := runeLen(u.text)
+		if remaining >= uLen {
+			remaining -= uLen
+			continue
+		}
+		if remaining > 0 {
+			runes := []rune(u.text)
+			u.text = string(runes[remaining:])
+			u.start += remaining
+			remaining = 0
+		}
+		out = append(out, u)
+	}
+	return out
+}
+
+// ParentChildResult holds the two-level chunking output.
+// Parent chunks provide context (large window), child chunks are used for
+// embedding/retrieval (small window). Each child carries its ParentIndex so
+// the caller can wire up ParentChunkID after DB insertion.
+type ParentChildResult struct {
+	Parents  []Chunk
+	Children []ChildChunk
+}
+
+// ChildChunk extends Chunk with a reference to its parent.
+type ChildChunk struct {
+	Chunk
+	ParentIndex int // index into ParentChildResult.Parents
+}
+
+// SplitTextParentChild performs two-level chunking:
+//  1. Split text into large parent chunks (parentCfg).
+//  2. Split each parent into smaller child chunks (childCfg) for embedding.
+//
+// The child Seq is globally unique across the entire document.
+func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult {
+	parents := SplitText(text, parentCfg)
+	if len(parents) == 0 {
+		return ParentChildResult{}
+	}
+
+	var newParents []Chunk
+	var children []ChildChunk
+	childSeq := 0
+	for _, parent := range parents {
+		subs := SplitText(parent.Content, childCfg)
+
+		parentIndex := -1
+		if len(subs) > 1 || (len(subs) == 1 && subs[0].Content != parent.Content) {
+			parentIndex = len(newParents)
+			newParents = append(newParents, parent)
+		}
+
+		for _, sub := range subs {
+			// Adjust offsets: sub positions are relative to parent content,
+			// shift to document-level offsets.
+			// Use additive shift (not Content-length based) so that chunks with
+			// prepended context headers keep correct positional tracking.
+			sub.Seq = childSeq
+			sub.Start += parent.Start
+			sub.End += parent.Start
+			children = append(children, ChildChunk{
+				Chunk:       sub,
+				ParentIndex: parentIndex,
+			})
+			childSeq++
+		}
+	}
+	return ParentChildResult{Parents: newParents, Children: children}
+}
+
+// ExtractImageRefs extracts markdown image references from text.
+// The URL group supports one level of balanced parentheses so that URLs
+// like https://example.com/item_(abc)/123 are captured in full.
+var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`)
+
+func ExtractImageRefs(text string) []ImageRef {
+	text = docparser.UnwrapLinkedImages(text)
+	matches := imageRefPattern.FindAllStringSubmatchIndex(text, -1)
+	var refs []ImageRef
+	for _, m := range matches {
+		refs = append(refs, ImageRef{
+			OriginalRef: text[m[4]:m[5]], // group 2: URL
+			AltText:     text[m[2]:m[3]], // group 1: alt text
+			Start:       m[0],
+			End:         m[1],
+		})
+	}
+	return refs
+}