WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/application/service/metric/bleu.go

@@ -0,0 +1,165 @@
+package metric
+
+// references: https://github.com/waygo/bleu
+
+// Package bleu implements the BLEU method, which is used to evaluate
+// the quality of machine translation. [1]
+//
+// The code in this package was largely ported from the corresponding package
+// in Python NLTK. [2]
+//
+// [1] Papineni, Kishore, et al. "BLEU: a method for automatic evaluation of
+//     machine translation." Proceedings of the 40th annual meeting on
+//     association for computational linguistics. Association for Computational
+//     Linguistics, 2002.
+//
+// [2] http://www.nltk.org/_modules/nltk/align/bleu.html
+
+import (
+	"encoding/json"
+	"log"
+	"math"
+	"strings"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+type BLEUMetric struct {
+	smoothing bool
+	weights   BLEUWeight
+}
+
+func NewBLEUMetric(smoothing bool, weights BLEUWeight) *BLEUMetric {
+	return &BLEUMetric{smoothing: smoothing, weights: weights}
+}
+
+type Sentence []string
+
+type BLEUWeight []float64
+
+var (
+	BLEU1Gram BLEUWeight = []float64{1.0, 0.0, 0.0, 0.0}
+	BLEU2Gram BLEUWeight = []float64{0.5, 0.5, 0.0, 0.0}
+	BLEU3Gram BLEUWeight = []float64{0.33, 0.33, 0.33, 0.0}
+	BLEU4Gram BLEUWeight = []float64{0.25, 0.25, 0.25, 0.25}
+)
+
+func (b *BLEUMetric) Compute(metricInput *types.MetricInput) float64 {
+	candidate := splitIntoWords(splitSentences(metricInput.GeneratedTexts))
+	references := []Sentence{splitIntoWords(splitSentences(metricInput.GeneratedGT))}
+
+	for i := range candidate {
+		candidate[i] = strings.ToLower(candidate[i])
+	}
+
+	for i := range references {
+		for u := range references[i] {
+			references[i][u] = strings.ToLower(references[i][u])
+		}
+	}
+
+	ps := make([]float64, len(b.weights))
+	for i := range b.weights {
+		ps[i] = b.modifiedPrecision(candidate, references, i+1)
+	}
+
+	s := 0.0
+	overlap := 0
+	for i := range b.weights {
+		w := b.weights[i]
+		pn := ps[i]
+		if pn > 0.0 {
+			overlap++
+			s += w * math.Log(pn)
+		}
+	}
+
+	if overlap == 0 {
+		return 0
+	}
+
+	bp := b.brevityPenalty(candidate, references)
+	return bp * math.Exp(s)
+}
+
+type phrase []string
+
+func (p phrase) String() string {
+	b, err := json.Marshal(p)
+	if err != nil {
+		log.Fatal("encode error:", err)
+	}
+	return string(b)
+}
+
+func (b *BLEUMetric) getNphrase(s Sentence, n int) []phrase {
+	nphrase := []phrase{}
+	for i := 0; i < len(s)-n+1; i++ {
+		nphrase = append(nphrase, phrase(s[i:i+n]))
+	}
+	return nphrase
+}
+
+func (b *BLEUMetric) countNphrase(nphrase []phrase) map[string]int {
+	counts := map[string]int{}
+	for _, gram := range nphrase {
+		counts[gram.String()]++
+	}
+	return counts
+}
+
+func (b *BLEUMetric) modifiedPrecision(candidate Sentence, references []Sentence, n int) float64 {
+	nphrase := b.getNphrase(candidate, n)
+	if len(nphrase) == 0 {
+		return 0.0
+	}
+
+	counts := b.countNphrase(nphrase)
+
+	if len(counts) == 0 {
+		return 0.0
+	}
+
+	maxCounts := map[string]int{}
+	for i := range references {
+		referenceCounts := b.countNphrase(b.getNphrase(references[i], n))
+		for ngram := range counts {
+			if v, ok := maxCounts[ngram]; !ok {
+				maxCounts[ngram] = referenceCounts[ngram]
+			} else if v < referenceCounts[ngram] {
+				maxCounts[ngram] = referenceCounts[ngram]
+			}
+		}
+	}
+
+	clippedCounts := map[string]int{}
+	for ngram, count := range counts {
+		clippedCounts[ngram] = min(count, maxCounts[ngram])
+	}
+
+	smoothingFactor := 0.0
+	if b.smoothing {
+		smoothingFactor = 1.0
+	}
+	return (float64(sum(clippedCounts)) + smoothingFactor) / (float64(sum(counts)) + smoothingFactor)
+}
+
+func (b *BLEUMetric) brevityPenalty(candidate Sentence, references []Sentence) float64 {
+	c := len(candidate)
+	refLens := []int{}
+	for i := range references {
+		refLens = append(refLens, len(references[i]))
+	}
+	minDiffInd, minDiff := 0, -1
+	for i := range refLens {
+		if minDiff == -1 || abs(refLens[i]-c) < minDiff {
+			minDiffInd = i
+			minDiff = abs(refLens[i] - c)
+		}
+	}
+	r := refLens[minDiffInd]
+	if c > r {
+		return 1
+	}
+	return math.Exp(float64(1 - float64(r)/float64(c)))
+}

internal/application/service/metric/common.go

@@ -0,0 +1,133 @@
+package metric
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func sum(m map[string]int) int {
+	s := 0
+	for _, v := range m {
+		s += v
+	}
+	return s
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func abs(a int) int {
+	if a < 0 {
+		return -a
+	}
+	return a
+}
+
+func max(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func splitSentences(text string) []string {
+	// 编译正则表达式（匹配中文句号或英文句号）
+	re := regexp.MustCompile(`([。.])`)
+
+	// 分割文本并保留分隔符用于定位
+	split := re.Split(text, -1)
+
+	var sentences []string
+	current := strings.Builder{}
+
+	for i, s := range split {
+		// 交替获取文本段和分隔符（奇数为分隔符）
+		if i%2 == 0 {
+			current.WriteString(s)
+		} else {
+			// 当遇到分隔符时，完成当前句子
+			if current.Len() > 0 {
+				sentence := strings.TrimSpace(current.String())
+				if sentence != "" {
+					sentences = append(sentences, sentence)
+				}
+				current.Reset()
+			}
+		}
+	}
+
+	// 处理最后一个无分隔符的文本段
+	if remaining := strings.TrimSpace(current.String()); remaining != "" {
+		sentences = append(sentences, remaining)
+	}
+
+	return sentences
+}
+
+func splitIntoWords(sentences []string) []string {
+	// 正则匹配中英文段落（中文块、英文块、其他字符）
+	re := regexp.MustCompile(`([\p{Han}]+)|([a-zA-Z0-9_.,!?]+)|(\p{P})`)
+
+	var tokens []string
+	for _, text := range sentences {
+		matches := re.FindAllStringSubmatch(text, -1)
+
+		for _, groups := range matches {
+			chineseBlock := groups[1]
+			englishBlock := groups[2]
+			punctuation := groups[3]
+
+			switch {
+			case chineseBlock != "": // 处理中文部分
+				words := types.Jieba.Cut(chineseBlock, true)
+				tokens = append(tokens, words...)
+			case englishBlock != "": // 处理英文部分
+				engTokens := strings.Fields(englishBlock)
+				tokens = append(tokens, engTokens...)
+			case punctuation != "": // 保留标点符号
+				tokens = append(tokens, punctuation)
+			}
+		}
+	}
+	return tokens
+}
+
+func ToSet[T comparable](li []T) map[T]struct{} {
+	res := make(map[T]struct{}, len(li))
+	for _, v := range li {
+		res[v] = struct{}{}
+	}
+	return res
+}
+
+func SliceMap[T any, Y any](li []T, fn func(T) Y) []Y {
+	res := make([]Y, len(li))
+	for i, v := range li {
+		res[i] = fn(v)
+	}
+	return res
+}
+
+func Hit[T comparable](li []T, set map[T]struct{}) int {
+	count := 0
+	for _, v := range li {
+		if _, exist := set[v]; exist {
+			count++
+		}
+	}
+	return count
+}
+
+func Fold[T any, Y any](slice []T, initial Y, f func(Y, T) Y) Y {
+	accumulator := initial
+	for _, item := range slice {
+		accumulator = f(accumulator, item)
+	}
+	return accumulator
+}

internal/application/service/metric/map.go

@@ -0,0 +1,70 @@
+package metric
+
+import (
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// MAPMetric calculates Mean Average Precision for retrieval evaluation
+type MAPMetric struct{}
+
+// NewMAPMetric creates a new MAPMetric instance
+func NewMAPMetric() *MAPMetric {
+	return &MAPMetric{}
+}
+
+// Compute calculates the Mean Average Precision score
+func (m *MAPMetric) Compute(metricInput *types.MetricInput) float64 {
+	// Convert ground truth to sets for efficient lookup
+	gts := metricInput.RetrievalGT
+	ids := metricInput.RetrievalIDs
+
+	// Create sets of relevant document IDs for each query
+	gtSets := make([]map[int]struct{}, len(gts))
+	for i, gt := range gts {
+		gtSets[i] = make(map[int]struct{})
+		for _, docID := range gt {
+			gtSets[i][docID] = struct{}{}
+		}
+	}
+
+	var apSum float64 // Sum of average precision for all queries
+
+	// Calculate average precision for each query
+	for _, gtSet := range gtSets {
+		// Mark which predicted documents are relevant
+		predHits := make([]bool, len(ids))
+		for i, predID := range ids {
+			if _, ok := gtSet[predID]; ok {
+				predHits[i] = true
+			} else {
+				predHits[i] = false
+			}
+		}
+
+		var (
+			ap       float64 // Average precision for current query
+			hitCount int     // Number of relevant documents found
+		)
+
+		// Calculate precision at each rank position
+		for k := 0; k < len(predHits); k++ {
+			if predHits[k] {
+				hitCount++
+				// Precision at k: relevant docs found up to k / k
+				ap += float64(hitCount) / float64(k+1)
+			}
+		}
+		// Normalize by number of relevant documents
+		if hitCount > 0 {
+			ap /= float64(hitCount)
+		}
+		apSum += ap
+	}
+
+	// Handle case with no ground truth
+	if len(gtSets) == 0 {
+		return 0
+	}
+	// Return mean of average precision across all queries
+	return apSum / float64(len(gtSets))
+}

internal/application/service/metric/map_test.go

@@ -0,0 +1,85 @@
+package metric
+
+import (
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestMAPMetric_Compute(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    *types.MetricInput
+		expected float64
+	}{
+		{
+			name: "total match",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{2, 4, 6}},
+				RetrievalIDs: []int{2, 4, 6},
+			},
+			expected: 1.0,
+		},
+		{
+			name: "no match",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 2}},
+				RetrievalIDs: []int{3, 4},
+			},
+			expected: 0.0,
+		},
+		{
+			name: "partial match",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 2, 3}},
+				RetrievalIDs: []int{2, 5, 1, 3},
+			},
+			// AP = (1/1 + 2/3 + 3/4)/3 ≈ 0.80555555
+			expected: 0.8055555555555555,
+		},
+		{
+			name: "empty ground truth",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{},
+				RetrievalIDs: []int{1, 2},
+			},
+			expected: 0.0,
+		},
+		{
+			name: "multiple queries",
+			input: &types.MetricInput{
+				RetrievalGT: [][]int{
+					{1, 2},
+					{3, 4},
+				},
+				RetrievalIDs: []int{1, 3, 2, 4},
+			},
+			// Query1 AP: (1/1 + 2/3)/2 ≈ 0.8333
+			// Query2 AP: (1/2 + 2/4)/2 = 0.5
+			// MAP: (0.8333 + 0.5)/2 ≈ 0.6667
+			expected: 0.6666666666666666,
+		},
+	}
+
+	metric := NewMAPMetric()
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := metric.Compute(tt.input)
+			if !almostEqual(got, tt.expected, 1e-6) {
+				t.Errorf("Compute() = %v, want %v", got, tt.expected)
+			}
+		})
+	}
+}
+
+// Helper function to compare floating point numbers with tolerance
+func almostEqual(a, b, tolerance float64) bool {
+	if a == b {
+		return true
+	}
+	diff := a - b
+	if diff < 0 {
+		diff = -diff
+	}
+	return diff < tolerance
+}

internal/application/service/metric/mrr.go

@@ -0,0 +1,48 @@
+package metric
+
+import (
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// MRRMetric calculates Mean Reciprocal Rank for retrieval evaluation
+type MRRMetric struct{}
+
+// NewMRRMetric creates a new MRRMetric instance
+func NewMRRMetric() *MRRMetric {
+	return &MRRMetric{}
+}
+
+// Compute calculates the Mean Reciprocal Rank score
+func (m *MRRMetric) Compute(metricInput *types.MetricInput) float64 {
+	// Get ground truth and predicted IDs
+	gts := metricInput.RetrievalGT
+	ids := metricInput.RetrievalIDs
+
+	// Convert ground truth to sets for efficient lookup
+	gtSets := make([]map[int]struct{}, len(gts))
+	for i, gt := range gts {
+		gtSets[i] = make(map[int]struct{})
+		for _, docID := range gt {
+			gtSets[i][docID] = struct{}{}
+		}
+	}
+
+	var sumRR float64 // Sum of reciprocal ranks
+	// Calculate reciprocal rank for each query
+	for _, gtSet := range gtSets {
+		// Find first relevant document in results
+		for i, predID := range ids {
+			if _, ok := gtSet[predID]; ok {
+				// Reciprocal rank is 1/position (1-based)
+				sumRR += 1.0 / float64(i+1)
+				break // Only consider first relevant document
+			}
+		}
+	}
+	// Handle case with no ground truth
+	if len(gtSets) == 0 {
+		return 0
+	}
+	// Return mean of reciprocal ranks
+	return sumRR / float64(len(gtSets))
+}

internal/application/service/metric/mrr_test.go

@@ -0,0 +1,72 @@
+package metric
+
+import (
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestMRRMetric_Compute(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    *types.MetricInput
+		expected float64
+	}{
+		{
+			name: "perfect match - first position",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 2}},
+				RetrievalIDs: []int{1, 2, 3},
+			},
+			// RR = 1/1 = 1.0
+			expected: 1.0,
+		},
+		{
+			name: "match at second position",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 2}},
+				RetrievalIDs: []int{3, 1, 2},
+			},
+			// RR = 1/2 = 0.5
+			expected: 0.5,
+		},
+		{
+			name: "no match",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 2}},
+				RetrievalIDs: []int{3, 4},
+			},
+			expected: 0.0,
+		},
+		{
+			name: "multiple queries",
+			input: &types.MetricInput{
+				RetrievalGT: [][]int{
+					{1, 2}, // RR = 1/1 = 1.0
+					{3, 4}, // RR = 1/2 = 0.5
+				},
+				RetrievalIDs: []int{1, 3, 2, 4},
+			},
+			// MRR = (1.0 + 0.5)/2 = 0.75
+			expected: 0.75,
+		},
+		{
+			name: "empty ground truth",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{},
+				RetrievalIDs: []int{1, 2},
+			},
+			expected: 0.0,
+		},
+	}
+
+	metric := NewMRRMetric()
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := metric.Compute(tt.input)
+			if !almostEqual(got, tt.expected, 1e-6) {
+				t.Errorf("Compute() = %v, want %v", got, tt.expected)
+			}
+		})
+	}
+}

internal/application/service/metric/ndcg.go

@@ -0,0 +1,78 @@
+package metric
+
+import (
+	"math"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// NDCGMetric calculates Normalized Discounted Cumulative Gain
+type NDCGMetric struct {
+	k int // Top k results to consider
+}
+
+// NewNDCGMetric creates a new NDCGMetric instance with given k value
+func NewNDCGMetric(k int) *NDCGMetric {
+	return &NDCGMetric{k: k}
+}
+
+// Compute calculates the NDCG score
+func (n *NDCGMetric) Compute(metricInput *types.MetricInput) float64 {
+	gts := metricInput.RetrievalGT
+	ids := metricInput.RetrievalIDs
+
+	// Limit results to top k
+	if len(ids) > n.k {
+		ids = ids[:n.k]
+	}
+
+	// Create set of relevant documents and count total relevant
+	gtSets := make(map[int]struct{}, len(gts))
+	countGt := 0
+	for _, gt := range gts {
+		countGt += len(gt)
+		for _, g := range gt {
+			gtSets[g] = struct{}{}
+		}
+	}
+
+	// Assign relevance scores (1 for relevant, 0 otherwise)
+	relevanceScores := make(map[int]int)
+	for _, docID := range ids {
+		if _, exist := gtSets[docID]; exist {
+			relevanceScores[docID] = 1
+		} else {
+			relevanceScores[docID] = 0
+		}
+	}
+
+	// Calculate DCG (Discounted Cumulative Gain)
+	var dcg float64
+	for i, docID := range ids {
+		dcg += (math.Pow(2, float64(relevanceScores[docID])) - 1) / math.Log2(float64(i+2))
+	}
+
+	// Create ideal ranking (all relevant docs first)
+	idealLen := min(countGt, len(ids))
+	idealPred := make([]int, len(ids))
+	for i := 0; i < len(ids); i++ {
+		if i < idealLen {
+			idealPred[i] = 1
+		} else {
+			idealPred[i] = 0
+		}
+	}
+
+	// Calculate IDCG (Ideal DCG)
+	var idcg float64
+	for i, relevance := range idealPred {
+		idcg += float64(relevance) / math.Log2(float64(i+2))
+	}
+
+	// Handle division by zero case
+	if idcg == 0 {
+		return 0
+	}
+	// NDCG = DCG / IDCG
+	return dcg / idcg
+}

internal/application/service/metric/precision.go

@@ -0,0 +1,44 @@
+package metric
+
+import (
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// PrecisionMetric calculates precision for retrieval evaluation
+type PrecisionMetric struct{}
+
+// NewPrecisionMetric creates a new PrecisionMetric instance
+func NewPrecisionMetric() *PrecisionMetric {
+	return &PrecisionMetric{}
+}
+
+// Compute calculates the precision score
+func (r *PrecisionMetric) Compute(metricInput *types.MetricInput) float64 {
+	// Get ground truth and predicted IDs
+	gts := metricInput.RetrievalGT
+	ids := metricInput.RetrievalIDs
+
+	// Convert ground truth to sets for efficient lookup
+	gtSets := SliceMap(gts, ToSet)
+	// Precision = retrieved items that are in ground truth / total retrieved items
+	// In the test cases, ground truth is a list of sets.
+	// We compute precision per ground truth set, and average them.
+	// But actually, precision is typically |retrieved ∩ relevant| / |retrieved|.
+	// Let's sum the precisions for each ground truth set and average them.
+
+	if len(gts) == 0 {
+		return 0.0
+	}
+
+	if len(ids) == 0 {
+		return 0.0
+	}
+
+	var totalPrecision float64
+	for _, gtSet := range gtSets {
+		hits := Hit(ids, gtSet)
+		totalPrecision += float64(hits) / float64(len(ids))
+	}
+
+	return totalPrecision / float64(len(gts))
+}

internal/application/service/metric/precision_test.go

@@ -0,0 +1,67 @@
+package metric
+
+import (
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestPrecisionMetric_Compute(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    *types.MetricInput
+		expected float64
+	}{
+		{
+			name: "perfect match",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 3, 5}},
+				RetrievalIDs: []int{1, 3, 5},
+			},
+			expected: 1.0,
+		},
+		{
+			name: "half match",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 2, 3}},
+				RetrievalIDs: []int{1, 4, 2},
+			},
+			expected: 0.6666666666666666,
+		},
+		{
+			name: "no match",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 2, 3}},
+				RetrievalIDs: []int{4, 5, 6},
+			},
+			expected: 0.0,
+		},
+		{
+			name: "empty retrieval",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 2, 3}},
+				RetrievalIDs: []int{},
+			},
+			expected: 0.0,
+		},
+		{
+			name: "multiple ground truths",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 2}, {3, 4}},
+				RetrievalIDs: []int{1, 3, 5},
+			},
+			expected: 0.3333333333333333,
+		},
+	}
+
+	pm := NewPrecisionMetric()
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := pm.Compute(tt.input)
+			if got != tt.expected {
+				t.Errorf("Compute() = %v, want %v", got, tt.expected)
+			}
+		})
+	}
+}

internal/application/service/metric/recall.go

@@ -0,0 +1,41 @@
+package metric
+
+import (
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// RecallMetric calculates recall for retrieval evaluation
+type RecallMetric struct{}
+
+// NewRecallMetric creates a new RecallMetric instance
+func NewRecallMetric() *RecallMetric {
+	return &RecallMetric{}
+}
+
+// Compute calculates the recall score
+func (r *RecallMetric) Compute(metricInput *types.MetricInput) float64 {
+	// Get ground truth and predicted IDs
+	gts := metricInput.RetrievalGT
+	ids := metricInput.RetrievalIDs
+
+	// Convert ground truth to sets for efficient lookup
+	gtSets := SliceMap(gts, ToSet)
+	if len(gtSets) == 0 {
+		return 0.0
+	}
+
+	if len(ids) == 0 {
+		return 0.0
+	}
+
+	var totalRecall float64
+	for _, gtSet := range gtSets {
+		hits := Hit(ids, gtSet)
+		if len(gtSet) > 0 {
+			totalRecall += float64(hits) / float64(len(gtSet))
+		}
+	}
+
+	// Recall = average recall across all ground truth sets
+	return totalRecall / float64(len(gtSets))
+}

internal/application/service/metric/recall_test.go

@@ -0,0 +1,70 @@
+package metric
+
+import (
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestRecallMetric_Compute(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    *types.MetricInput
+		expected float64
+	}{
+		{
+			name: "perfect recall - all ground truth retrieved",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 2, 3}},
+				RetrievalIDs: []int{1, 2, 3, 4},
+			},
+			expected: 1.0,
+		},
+		{
+			name: "partial recall - some ground truth retrieved",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 2, 3}, {4, 5}},
+				RetrievalIDs: []int{1, 4, 6},
+			},
+			// 命中2个ground truth集合中的元素(a和d)
+			expected: 0.41666666666666663, // (1/3 + 1/2) / 2 = 0.41666 (每个ground truth集合只要命中一个就算召回)
+
+		},
+		{
+			name: "no recall - no ground truth retrieved",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 2, 3}},
+				RetrievalIDs: []int{4, 5, 6},
+			},
+			expected: 0.0,
+		},
+		{
+			name: "empty retrieval list",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 2, 3}},
+				RetrievalIDs: []int{},
+			},
+			expected: 0.0,
+		},
+		{
+			name: "multiple ground truth sets",
+			input: &types.MetricInput{
+				RetrievalGT:  [][]int{{1, 2}, {3, 4}, {5, 6}},
+				RetrievalIDs: []int{1, 3, 7},
+			},
+			// 命中了前两个ground truth集合(a和c)
+			expected: 0.3333333333333333, // 1/3≈0.333...
+		},
+	}
+
+	rm := NewRecallMetric()
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := rm.Compute(tt.input)
+			if got != tt.expected {
+				t.Errorf("Compute() = %v, want %v", got, tt.expected)
+			}
+		})
+	}
+}

internal/application/service/metric/rouge.go

@@ -0,0 +1,72 @@
+package metric
+
+import "github.com/Tencent/WeKnora/internal/types"
+
+// reference: https://github.com/dd-Rebecca/rouge
+
+// RougeMetric implements ROUGE (Recall-Oriented Understudy for Gisting Evaluation) metrics
+// for evaluating text summarization quality by comparing generated text to reference text
+type RougeMetric struct {
+	exclusive bool   // Whether to use exclusive matching mode
+	metric    string // ROUGE metric type (e.g. "rouge-1", "rouge-l")
+	stats     string // Statistic to return (e.g. "f", "p", "r")
+}
+
+// AvailableMetrics defines all supported ROUGE variants and their calculation functions
+var AvailableMetrics = map[string]func([]string, []string, bool) map[string]float64{
+	"rouge-1": func(hyp, ref []string, exclusive bool) map[string]float64 {
+		return rougeN(hyp, ref, 1, false, exclusive) // Unigram-based ROUGE
+	},
+	"rouge-2": func(hyp, ref []string, exclusive bool) map[string]float64 {
+		return rougeN(hyp, ref, 2, false, exclusive) // Bigram-based ROUGE
+	},
+	"rouge-3": func(hyp, ref []string, exclusive bool) map[string]float64 {
+		return rougeN(hyp, ref, 3, false, exclusive) // Trigram-based ROUGE
+	},
+	"rouge-4": func(hyp, ref []string, exclusive bool) map[string]float64 {
+		return rougeN(hyp, ref, 4, false, exclusive) // 4-gram based ROUGE
+	},
+	"rouge-5": func(hyp, ref []string, exclusive bool) map[string]float64 {
+		return rougeN(hyp, ref, 5, false, exclusive) // 5-gram based ROUGE
+	},
+	"rouge-l": func(hyp, ref []string, exclusive bool) map[string]float64 {
+		return rougeLSummaryLevel(hyp, ref, false, exclusive) // Longest common subsequence based ROUGE
+	},
+}
+
+// NewRougeMetric creates a new ROUGE metric calculator
+func NewRougeMetric(exclusive bool, metrics, stats string) *RougeMetric {
+	r := &RougeMetric{
+		exclusive: exclusive,
+		metric:    metrics,
+		stats:     stats,
+	}
+	return r
+}
+
+// Compute calculates the ROUGE score between generated text and reference text
+func (r *RougeMetric) Compute(metricInput *types.MetricInput) float64 {
+	hyps := []string{metricInput.GeneratedTexts} // Generated/hypothesis text
+	refs := []string{metricInput.GeneratedGT}    // Reference/ground truth text
+
+	scores := 0.0
+	count := 0
+
+	// Calculate scores for each hypothesis-reference pair
+	for i := 0; i < len(hyps); i++ {
+		hyp := splitSentences(hyps[i]) // Split into sentences
+		ref := splitSentences(refs[i])
+
+		// Get appropriate ROUGE calculation function
+		fn := AvailableMetrics[r.metric]
+		sc := fn(hyp, ref, r.exclusive)
+		scores += sc[r.stats] // Accumulate specified statistic (f1/precision/recall)
+
+		count++
+	}
+
+	if count == 0 {
+		return 0 // Avoid division by zero
+	}
+	return scores / float64(count) // Return average score
+}

internal/application/service/metric/rouge_score.go

@@ -0,0 +1,250 @@
+package metric
+
+/*
+# -*- coding: utf-8 -*-
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""ROUGE Metric Implementation
+
+This is a very slightly version of:
+https://github.com/pltrdy/seq2seq/blob/master/seq2seq/metrics/rouge.py
+
+---
+
+ROUGe metric implementation.
+
+This is a modified and slightly extended verison of
+https://github.com/miso-belica/sumy/blob/dev/sumy/evaluation/rouge.py.
+*/
+
+import (
+	"strings"
+)
+
+type Ngrams struct {
+	ngrams    map[string]int
+	exclusive bool
+}
+
+func NewNgrams(exclusive bool) *Ngrams {
+	return &Ngrams{ngrams: make(map[string]int), exclusive: exclusive}
+}
+
+func (n *Ngrams) Add(o string) {
+	if n.exclusive {
+		n.ngrams[o] = 1
+	} else {
+		n.ngrams[o]++
+	}
+}
+
+func (n *Ngrams) Len() int {
+	return len(n.ngrams)
+}
+
+func (n *Ngrams) Intersection(o *Ngrams) *Ngrams {
+	intersection := NewNgrams(n.exclusive)
+	for k := range n.ngrams {
+		if _, ok := o.ngrams[k]; ok {
+			intersection.Add(k)
+		}
+	}
+	return intersection
+}
+
+func (n *Ngrams) BatchAdd(o []string) {
+	for _, v := range o {
+		n.Add(v)
+	}
+}
+
+func (n *Ngrams) Union(others ...*Ngrams) *Ngrams {
+	union := NewNgrams(n.exclusive)
+	for k := range n.ngrams {
+		union.Add(k)
+	}
+	for _, other := range others {
+		for k := range other.ngrams {
+			union.Add(k)
+		}
+	}
+	return union
+}
+
+func getNgrams(n int, text []string, exclusive bool) *Ngrams {
+	ngramSet := NewNgrams(exclusive)
+	for i := 0; i <= len(text)-n; i++ {
+		ngramSet.Add(strings.Join(text[i:i+n], " "))
+	}
+	return ngramSet
+}
+
+func getWordNgrams(n int, sentences []string, exclusive bool) *Ngrams {
+	words := splitIntoWords(sentences)
+	return getNgrams(n, words, exclusive)
+}
+
+func lcs(x, y []string) [][]int {
+	n, m := len(x), len(y)
+	table := make([][]int, n+1)
+	for i := range table {
+		table[i] = make([]int, m+1)
+	}
+	for i := 1; i <= n; i++ {
+		for j := 1; j <= m; j++ {
+			if x[i-1] == y[j-1] {
+				table[i][j] = table[i-1][j-1] + 1
+			} else {
+				table[i][j] = max(table[i-1][j], table[i][j-1])
+			}
+		}
+	}
+	return table
+}
+
+func reconLcs(x, y []string, exclusive bool) *Ngrams {
+	i, j := len(x), len(y)
+	table := lcs(x, y)
+
+	var reconFunc func(int, int) []string
+	reconFunc = func(i, j int) []string {
+		if i == 0 || j == 0 {
+			return []string{}
+		} else if x[i-1] == y[j-1] {
+			return append(reconFunc(i-1, j-1), x[i-1])
+		} else if table[i-1][j] > table[i][j-1] {
+			return reconFunc(i-1, j)
+		} else {
+			return reconFunc(i, j-1)
+		}
+	}
+
+	reconList := reconFunc(i, j)
+	ngramList := NewNgrams(exclusive)
+	for _, word := range reconList {
+		ngramList.Add(word)
+	}
+	return ngramList
+}
+
+func rougeN(evaluatedSentences, referenceSentences []string, n int, rawResults, exclusive bool) map[string]float64 {
+	evaluatedNgrams := getWordNgrams(n, evaluatedSentences, exclusive)
+	referenceNgrams := getWordNgrams(n, referenceSentences, exclusive)
+	referenceCount := referenceNgrams.Len()
+	evaluatedCount := evaluatedNgrams.Len()
+
+	overlappingNgrams := evaluatedNgrams.Intersection(referenceNgrams)
+	overlappingCount := overlappingNgrams.Len()
+
+	results := make(map[string]float64)
+	if rawResults {
+		results["hyp"] = float64(evaluatedCount)
+		results["ref"] = float64(referenceCount)
+		results["overlap"] = float64(overlappingCount)
+		return results
+	} else {
+		return calculateRougeN(evaluatedCount, referenceCount, overlappingCount)
+	}
+}
+
+func calculateRougeN(evaluatedCount, referenceCount, overlappingCount int) map[string]float64 {
+	results := make(map[string]float64)
+	if evaluatedCount == 0 {
+		results["p"] = 0.0
+	} else {
+		results["p"] = float64(overlappingCount) / float64(evaluatedCount)
+	}
+
+	if referenceCount == 0 {
+		results["r"] = 0.0
+	} else {
+		results["r"] = float64(overlappingCount) / float64(referenceCount)
+	}
+
+	results["f"] = 2.0 * ((results["p"] * results["r"]) / (results["p"] + results["r"] + 1e-8))
+	return results
+}
+
+func unionLcs(evaluatedSentences []string, referenceSentence string, prevUnion *Ngrams, exclusive bool) (int, *Ngrams) {
+	if prevUnion == nil {
+		prevUnion = NewNgrams(exclusive)
+	}
+
+	lcsUnion := prevUnion
+	prevCount := len(prevUnion.ngrams)
+	referenceWords := splitIntoWords([]string{referenceSentence})
+
+	combinedLcsLength := 0
+	for _, evalS := range evaluatedSentences {
+		evaluatedWords := splitIntoWords([]string{evalS})
+		lcs := reconLcs(referenceWords, evaluatedWords, exclusive)
+		combinedLcsLength += lcs.Len()
+		lcsUnion = lcsUnion.Union(lcs)
+	}
+
+	newLcsCount := lcsUnion.Len() - prevCount
+	return newLcsCount, lcsUnion
+}
+
+func rougeLSummaryLevel(
+	evaluatedSentences, referenceSentences []string,
+	rawResults, exclusive bool,
+) map[string]float64 {
+	referenceNgrams := NewNgrams(exclusive)
+	referenceNgrams.BatchAdd(splitIntoWords(referenceSentences))
+	m := referenceNgrams.Len()
+
+	evaluatedNgrams := NewNgrams(exclusive)
+	evaluatedNgrams.BatchAdd(splitIntoWords(evaluatedSentences))
+	n := evaluatedNgrams.Len()
+
+	unionLcsSumAcrossAllReferences := 0
+	union := NewNgrams(exclusive)
+	for _, refS := range referenceSentences {
+		lcsCount, newUnion := unionLcs(evaluatedSentences, refS, union, exclusive)
+		union = newUnion
+		unionLcsSumAcrossAllReferences += lcsCount
+	}
+
+	llcs := unionLcsSumAcrossAllReferences
+
+	var rLcs float64
+	if m == 0 {
+		rLcs = 0.0
+	} else {
+		rLcs = float64(llcs) / float64(m)
+	}
+	var pLcs float64
+	if n == 0 {
+		pLcs = 0.0
+	} else {
+		pLcs = float64(llcs) / float64(n)
+	}
+
+	fLcs := 2.0 * ((pLcs * rLcs) / (pLcs + rLcs + 1e-8))
+
+	results := make(map[string]float64)
+	if rawResults {
+		results["hyp"] = float64(n)
+		results["ref"] = float64(m)
+		results["overlap"] = float64(llcs)
+		return results
+	} else {
+		results["f"] = fLcs
+		results["p"] = pLcs
+		results["r"] = rLcs
+		return results
+	}
+}