WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/application/repository/retriever/milvus/filter.go

@@ -0,0 +1,304 @@
+package milvus
+
+import (
+	"encoding/json"
+	"fmt"
+	"maps"
+	"reflect"
+	"strings"
+	"time"
+)
+
+const (
+	// operatorAnd is the "and" operator.
+	operatorAnd = "and"
+
+	// operatorOr is the "or" operator.
+	operatorOr = "or"
+
+	// operatorEqual is the "equal" operator.
+	operatorEqual = "eq"
+
+	// operatorNotEqual is the "not equal" operator.
+	operatorNotEqual = "ne"
+
+	// operatorGreaterThan is the "greater than" operator.
+	operatorGreaterThan = "gt"
+
+	// operatorGreaterThanOrEqual is the "greater than or equal" operator.
+	operatorGreaterThanOrEqual = "gte"
+
+	// operatorLessThan is the "less than" operator.
+	operatorLessThan = "lt"
+
+	// operatorLessThanOrEqual is the "less than or equal" operator.
+	operatorLessThanOrEqual = "lte"
+
+	// operatorIn is the "in" operator.
+	operatorIn = "in"
+
+	// operatorNotIn is the "not in" operator.
+	operatorNotIn = "not in"
+
+	// operatorLike is the "contains" operator.
+	operatorLike = "like"
+
+	// operatorNotLike is the "not contains" operator.
+	operatorNotLike = "not like"
+
+	// operatorBetween is the "between" operator.
+	operatorBetween = "between"
+)
+
+var comparisonOperators = map[string]string{
+	operatorEqual:              "==",
+	operatorNotEqual:           "!=",
+	operatorGreaterThan:        ">",
+	operatorGreaterThanOrEqual: ">=",
+	operatorLessThan:           "<",
+	operatorLessThanOrEqual:    "<=",
+	operatorLike:               "like",
+	operatorNotLike:            "not like",
+}
+
+type convertResult struct {
+	exprStr string
+	params  map[string]any
+}
+
+type filter struct{}
+
+func (c *filter) Convert(cond *universalFilterCondition) (*convertResult, error) {
+	var counter int
+	return c.convertCondition(cond, &counter)
+}
+
+func (c *filter) convertComparisonCondition(
+	cond *universalFilterCondition,
+	counter *int,
+) (*convertResult, error) {
+	condField := cond.Field
+	if condField == "" || cond.Value == nil {
+		return nil, fmt.Errorf("milvus filter condition is nil")
+	}
+	operator, ok := comparisonOperators[cond.Operator]
+	if !ok {
+		return nil, fmt.Errorf("unsupported comparison operator: %s", cond.Operator)
+	}
+
+	paramName := c.convertParamName(cond.Field, counter)
+	return &convertResult{
+		exprStr: fmt.Sprintf("%s %s {%s}", condField, operator, paramName),
+		params:  map[string]any{paramName: cond.Value},
+	}, nil
+}
+
+func (c *filter) convertLogicalCondition(
+	cond *universalFilterCondition,
+	counter *int,
+) (*convertResult, error) {
+	if cond.Value == nil {
+		return nil, fmt.Errorf("milvus filter condition is nil")
+	}
+	conds, ok := cond.Value.([]*universalFilterCondition)
+	if !ok {
+		return nil, fmt.Errorf("invalid logical condition value type")
+	}
+
+	var condResult *convertResult
+	for _, childCond := range conds {
+		childRes, err := c.convertCondition(childCond, counter)
+		if err != nil {
+			return nil, err
+		}
+		if childRes == nil || childRes.exprStr == "" {
+			continue
+		}
+		if condResult == nil {
+			condResult = childRes
+			continue
+		}
+
+		condResult.exprStr = fmt.Sprintf(
+			"(%s) %s (%s)",
+			condResult.exprStr,
+			strings.ToLower(cond.Operator),
+			childRes.exprStr,
+		)
+		maps.Copy(condResult.params, childRes.params)
+	}
+
+	if condResult == nil {
+		return nil, fmt.Errorf("empty logical condition")
+	}
+	return condResult, nil
+}
+
+func (c *filter) convertCondition(
+	cond *universalFilterCondition,
+	counter *int,
+) (*convertResult, error) {
+	if cond == nil {
+		return nil, fmt.Errorf("milvus filter condition is nil")
+	}
+	switch cond.Operator {
+	case operatorEqual, operatorNotEqual, operatorGreaterThan,
+		operatorGreaterThanOrEqual, operatorLessThan,
+		operatorLessThanOrEqual, operatorLike, operatorNotLike:
+		return c.convertComparisonCondition(cond, counter)
+	case operatorAnd, operatorOr:
+		return c.convertLogicalCondition(cond, counter)
+	case operatorIn, operatorNotIn:
+		return c.convertInCondition(cond, counter)
+	case operatorBetween:
+		return c.convertBetweenCondition(cond, counter)
+	default:
+		return nil, fmt.Errorf("unsupported operator: %v", cond.Operator)
+	}
+}
+
+func (c *filter) convertInCondition(
+	cond *universalFilterCondition,
+	counter *int,
+) (*convertResult, error) {
+	condField := cond.Field
+	if condField == "" || cond.Value == nil {
+		return nil, fmt.Errorf("milvus filter condition is nil")
+	}
+
+	s := reflect.ValueOf(cond.Value)
+	if s.Kind() != reflect.Slice || s.Len() <= 0 {
+		return nil, fmt.Errorf("in operator value must be a slice with at least one value: %v", cond.Value)
+	}
+
+	paramName := c.convertParamName(cond.Field, counter)
+	return &convertResult{
+		exprStr: fmt.Sprintf("%s %s {%s}", condField, strings.ToLower(cond.Operator), paramName),
+		params:  map[string]any{paramName: cond.Value},
+	}, nil
+}
+
+func (c *filter) convertBetweenCondition(
+	cond *universalFilterCondition,
+	counter *int,
+) (*convertResult, error) {
+	condField := cond.Field
+	if condField == "" || cond.Value == nil {
+		return nil, fmt.Errorf("milvus filter condition is nil")
+	}
+
+	value := reflect.ValueOf(cond.Value)
+	if value.Kind() != reflect.Slice || value.Len() != 2 {
+		return nil, fmt.Errorf("between operator value must be a slice with two elements: %v", cond.Value)
+	}
+
+	paramBase := c.convertParamName(cond.Field, counter)
+	paramName1 := fmt.Sprintf("%s_%d", paramBase, 0)
+	paramName2 := fmt.Sprintf("%s_%d", paramBase, 1)
+	return &convertResult{
+		exprStr: fmt.Sprintf("%s >= {%s} and %s <= {%s}", condField, paramName1, condField, paramName2),
+		params: map[string]any{
+			paramName1: value.Index(0).Interface(),
+			paramName2: value.Index(1).Interface(),
+		},
+	}, nil
+}
+
+func formatValue(value any) string {
+	switch v := value.(type) {
+	case string:
+		return fmt.Sprintf("\"%s\"", escapeDoubleQuotes(v))
+	case int, int8, int16, int32, int64:
+		return fmt.Sprintf("%d", v)
+	case uint, uint8, uint16, uint32, uint64:
+		return fmt.Sprintf("%d", v)
+	case float32, float64:
+		return fmt.Sprintf("%v", v)
+	case bool:
+		if v {
+			return "true"
+		}
+		return "false"
+	case time.Time:
+		return fmt.Sprintf("%d", v.Unix())
+	default:
+		return fmt.Sprintf("\"%v\"", value)
+	}
+}
+
+// escapeDoubleQuotes escapes double quotes in a string for use in Milvus expressions.
+func escapeDoubleQuotes(s string) string {
+	return strings.ReplaceAll(s, "\"", "\\\"")
+}
+
+// convertParamName converts field name to a valid Milvus template parameter name.
+// Milvus template parameters don't support '.' character, so we replace it with '_'.
+func (c *filter) convertParamName(field string, counter *int) string {
+	*counter++
+	return fmt.Sprintf("%s_%d", strings.ReplaceAll(field, ".", "_"), *counter)
+}
+
+type universalFilterCondition struct {
+	Field    string `json:"field,omitempty" jsonschema:"description=The metadata field to filter on (required for comparison operators)"`
+	Operator string `json:"operator" jsonschema:"description=The operator to use,enum=eq,enum=ne,enum=gt,enum=gte,enum=lt,enum=lte,enum=in,enum=not in,enum=like,enum=not like,enum=between,enum=and,enum=or"`
+	Value    any    `json:"value,omitempty" jsonschema:"description=The value to compare against (for comparison operators) or array of sub-conditions (for logical operators and/or)"`
+}
+
+func (c *universalFilterCondition) UnmarshalJSON(data []byte) error {
+	type Alias struct {
+		Field    string `json:"field,omitempty"`
+		Operator string `json:"operator"`
+		Value    any    `json:"value,omitempty"`
+	}
+
+	var aux Alias
+	if err := json.Unmarshal(data, &aux); err != nil {
+		return err
+	}
+
+	c.Field = aux.Field
+	c.Operator = strings.ToLower(aux.Operator)
+
+	// Handle logical operators (and/or) - Value should be []*UniversalFilterCondition
+	if c.Operator == operatorAnd || c.Operator == operatorOr {
+		// Value can be an array of conditions
+		valueSlice, ok := aux.Value.([]any)
+		if !ok {
+			return fmt.Errorf("logical operator %s requires an array of conditions", c.Operator)
+		}
+
+		conditions := make([]*universalFilterCondition, 0, len(valueSlice))
+		for i, v := range valueSlice {
+			condBytes, err := json.Marshal(v)
+			if err != nil {
+				return fmt.Errorf("failed to marshal condition at index %d: %w", i, err)
+			}
+
+			var cond universalFilterCondition
+			if err := json.Unmarshal(condBytes, &cond); err != nil {
+				return fmt.Errorf("failed to unmarshal condition at index %d: %w", i, err)
+			}
+			conditions = append(conditions, &cond)
+		}
+		c.Value = conditions
+	} else {
+		c.Value = aux.Value
+	}
+
+	return nil
+}
+
+// MarshalJSON implements custom JSON marshaling for UniversalFilterCondition.
+func (c *universalFilterCondition) MarshalJSON() ([]byte, error) {
+	type Alias struct {
+		Field    string `json:"field,omitempty"`
+		Operator string `json:"operator"`
+		Value    any    `json:"value,omitempty"`
+	}
+
+	return json.Marshal(&Alias{
+		Field:    c.Field,
+		Operator: c.Operator,
+		Value:    c.Value,
+	})
+}

internal/application/repository/retriever/milvus/repository_test.go

@@ -0,0 +1,243 @@
+package milvus
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/milvus-io/milvus/client/v2/column"
+	"github.com/milvus-io/milvus/client/v2/entity"
+	client "github.com/milvus-io/milvus/client/v2/milvusclient"
+	"github.com/stretchr/testify/require"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestContentFieldUsesChineseAnalyzer(t *testing.T) {
+	field := newContentField().ProtoMessage()
+	require.Equal(t, fieldContent, field.GetName())
+	require.Equal(t, int32(entity.FieldTypeVarChar), int32(field.GetDataType()))
+	params := make(map[string]string)
+	for _, param := range field.GetTypeParams() {
+		params[param.GetKey()] = param.GetValue()
+	}
+	require.Equal(t, "65535", params["max_length"])
+	require.Equal(t, "true", params["enable_analyzer"])
+	require.Equal(t, "true", params["enable_match"])
+	require.JSONEq(t, `{"type":"chinese"}`, params["analyzer_params"])
+}
+
+func TestUpdateChunkEnabledStatusInCollectionSkipsEmptyChunkIDs(t *testing.T) {
+	repo := &milvusRepository{}
+
+	require.NoError(t, repo.updateChunkEnabledStatusInCollection(
+		context.Background(),
+		"weknora_embeddings_1024",
+		nil,
+		false,
+	))
+	require.NoError(t, repo.updateChunkEnabledStatusInCollection(
+		context.Background(),
+		"weknora_embeddings_1024",
+		[]string{},
+		true,
+	))
+}
+
+func TestUpdateChunkEnabledStatusInCollectionsPropagatesFailure(t *testing.T) {
+	wantErr := errors.New("upsert failed")
+	err := updateChunkEnabledStatusInCollections(
+		context.Background(),
+		[]string{"other_collection", "weknora_embeddings_1024"},
+		"weknora_embeddings",
+		nil,
+		[]string{"chunk-1"},
+		func(_ context.Context, collection string, _ []string, enabled bool) error {
+			if collection == "weknora_embeddings_1024" && !enabled {
+				return wantErr
+			}
+			return nil
+		},
+	)
+	require.ErrorIs(t, err, wantErr)
+}
+
+func TestMatchesCollection(t *testing.T) {
+	for _, name := range []string{"weknora_embeddings_1", "weknora_embeddings_1024"} {
+		require.True(t, matchesCollection(name, "weknora_embeddings"), name)
+	}
+	for _, name := range []string{
+		"weknora_embeddings", "weknora_embeddings_", "weknora_embeddings_0",
+		"weknora_embeddings_-1", "weknora_embeddings_+1", "weknora_embeddings_01024",
+		"weknora_embeddings_1024_backup", "weknora_embeddings_backup_1024",
+		"weknora_embeddings_other_tenant_1024", "weknora_embeddings2_1024",
+		"weknora_embeddings_9999999999999999999999999", "other_1024", "_1024",
+	} {
+		require.False(t, matchesCollection(name, "weknora_embeddings"), name)
+	}
+	require.False(t, matchesCollection("_1024", ""))
+}
+
+func TestUpdateChunkEnabledStatusDoesNotCrossNamespaces(t *testing.T) {
+	var touched []string
+	err := updateChunkEnabledStatusInCollections(
+		context.Background(),
+		[]string{"weknora_embeddings_1024_backup", "weknora_embeddings_other_1024", "weknora_embeddings_1024"},
+		"weknora_embeddings", nil, []string{"chunk-1"},
+		func(_ context.Context, collection string, _ []string, _ bool) error {
+			touched = append(touched, collection)
+			return nil
+		},
+	)
+	require.NoError(t, err)
+	require.Equal(t, []string{"weknora_embeddings_1024", "weknora_embeddings_1024"}, touched)
+}
+
+func keywordResult(chunkID string, score float32) []client.ResultSet {
+	return []client.ResultSet{{
+		ResultCount: 1,
+		Scores:      []float32{score},
+		Fields: client.DataSet{
+			column.NewColumnVarChar(fieldID, []string{"index-" + chunkID}),
+			column.NewColumnVarChar(fieldChunkID, []string{chunkID}),
+			column.NewColumnVarChar(fieldContent, []string{"test content"}),
+		},
+	}}
+}
+
+func TestKeywordsRetrieveReturnsAllCollectionFailures(t *testing.T) {
+	repo := &milvusRepository{collectionBaseName: "weknora_embeddings"}
+	missingSparseField := errors.New("field content_sparse does not exist")
+	conversionFailure := errors.New("cannot decode search result")
+	var searched []string
+	results, err := repo.keywordsRetrieveInCollections(context.Background(), types.RetrieveParams{
+		Query: "test query", TopK: 10,
+	}, []string{"weknora_embeddings_384", "weknora_embeddings_1024", "weknora_embeddings_backup_1024"},
+		func(_ context.Context, option client.SearchOption) ([]client.ResultSet, error) {
+			request, err := option.Request()
+			require.NoError(t, err)
+			searched = append(searched, request.CollectionName)
+			if request.CollectionName == "weknora_embeddings_384" {
+				return nil, missingSparseField
+			}
+			return []client.ResultSet{{Err: conversionFailure}}, nil
+		})
+	require.Nil(t, results)
+	require.ErrorIs(t, err, missingSparseField)
+	require.ErrorIs(t, err, conversionFailure)
+	require.ErrorContains(t, err, "all 2 matching Milvus collections")
+	require.ErrorContains(t, err, "weknora_embeddings_384")
+	require.ErrorContains(t, err, "weknora_embeddings_1024")
+	require.Equal(t, []string{"weknora_embeddings_384", "weknora_embeddings_1024"}, searched)
+}
+
+func TestKeywordsRetrievePreservesBM25AndRanksAcrossCollections(t *testing.T) {
+	repo := &milvusRepository{collectionBaseName: "weknora_embeddings"}
+	results, err := repo.keywordsRetrieveInCollections(context.Background(), types.RetrieveParams{
+		Query: "test query", TopK: 1, KnowledgeBaseIDs: []string{"kb-allowed"},
+		TagIDs: []string{"tag-allowed"}, ExcludeChunkIDs: []string{"chunk-excluded"},
+	}, []string{"weknora_embeddings_384", "weknora_embeddings_1024", "weknora_embeddings_other_1536"},
+		func(_ context.Context, option client.SearchOption) ([]client.ResultSet, error) {
+			request, err := option.Request()
+			require.NoError(t, err)
+			require.Contains(t, request.Dsl, fieldKnowledgeBaseID)
+			require.Contains(t, request.Dsl, fieldTagID)
+			require.Contains(t, request.Dsl, fieldChunkID)
+			require.Contains(t, request.Dsl, fieldIsEnabled)
+			require.NotEmpty(t, request.ExprTemplateValues)
+			for _, param := range request.SearchParams {
+				if param.Key == "anns_field" {
+					require.Equal(t, fieldContentSparse, param.Value)
+				}
+			}
+			switch request.CollectionName {
+			case "weknora_embeddings_384":
+				return keywordResult("low-score", 2.5), nil
+			case "weknora_embeddings_1024":
+				return keywordResult("high-score", 17.25), nil
+			default:
+				t.Fatalf("searched unrelated collection %s", request.CollectionName)
+				return nil, nil
+			}
+		})
+	require.NoError(t, err)
+	require.Len(t, results, 1)
+	require.Equal(t, types.KeywordsRetrieverType, results[0].RetrieverType)
+	require.Len(t, results[0].Results, 1)
+	require.Equal(t, "high-score", results[0].Results[0].ChunkID)
+	require.Equal(t, 17.25, results[0].Results[0].Score)
+	require.Equal(t, types.MatchTypeKeywords, results[0].Results[0].MatchType)
+}
+
+func TestKeywordsRetrieveKeepsSuccessfulCollections(t *testing.T) {
+	for _, empty := range []bool{false, true} {
+		t.Run(map[bool]string{false: "results", true: "empty"}[empty], func(t *testing.T) {
+			repo := &milvusRepository{collectionBaseName: "weknora_embeddings"}
+			results, err := repo.keywordsRetrieveInCollections(context.Background(), types.RetrieveParams{
+				Query: "query", TopK: 5,
+			}, []string{"weknora_embeddings_384", "weknora_embeddings_1024"},
+				func(_ context.Context, option client.SearchOption) ([]client.ResultSet, error) {
+					request, err := option.Request()
+					require.NoError(t, err)
+					if request.CollectionName == "weknora_embeddings_384" {
+						return nil, errors.New("missing sparse field")
+					}
+					if empty {
+						return nil, nil
+					}
+					return keywordResult("available", 8), nil
+				})
+			require.NoError(t, err)
+			require.Len(t, results, 1)
+			if empty {
+				require.Empty(t, results[0].Results)
+			} else {
+				require.Len(t, results[0].Results, 1)
+				require.Equal(t, "available", results[0].Results[0].ChunkID)
+			}
+		})
+	}
+}
+
+func TestKeywordsRetrieveRejectsMissingScores(t *testing.T) {
+	repo := &milvusRepository{collectionBaseName: "weknora_embeddings"}
+	results, err := repo.keywordsRetrieveInCollections(context.Background(), types.RetrieveParams{
+		Query: "query", TopK: 5,
+	}, []string{"weknora_embeddings_384"},
+		func(context.Context, client.SearchOption) ([]client.ResultSet, error) {
+			result := keywordResult("missing-score", 0)
+			result[0].Scores = nil
+			return result, nil
+		})
+	require.Nil(t, results)
+	require.ErrorContains(t, err, "expected 1 BM25 scores, received 0")
+}
+
+func TestKeywordsRetrieveNoMatchingCollections(t *testing.T) {
+	repo := &milvusRepository{collectionBaseName: "weknora_embeddings"}
+	results, err := repo.keywordsRetrieveInCollections(context.Background(), types.RetrieveParams{
+		Query: "query", TopK: 5,
+	}, []string{"weknora_embeddings_backup_384"},
+		func(context.Context, client.SearchOption) ([]client.ResultSet, error) {
+			t.Fatal("must not search unrelated collection")
+			return nil, nil
+		})
+	require.NoError(t, err)
+	require.Len(t, results, 1)
+	require.Empty(t, results[0].Results)
+}
+
+func TestKeywordsRetrieveHonorsCancellation(t *testing.T) {
+	repo := &milvusRepository{collectionBaseName: "weknora_embeddings"}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	results, err := repo.keywordsRetrieveInCollections(ctx, types.RetrieveParams{
+		Query: "query", TopK: 5,
+	}, []string{"weknora_embeddings_384", "weknora_embeddings_1024"},
+		func(context.Context, client.SearchOption) ([]client.ResultSet, error) {
+			cancel()
+			return keywordResult("cancelled", 8), nil
+		})
+	require.Nil(t, results)
+	require.ErrorIs(t, err, context.Canceled)
+}

internal/application/repository/retriever/milvus/structs.go

@@ -0,0 +1,37 @@
+package milvus
+
+import (
+	"sync"
+
+	"github.com/milvus-io/milvus/client/v2/entity"
+	client "github.com/milvus-io/milvus/client/v2/milvusclient"
+)
+
+type milvusRepository struct {
+	filter
+	client             *client.Client
+	collectionBaseName string
+	metricType         entity.MetricType
+	shardsNum          int // 0 = use Milvus default (1)
+	replicaNumber      int // 0 = use Milvus default (1); set at LoadCollection time
+	// Cache for initialized collections (dimension -> true)
+	initializedCollections sync.Map
+}
+
+type MilvusVectorEmbedding struct {
+	ID              string    `json:"id"`
+	Content         string    `json:"content"`
+	SourceID        string    `json:"source_id"`
+	SourceType      int       `json:"source_type"`
+	ChunkID         string    `json:"chunk_id"`
+	KnowledgeID     string    `json:"knowledge_id"`
+	KnowledgeBaseID string    `json:"knowledge_base_id"`
+	TagID           string    `json:"tag_id"`
+	Embedding       []float32 `json:"embedding"`
+	IsEnabled       bool      `json:"is_enabled"`
+}
+
+type MilvusVectorEmbeddingWithScore struct {
+	MilvusVectorEmbedding
+	Score float64
+}

internal/application/repository/retriever/milvus/vector_copy_test.go

@@ -0,0 +1,29 @@
+package milvus
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/milvus-io/milvus/client/v2/column"
+	client "github.com/milvus-io/milvus/client/v2/milvusclient"
+)
+
+func TestCopyResultPreservesNativeFloatVector(t *testing.T) {
+	want := []float32{0.1, 0.25, -0.5}
+	for _, field := range []column.Column{
+		column.NewColumnFloatVector(fieldEmbedding, 3, [][]float32{want}),
+		column.NewColumnDoubleArray(fieldEmbedding, [][]float64{{0.1, 0.25, -0.5}}),
+	} {
+		result, _, err := convertResultSet([]client.ResultSet{{ResultCount: 1, Fields: client.DataSet{column.NewColumnVarChar(fieldID, []string{"source"}), field}}})
+		if err != nil || len(result) != 1 || !reflect.DeepEqual(result[0].Embedding, want) {
+			t.Fatalf("copy dropped vector from %T: %#v, %v", field, result, err)
+		}
+	}
+}
+
+func TestCopyResultRejectsUnknownVectorColumn(t *testing.T) {
+	_, _, err := convertResultSet([]client.ResultSet{{ResultCount: 1, Fields: client.DataSet{column.NewColumnVarChar(fieldEmbedding, []string{"invalid"})}}})
+	if err == nil {
+		t.Fatal("invalid vector was silently converted into a zero-dimensional embedding")
+	}
+}