WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/application/repository/retriever/doris/compat.go

@@ -0,0 +1,228 @@
+package doris
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/Tencent/WeKnora/internal/logger"
+)
+
+const envDorisCompatMode = "DORIS_COMPAT_MODE"
+
+type dorisCompatMode string
+
+const (
+	dorisCompatModeAuto                  dorisCompatMode = "auto"
+	dorisCompatModeLegacy                dorisCompatMode = "legacy"
+	dorisCompatModeInnerProductDuplicate dorisCompatMode = "inner_product_duplicate"
+)
+
+type dorisCompatProbe struct {
+	innerProductApproximate   bool
+	cosineDistanceApproximate bool
+}
+
+func resolveConfiguredDorisCompatMode() (mode dorisCompatMode, invalid string) {
+	raw := strings.TrimSpace(os.Getenv(envDorisCompatMode))
+	if raw == "" {
+		return dorisCompatModeAuto, ""
+	}
+
+	switch strings.ToLower(raw) {
+	case string(dorisCompatModeAuto):
+		return dorisCompatModeAuto, ""
+	case string(dorisCompatModeLegacy):
+		return dorisCompatModeLegacy, ""
+	case string(dorisCompatModeInnerProductDuplicate), "inner-product-duplicate", "inner_product", "inner-product":
+		return dorisCompatModeInnerProductDuplicate, ""
+	default:
+		return dorisCompatModeAuto, raw
+	}
+}
+
+func (r *dorisRepository) resolveCompatMode(ctx context.Context) (dorisCompatMode, error) {
+	r.compatResolveOnce.Do(func() {
+		log := logger.GetLogger(ctx)
+		requested := r.compatModeRequested
+		if requested == "" {
+			r.compatModeResolved = dorisCompatModeInnerProductDuplicate
+			return
+		}
+
+		existingMode, exampleTable, foundExisting, err := r.detectExistingCompatMode(ctx)
+		if err != nil {
+			r.compatResolveErr = err
+			return
+		}
+
+		if foundExisting {
+			if requested != dorisCompatModeAuto && requested != existingMode {
+				r.compatResolveErr = fmt.Errorf(
+					"Doris compat mode %q does not match existing embedding tables (detected %q from %s). %s is not interchangeable after %s_* tables are created. Recreate the existing %s_* tables before switching modes, or set %s=%s",
+					requested,
+					existingMode,
+					exampleTable,
+					envDorisCompatMode,
+					r.tableBaseName,
+					r.tableBaseName,
+					envDorisCompatMode,
+					existingMode,
+				)
+				log.Errorf("[Doris] %v", r.compatResolveErr)
+				return
+			}
+
+			r.compatModeResolved = existingMode
+			log.Warnf(
+				"[Doris] Using compat mode %s from existing embedding tables (%s). %s is not interchangeable after %s_* tables are created; recreate those tables before switching modes.",
+				existingMode,
+				exampleTable,
+				envDorisCompatMode,
+				r.tableBaseName,
+			)
+			return
+		}
+
+		if requested == dorisCompatModeAuto {
+			probe, probeErr := r.probeCompatMode(ctx)
+			if probeErr != nil {
+				r.compatResolveErr = probeErr
+				log.Errorf("[Doris] %v", probeErr)
+				return
+			}
+			log.Infof(
+				"[Doris] Compat probe result: inner_product_approximate=%t, cosine_distance_approximate=%t",
+				probe.innerProductApproximate,
+				probe.cosineDistanceApproximate,
+			)
+
+			switch {
+			case probe.innerProductApproximate:
+				r.compatModeResolved = dorisCompatModeInnerProductDuplicate
+			case probe.cosineDistanceApproximate:
+				r.compatModeResolved = dorisCompatModeLegacy
+			default:
+				r.compatResolveErr = fmt.Errorf(
+					"Doris compatibility auto-detection could not find a supported vector function. Set %s=%s or %s=%s explicitly after verifying your Doris build. %s is not interchangeable after %s_* tables are created",
+					envDorisCompatMode,
+					dorisCompatModeInnerProductDuplicate,
+					envDorisCompatMode,
+					dorisCompatModeLegacy,
+					envDorisCompatMode,
+					r.tableBaseName,
+				)
+				log.Errorf("[Doris] %v", r.compatResolveErr)
+				return
+			}
+
+			log.Warnf(
+				"[Doris] Auto-selected compat mode %s for new embedding tables. %s is not interchangeable after %s_* tables are created; recreate those tables before switching modes.",
+				r.compatModeResolved,
+				envDorisCompatMode,
+				r.tableBaseName,
+			)
+			return
+		}
+
+		r.compatModeResolved = requested
+		log.Warnf(
+			"[Doris] Using configured compat mode %s for new embedding tables. %s is not interchangeable after %s_* tables are created; recreate those tables before switching modes.",
+			r.compatModeResolved,
+			envDorisCompatMode,
+			r.tableBaseName,
+		)
+	})
+
+	return r.compatModeResolved, r.compatResolveErr
+}
+
+func (r *dorisRepository) detectExistingCompatMode(ctx context.Context) (dorisCompatMode, string, bool, error) {
+	tables, err := r.listEmbeddingTables(ctx)
+	if err != nil {
+		return "", "", false, fmt.Errorf("list Doris embedding tables: %w", err)
+	}
+	if len(tables) == 0 {
+		return "", "", false, nil
+	}
+	sort.Strings(tables)
+
+	var detected dorisCompatMode
+	for _, table := range tables {
+		ddl, err := r.showCreateTable(ctx, table)
+		if err != nil {
+			return "", "", false, fmt.Errorf("show create table %s: %w", table, err)
+		}
+		mode, err := compatModeFromDDL(ddl)
+		if err != nil {
+			return "", "", false, fmt.Errorf("detect compat mode from %s: %w", table, err)
+		}
+		if detected == "" {
+			detected = mode
+			continue
+		}
+		if detected != mode {
+			return "", "", false, fmt.Errorf(
+				"existing Doris embedding tables use mixed compat modes (%s and %s). %s is not interchangeable after %s_* tables are created; recreate the existing %s_* tables with a single mode",
+				detected,
+				mode,
+				envDorisCompatMode,
+				r.tableBaseName,
+				r.tableBaseName,
+			)
+		}
+	}
+
+	return detected, tables[0], true, nil
+}
+
+func (r *dorisRepository) showCreateTable(ctx context.Context, table string) (string, error) {
+	row := r.db.QueryRowContext(ctx, fmt.Sprintf("SHOW CREATE TABLE `%s`", table))
+	var tableName, ddl string
+	if err := row.Scan(&tableName, &ddl); err != nil {
+		return "", err
+	}
+	return ddl, nil
+}
+
+func compatModeFromDDL(ddl string) (dorisCompatMode, error) {
+	normalized := strings.ToLower(ddl)
+	switch {
+	case strings.Contains(normalized, "duplicate key("):
+		return dorisCompatModeInnerProductDuplicate, nil
+	case strings.Contains(normalized, "unique key("):
+		return dorisCompatModeLegacy, nil
+	default:
+		return "", fmt.Errorf("unsupported table definition; expected UNIQUE KEY or DUPLICATE KEY")
+	}
+}
+
+func (r *dorisRepository) probeCompatMode(ctx context.Context) (dorisCompatProbe, error) {
+	probe := dorisCompatProbe{}
+	probe.innerProductApproximate = r.vectorFunctionSupported(ctx, "inner_product_approximate([1.0],[1.0])")
+	probe.cosineDistanceApproximate = r.vectorFunctionSupported(ctx, "cosine_distance_approximate([1.0],[1.0])")
+	return probe, nil
+}
+
+func (r *dorisRepository) vectorFunctionSupported(ctx context.Context, expr string) bool {
+	var value sql.NullFloat64
+	if err := r.db.QueryRowContext(ctx, "SELECT "+expr).Scan(&value); err != nil {
+		return false
+	}
+	return true
+}
+
+func (m dorisCompatMode) normalizeEmbeddings() bool {
+	return m != dorisCompatModeLegacy
+}
+
+func (m dorisCompatMode) usesReplaceWrite() bool {
+	return m != dorisCompatModeLegacy
+}
+
+func (m dorisCompatMode) usesRewriteChunkUpdates() bool {
+	return m != dorisCompatModeLegacy
+}