WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/common/db_retry.go

@@ -0,0 +1,45 @@
+package common
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/logger"
+)
+
+const defaultMaxDeadlockRetries = 3
+
+// WithDeadlockRetry retries fn when MySQL returns Error 1213 (deadlock).
+// Retries up to 3 times with exponential backoff (50ms, 100ms, 200ms).
+// Safe for all DB drivers — non-deadlock errors are returned immediately.
+func WithDeadlockRetry(ctx context.Context, fn func() error) error {
+	var err error
+	for attempt := 0; attempt <= defaultMaxDeadlockRetries; attempt++ {
+		err = fn()
+		if err == nil || !IsDeadlockError(err) {
+			return err
+		}
+		if attempt < defaultMaxDeadlockRetries {
+			wait := time.Duration(50<<attempt) * time.Millisecond
+			logger.Warnf(ctx, "[DeadlockRetry] attempt %d/%d failed with deadlock, retrying in %v: %v",
+				attempt+1, defaultMaxDeadlockRetries, wait, err)
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-time.After(wait):
+			}
+		}
+	}
+	return err
+}
+
+// IsDeadlockError checks whether err is a MySQL deadlock (Error 1213).
+func IsDeadlockError(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := err.Error()
+	return strings.Contains(msg, "Error 1213") ||
+		strings.Contains(msg, "Deadlock found when trying to get lock")
+}

internal/common/redis_tls.go

@@ -0,0 +1,47 @@
+package common
+
+import (
+	"crypto/tls"
+	"os"
+	"strings"
+)
+
+// RedisTLSConfig builds a *tls.Config for Redis connections from environment
+// variables. It returns nil when TLS is disabled, so the result can be assigned
+// directly to redis.Options.TLSConfig or asynq.RedisClientOpt.TLSConfig (a nil
+// value keeps the existing plaintext behavior).
+//
+// Environment variables:
+//
+//	REDIS_USE_TLS                   Enable TLS when "true" (default: disabled).
+//	REDIS_TLS_SERVER_NAME          Optional server name for certificate
+//	                               verification and SNI (useful when the address
+//	                               is an IP rather than a hostname).
+//	REDIS_TLS_INSECURE_SKIP_VERIFY Skip server certificate verification when
+//	                               "true". INSECURE — intended for development or
+//	                               self-signed setups only; do not use in production.
+//
+// The server certificate is verified by default; verification is only relaxed
+// when REDIS_TLS_INSECURE_SKIP_VERIFY is explicitly set to "true".
+func RedisTLSConfig() *tls.Config {
+	if !envTrue("REDIS_USE_TLS") {
+		return nil
+	}
+
+	cfg := &tls.Config{MinVersion: tls.VersionTLS12}
+
+	if serverName := strings.TrimSpace(os.Getenv("REDIS_TLS_SERVER_NAME")); serverName != "" {
+		cfg.ServerName = serverName
+	}
+	if envTrue("REDIS_TLS_INSECURE_SKIP_VERIFY") {
+		cfg.InsecureSkipVerify = true
+	}
+
+	return cfg
+}
+
+// envTrue reports whether the named environment variable is set to "true"
+// (case-insensitive, surrounding whitespace ignored).
+func envTrue(name string) bool {
+	return strings.EqualFold(strings.TrimSpace(os.Getenv(name)), "true")
+}

internal/common/redis_tls_test.go

@@ -0,0 +1,63 @@
+package common
+
+import (
+	"crypto/tls"
+	"testing"
+)
+
+func resetRedisTLSEnv(t *testing.T) {
+	t.Helper()
+	t.Setenv("REDIS_USE_TLS", "")
+	t.Setenv("REDIS_TLS_SERVER_NAME", "")
+	t.Setenv("REDIS_TLS_INSECURE_SKIP_VERIFY", "")
+}
+
+func TestRedisTLSConfig_DisabledByDefault(t *testing.T) {
+	resetRedisTLSEnv(t)
+
+	if cfg := RedisTLSConfig(); cfg != nil {
+		t.Fatalf("expected nil tls.Config when REDIS_USE_TLS is unset, got %#v", cfg)
+	}
+
+	t.Setenv("REDIS_USE_TLS", "false")
+	if cfg := RedisTLSConfig(); cfg != nil {
+		t.Fatalf("expected nil tls.Config when REDIS_USE_TLS=false, got %#v", cfg)
+	}
+}
+
+func TestRedisTLSConfig_EnabledSecureByDefault(t *testing.T) {
+	resetRedisTLSEnv(t)
+	t.Setenv("REDIS_USE_TLS", "true")
+
+	cfg := RedisTLSConfig()
+	if cfg == nil {
+		t.Fatal("expected non-nil tls.Config when REDIS_USE_TLS=true")
+	}
+	if cfg.MinVersion != tls.VersionTLS12 {
+		t.Errorf("expected MinVersion TLS 1.2, got %x", cfg.MinVersion)
+	}
+	if cfg.InsecureSkipVerify {
+		t.Error("expected certificate verification to be enabled by default")
+	}
+	if cfg.ServerName != "" {
+		t.Errorf("expected empty ServerName by default, got %q", cfg.ServerName)
+	}
+}
+
+func TestRedisTLSConfig_Options(t *testing.T) {
+	resetRedisTLSEnv(t)
+	t.Setenv("REDIS_USE_TLS", "TRUE") // case-insensitive
+	t.Setenv("REDIS_TLS_SERVER_NAME", "redis.example.com")
+	t.Setenv("REDIS_TLS_INSECURE_SKIP_VERIFY", "true")
+
+	cfg := RedisTLSConfig()
+	if cfg == nil {
+		t.Fatal("expected non-nil tls.Config")
+	}
+	if !cfg.InsecureSkipVerify {
+		t.Error("expected InsecureSkipVerify=true when opted in")
+	}
+	if cfg.ServerName != "redis.example.com" {
+		t.Errorf("expected ServerName redis.example.com, got %q", cfg.ServerName)
+	}
+}

internal/common/redislock/token_lock.go

@@ -0,0 +1,324 @@
+// Package redislock provides token-owned Redis locks with atomic renewal and
+// release. Callers must treat loss of ownership as loss of exclusive access.
+package redislock
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+)
+
+const releaseTimeout = 5 * time.Second
+
+// ErrLockOwnershipLost means exclusive ownership can no longer be guaranteed.
+var ErrLockOwnershipLost = errors.New("redis lock ownership lost")
+
+type ownershipContextKey struct{}
+type ownershipCheckContextKey struct{}
+
+// EnsureOwnership is a final lease check before a durable mutation. A canceled
+// context or lost lease must never permit publishing a stale generated result.
+// Callers outside a renewable lock still honor normal context cancellation.
+func EnsureOwnership(ctx context.Context) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if check, ok := ctx.Value(ownershipCheckContextKey{}).(func() error); ok {
+		return check()
+	}
+	return nil
+}
+
+var (
+	releaseScript = redis.NewScript(`
+if redis.call('GET', KEYS[1]) == ARGV[1] then
+	return redis.call('DEL', KEYS[1])
+end
+return 0
+`)
+	renewScript = redis.NewScript(`
+if redis.call('GET', KEYS[1]) == ARGV[1] then
+	return redis.call('PEXPIRE', KEYS[1], ARGV[2])
+end
+return 0
+`)
+)
+
+// OwnershipContext returns a context that is canceled only when lock
+// ownership is lost, not when the request context is canceled. Outside a
+// renewable-lock callback it falls back to a cancellation-detached context.
+func OwnershipContext(ctx context.Context) context.Context {
+	if ctx == nil {
+		return context.Background()
+	}
+	if ownershipCtx, ok := ctx.Value(ownershipContextKey{}).(context.Context); ok {
+		return ownershipCtx
+	}
+	return context.WithoutCancel(ctx)
+}
+
+// NewToken returns a random owner token suitable for compare-and-* scripts.
+func NewToken() (string, error) {
+	var token [16]byte
+	if _, err := rand.Read(token[:]); err != nil {
+		return "", fmt.Errorf("generate redis lock token: %w", err)
+	}
+	return hex.EncodeToString(token[:]), nil
+}
+
+// TryAcquire attempts one SET NX acquisition.
+func TryAcquire(
+	ctx context.Context,
+	client redis.UniversalClient,
+	key string,
+	token string,
+	lease time.Duration,
+) (bool, error) {
+	if err := validateOperation(client, key, token, lease); err != nil {
+		return false, err
+	}
+	acquired, err := client.SetNX(ctx, key, token, lease).Result()
+	if err != nil {
+		return false, fmt.Errorf("acquire redis lock %q: %w", key, err)
+	}
+	return acquired, nil
+}
+
+// Acquire waits until the lock is acquired or ctx is cancelled.
+func Acquire(
+	ctx context.Context,
+	client redis.UniversalClient,
+	key string,
+	token string,
+	lease time.Duration,
+) error {
+	for {
+		acquired, err := TryAcquire(ctx, client, key, token, lease)
+		if err != nil {
+			return err
+		}
+		if acquired {
+			return nil
+		}
+
+		timer := time.NewTimer(retryDelay())
+		select {
+		case <-ctx.Done():
+			if !timer.Stop() {
+				<-timer.C
+			}
+			return ctx.Err()
+		case <-timer.C:
+		}
+	}
+}
+
+// Release deletes the lock only if token still owns it.
+func Release(
+	ctx context.Context,
+	client redis.UniversalClient,
+	key string,
+	token string,
+) (bool, error) {
+	if client == nil {
+		return false, errors.New("redis client is required")
+	}
+	if key == "" || token == "" {
+		return false, errors.New("redis lock key and token are required")
+	}
+	released, err := releaseScript.Run(ctx, client, []string{key}, token).Int64()
+	if err != nil {
+		return false, fmt.Errorf("release redis lock %q: %w", key, err)
+	}
+	return released != 0, nil
+}
+
+// Renew extends the lease only if token still owns the lock.
+func Renew(
+	ctx context.Context,
+	client redis.UniversalClient,
+	key string,
+	token string,
+	lease time.Duration,
+) (bool, error) {
+	if err := validateOperation(client, key, token, lease); err != nil {
+		return false, err
+	}
+	renewed, err := renewScript.Run(
+		ctx,
+		client,
+		[]string{key},
+		token,
+		lease.Milliseconds(),
+	).Int64()
+	if err != nil {
+		return false, fmt.Errorf("renew redis lock %q: %w", key, err)
+	}
+	return renewed != 0, nil
+}
+
+// WithRenewableLock acquires key, renews it while fn runs, and atomically
+// releases it on exit. Renewal failure cancels the context passed to fn.
+func WithRenewableLock(
+	ctx context.Context,
+	client redis.UniversalClient,
+	key string,
+	lease time.Duration,
+	renewInterval time.Duration,
+	fn func(context.Context) error,
+) (resultErr error) {
+	return WithRenewableLockWait(ctx, client, key, lease, renewInterval, 0, fn)
+}
+
+// WithRenewableLockWait bounds only acquisition time; a long callback keeps
+// the parent context and a renewable lease. Zero wait means no extra deadline.
+func WithRenewableLockWait(
+	ctx context.Context,
+	client redis.UniversalClient,
+	key string,
+	lease time.Duration,
+	renewInterval time.Duration,
+	wait time.Duration,
+	fn func(context.Context) error,
+) (resultErr error) {
+	if fn == nil {
+		return errors.New("redis lock callback is required")
+	}
+	if renewInterval <= 0 || renewInterval >= lease {
+		return errors.New("redis lock renewal interval must be positive and shorter than the lease")
+	}
+
+	token, err := NewToken()
+	if err != nil {
+		return err
+	}
+	acquireCtx := ctx
+	if wait > 0 {
+		var cancel context.CancelFunc
+		acquireCtx, cancel = context.WithTimeout(ctx, wait)
+		defer cancel()
+	}
+	if err := Acquire(acquireCtx, client, key, token, lease); err != nil {
+		return err
+	}
+
+	lockCtx, cancelLock := context.WithCancelCause(ctx)
+	ownershipCtx, cancelOwnership := context.WithCancelCause(context.Background())
+	lockCtx = context.WithValue(lockCtx, ownershipContextKey{}, ownershipCtx)
+	lockCtx = context.WithValue(lockCtx, ownershipCheckContextKey{}, func() error {
+		renewed, err := Renew(lockCtx, client, key, token, lease)
+		if err != nil || !renewed {
+			lost := fmt.Errorf("%w for %q", ErrLockOwnershipLost, key)
+			cancelOwnership(lost)
+			cancelLock(lost)
+			return lost
+		}
+		return nil
+	})
+	renewCtx, stopRenewal := context.WithCancel(context.WithoutCancel(ctx))
+	renewResult := make(chan error, 1)
+	go renewLoop(
+		renewCtx,
+		cancelLock,
+		cancelOwnership,
+		client,
+		key,
+		token,
+		lease,
+		renewInterval,
+		renewResult,
+	)
+
+	defer func() {
+		stopRenewal()
+		renewErr := <-renewResult
+		cancelLock(nil)
+		cancelOwnership(nil)
+
+		releaseCtx, cancelRelease := context.WithTimeout(context.Background(), releaseTimeout)
+		released, releaseErr := Release(releaseCtx, client, key, token)
+		cancelRelease()
+		if releaseErr == nil && !released && renewErr == nil {
+			releaseErr = fmt.Errorf("%w for %q", ErrLockOwnershipLost, key)
+		}
+
+		resultErr = errors.Join(resultErr, renewErr, ctx.Err(), releaseErr)
+	}()
+
+	resultErr = fn(lockCtx)
+	return resultErr
+}
+
+func renewLoop(
+	ctx context.Context,
+	cancelLock context.CancelCauseFunc,
+	cancelOwnership context.CancelCauseFunc,
+	client redis.UniversalClient,
+	key string,
+	token string,
+	lease time.Duration,
+	renewInterval time.Duration,
+	result chan<- error,
+) {
+	ticker := time.NewTicker(renewInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			result <- nil
+			return
+		case <-ticker.C:
+			renewed, err := Renew(ctx, client, key, token, lease)
+			if err != nil {
+				if errors.Is(err, context.Canceled) && ctx.Err() != nil {
+					result <- nil
+					return
+				}
+				ownershipErr := fmt.Errorf("%w: %w", ErrLockOwnershipLost, err)
+				cancelOwnership(ownershipErr)
+				cancelLock(ownershipErr)
+				result <- ownershipErr
+				return
+			}
+			if !renewed {
+				ownershipErr := fmt.Errorf("%w for %q", ErrLockOwnershipLost, key)
+				cancelOwnership(ownershipErr)
+				cancelLock(ownershipErr)
+				result <- ownershipErr
+				return
+			}
+		}
+	}
+}
+
+func validateOperation(
+	client redis.UniversalClient,
+	key string,
+	token string,
+	lease time.Duration,
+) error {
+	if client == nil {
+		return errors.New("redis client is required")
+	}
+	if key == "" || token == "" {
+		return errors.New("redis lock key and token are required")
+	}
+	if lease <= 0 {
+		return errors.New("redis lock lease must be positive")
+	}
+	return nil
+}
+
+func retryDelay() time.Duration {
+	var random [1]byte
+	if _, err := rand.Read(random[:]); err != nil {
+		return 50 * time.Millisecond
+	}
+	return 25*time.Millisecond + time.Duration(random[0])%50*time.Millisecond
+}

internal/common/redislock/token_lock_test.go

@@ -0,0 +1,269 @@
+package redislock_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/alicebob/miniredis/v2"
+	"github.com/redis/go-redis/v9"
+	"github.com/stretchr/testify/require"
+
+	"github.com/Tencent/WeKnora/internal/common/redislock"
+)
+
+func newTokenLockTestClient(t *testing.T) (*redis.Client, *miniredis.Miniredis) {
+	t.Helper()
+
+	mini := miniredis.RunT(t)
+	client := redis.NewClient(&redis.Options{Addr: mini.Addr()})
+	t.Cleanup(func() { require.NoError(t, client.Close()) })
+	return client, mini
+}
+
+func TestRenewableLockWaitOnlyLimitsAcquisition(t *testing.T) {
+	client, _ := newTokenLockTestClient(t)
+	err := redislock.WithRenewableLockWait(context.Background(), client, "lock:long", time.Second, 10*time.Millisecond, 15*time.Millisecond, func(ctx context.Context) error {
+		time.Sleep(50 * time.Millisecond)
+		require.NoError(t, ctx.Err(), "acquisition deadline must not cancel running synthesis")
+		return redislock.EnsureOwnership(ctx)
+	})
+	require.NoError(t, err)
+	require.NoError(t, client.Set(context.Background(), "lock:busy", "other-owner", time.Minute).Err())
+	called := false
+	err = redislock.WithRenewableLockWait(context.Background(), client, "lock:busy", time.Second, 10*time.Millisecond, 15*time.Millisecond, func(context.Context) error {
+		called = true
+		return nil
+	})
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+	require.False(t, called)
+	require.Equal(t, "other-owner", client.Get(context.Background(), "lock:busy").Val())
+}
+
+func TestEnsureOwnershipRejectsStaleWriterBeforeNextRenewal(t *testing.T) {
+	client, _ := newTokenLockTestClient(t)
+	err := redislock.WithRenewableLockWait(context.Background(), client, "lock:stale", time.Minute, 20*time.Second, time.Second, func(ctx context.Context) error {
+		require.NoError(t, client.Set(context.Background(), "lock:stale", "successor", time.Minute).Err())
+		err := redislock.EnsureOwnership(ctx)
+		require.ErrorIs(t, err, redislock.ErrLockOwnershipLost)
+		require.Error(t, ctx.Err())
+		return err
+	})
+	require.ErrorIs(t, err, redislock.ErrLockOwnershipLost)
+	require.Equal(t, "successor", client.Get(context.Background(), "lock:stale").Val(), "stale release must not delete the successor")
+}
+
+func TestNewTokenReturnsUniqueOpaqueOwners(t *testing.T) {
+	t.Parallel()
+
+	first, err := redislock.NewToken()
+	require.NoError(t, err)
+	second, err := redislock.NewToken()
+	require.NoError(t, err)
+
+	require.Len(t, first, 32)
+	require.Len(t, second, 32)
+	require.NotEqual(t, first, second)
+}
+
+func TestTokenLockAcquireAndReleaseRequiresOwnership(t *testing.T) {
+	client, _ := newTokenLockTestClient(t)
+	ctx := context.Background()
+	key := "lock:ownership"
+
+	acquired, err := redislock.TryAcquire(ctx, client, key, "owner-a", time.Minute)
+	require.NoError(t, err)
+	require.True(t, acquired)
+
+	acquired, err = redislock.TryAcquire(ctx, client, key, "owner-b", time.Minute)
+	require.NoError(t, err)
+	require.False(t, acquired)
+
+	released, err := redislock.Release(ctx, client, key, "owner-b")
+	require.NoError(t, err)
+	require.False(t, released)
+	require.Equal(t, "owner-a", client.Get(ctx, key).Val())
+
+	released, err = redislock.Release(ctx, client, key, "owner-a")
+	require.NoError(t, err)
+	require.True(t, released)
+	require.ErrorIs(t, client.Get(ctx, key).Err(), redis.Nil)
+}
+
+func TestTokenLockRenewRequiresOwnership(t *testing.T) {
+	client, mini := newTokenLockTestClient(t)
+	ctx := context.Background()
+	key := "lock:renew"
+
+	require.NoError(t, client.Set(ctx, key, "owner-a", time.Second).Err())
+	renewed, err := redislock.Renew(ctx, client, key, "owner-b", time.Minute)
+	require.NoError(t, err)
+	require.False(t, renewed)
+
+	renewed, err = redislock.Renew(ctx, client, key, "owner-a", time.Minute)
+	require.NoError(t, err)
+	require.True(t, renewed)
+	mini.FastForward(2 * time.Second)
+	require.Equal(t, "owner-a", client.Get(ctx, key).Val())
+}
+
+func TestTokenLockRejectsUnsafeLeaseConfiguration(t *testing.T) {
+	client, _ := newTokenLockTestClient(t)
+	ctx := context.Background()
+
+	_, err := redislock.TryAcquire(ctx, client, "lock:key", "owner", 0)
+	require.Error(t, err)
+	_, err = redislock.Renew(ctx, client, "lock:key", "owner", -time.Second)
+	require.Error(t, err)
+	err = redislock.WithRenewableLock(
+		ctx,
+		client,
+		"lock:key",
+		time.Second,
+		time.Second,
+		func(context.Context) error { return nil },
+	)
+	require.Error(t, err)
+}
+
+func TestWithRenewableLockReturnsCallbackErrorAndReleases(t *testing.T) {
+	client, _ := newTokenLockTestClient(t)
+	ctx := context.Background()
+	key := "lock:callback"
+	want := errors.New("callback failed")
+
+	err := redislock.WithRenewableLock(
+		ctx,
+		client,
+		key,
+		time.Minute,
+		20*time.Second,
+		func(context.Context) error { return want },
+	)
+
+	require.ErrorIs(t, err, want)
+	require.ErrorIs(t, client.Get(ctx, key).Err(), redis.Nil)
+}
+
+func TestWithRenewableLockHonorsContextWhileWaiting(t *testing.T) {
+	client, _ := newTokenLockTestClient(t)
+	ctx := context.Background()
+	key := "lock:busy"
+	require.NoError(t, client.Set(ctx, key, "owner-a", time.Minute).Err())
+
+	waitCtx, cancel := context.WithTimeout(ctx, 25*time.Millisecond)
+	defer cancel()
+	called := false
+	err := redislock.WithRenewableLock(
+		waitCtx,
+		client,
+		key,
+		time.Minute,
+		20*time.Second,
+		func(context.Context) error {
+			called = true
+			return nil
+		},
+	)
+
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+	require.False(t, called)
+}
+
+func TestWithRenewableLockCancelsCallbackAfterOwnershipLoss(t *testing.T) {
+	client, _ := newTokenLockTestClient(t)
+	key := "lock:lost"
+
+	err := redislock.WithRenewableLock(
+		context.Background(),
+		client,
+		key,
+		100*time.Millisecond,
+		10*time.Millisecond,
+		func(lockCtx context.Context) error {
+			require.NoError(t, client.Set(context.Background(), key, "replacement", time.Minute).Err())
+			select {
+			case <-lockCtx.Done():
+				return nil
+			case <-time.After(time.Second):
+				return errors.New("callback was not cancelled")
+			}
+		},
+	)
+
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "ownership lost")
+	require.Equal(t, "replacement", client.Get(context.Background(), key).Val())
+}
+
+func TestWithRenewableLockReportsOwnershipLossAtRelease(t *testing.T) {
+	client, _ := newTokenLockTestClient(t)
+	key := "lock:lost-before-renewal"
+
+	err := redislock.WithRenewableLock(
+		context.Background(),
+		client,
+		key,
+		time.Minute,
+		20*time.Second,
+		func(context.Context) error {
+			return client.Set(context.Background(), key, "replacement", time.Minute).Err()
+		},
+	)
+
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "ownership lost")
+	require.Equal(t, "replacement", client.Get(context.Background(), key).Val())
+}
+
+func TestOwnershipContextSurvivesCallerCancellation(t *testing.T) {
+	client, _ := newTokenLockTestClient(t)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	err := redislock.WithRenewableLock(
+		ctx,
+		client,
+		"lock:caller-cancel",
+		time.Minute,
+		20*time.Second,
+		func(lockCtx context.Context) error {
+			ownershipCtx := redislock.OwnershipContext(lockCtx)
+			cancel()
+			require.ErrorIs(t, lockCtx.Err(), context.Canceled)
+			select {
+			case <-ownershipCtx.Done():
+				t.Fatal("caller cancellation must not imply lock ownership loss")
+			default:
+			}
+			return nil
+		},
+	)
+
+	require.ErrorIs(t, err, context.Canceled)
+}
+
+func TestOwnershipContextCancelsWhenRenewalLosesOwnership(t *testing.T) {
+	client, _ := newTokenLockTestClient(t)
+	key := "lock:ownership-context"
+
+	err := redislock.WithRenewableLock(
+		context.Background(),
+		client,
+		key,
+		100*time.Millisecond,
+		10*time.Millisecond,
+		func(lockCtx context.Context) error {
+			ownershipCtx := redislock.OwnershipContext(lockCtx)
+			require.NoError(t, client.Set(context.Background(), key, "replacement", time.Minute).Err())
+			select {
+			case <-ownershipCtx.Done():
+				return nil
+			case <-time.After(time.Second):
+				return errors.New("ownership context was not cancelled")
+			}
+		},
+	)
+
+	require.ErrorIs(t, err, redislock.ErrLockOwnershipLost)
+}

internal/common/tools.go

@@ -0,0 +1,293 @@
+package common
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"maps"
+	"regexp"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/Tencent/WeKnora/internal/logger"
+	secutils "github.com/Tencent/WeKnora/internal/utils"
+)
+
+// ToInterfaceSlice converts a slice of strings to a slice of empty interfaces.
+func ToInterfaceSlice[T any](slice []T) []interface{} {
+	interfaceSlice := make([]interface{}, len(slice))
+	for i, v := range slice {
+		interfaceSlice[i] = v
+	}
+	return interfaceSlice
+}
+
+// []string -> string, " join, space separated
+func StringSliceJoin(slice []string) string {
+	result := make([]string, len(slice))
+	for i, v := range slice {
+		result[i] = `"` + v + `"`
+	}
+	return strings.Join(result, " ")
+}
+
+func GetAttrs[A, B any](extract func(A) B, attrs ...A) []B {
+	result := make([]B, len(attrs))
+	for i, attr := range attrs {
+		result[i] = extract(attr)
+	}
+	return result
+}
+
+// Deduplicate removes duplicates from a slice based on a key function
+// T: the type of elements in the slice
+// K: the type of key used for deduplication
+func Deduplicate[T any, K comparable](keyFunc func(T) K, items ...T) []T {
+	seen := make(map[K]T)
+	for _, item := range items {
+		key := keyFunc(item)
+		if _, exists := seen[key]; !exists {
+			seen[key] = item
+		}
+	}
+	return slices.Collect(maps.Values(seen))
+}
+
+// ScoreComparable is an interface for types that have a Score method returning float64
+type ScoreComparable interface {
+	GetScore() float64
+}
+
+// DeduplicateWithScore removes duplicates from a slice based on a key function,
+// keeping the item with the highest score for each key, then sorts by score descending
+// T: the type of elements in the slice (must implement ScoreComparable)
+// K: the type of key used for deduplication
+func DeduplicateWithScore[T ScoreComparable, K comparable](keyFunc func(T) K, items ...T) []T {
+	seen := make(map[K]T)
+	for _, item := range items {
+		key := keyFunc(item)
+		if existing, exists := seen[key]; !exists {
+			seen[key] = item
+		} else if item.GetScore() > existing.GetScore() {
+			seen[key] = item
+		}
+	}
+	result := slices.Collect(maps.Values(seen))
+	// Sort by score descending
+	slices.SortFunc(result, func(a, b T) int {
+		scoreA := a.GetScore()
+		scoreB := b.GetScore()
+		if scoreA > scoreB {
+			return -1
+		} else if scoreA < scoreB {
+			return 1
+		}
+		return 0
+	})
+	return result
+}
+
+// ParseLLMJsonResponse parses a JSON response from LLM, handling cases where JSON is wrapped in code blocks.
+// This is useful when LLMs return responses like:
+// ```json
+// {"key": "value"}
+// ```
+// or regular JSON responses directly.
+// jsonCodeFenceRE extracts a JSON payload wrapped in a Markdown code fence.
+// Compiled once: ParseLLMJsonResponse runs on the graph-extraction path.
+var jsonCodeFenceRE = regexp.MustCompile("```(?:json)?\\s*([\\s\\S]*?)```")
+
+func ParseLLMJsonResponse(content string, target interface{}) error {
+	// First, try to parse directly as JSON
+	err := json.Unmarshal([]byte(content), target)
+	if err == nil {
+		return nil
+	}
+
+	// If direct parsing fails, try to extract JSON from code blocks
+	matches := jsonCodeFenceRE.FindStringSubmatch(content)
+	if len(matches) >= 2 {
+		// Extract the JSON content within the code block
+		jsonContent := strings.TrimSpace(matches[1])
+		if fenceErr := json.Unmarshal([]byte(jsonContent), target); fenceErr == nil {
+			return nil
+		}
+	}
+
+	// Last resort: models often wrap the payload in prose ("Sure, here is
+	// the JSON: {...}"). Scan for a balanced object/array so trailing
+	// commentary — including bracket-like text such as "[1]" — cannot
+	// truncate the payload.
+	if extracted := ExtractBalancedJSON(content); extracted != "" {
+		if scanErr := json.Unmarshal([]byte(extracted), target); scanErr == nil {
+			return nil
+		}
+	}
+
+	// Report the direct-parse failure, which is the most descriptive one.
+	return err
+}
+
+// ExtractBalancedJSON returns the first balanced JSON object or array embedded
+// in s, or an empty string when there is none. Whichever bracket type opens
+// first wins, and quoted strings are skipped so braces inside string literals
+// do not unbalance the scan. The result is not validated as JSON; callers must
+// still unmarshal it.
+func ExtractBalancedJSON(s string) string {
+	objStart := strings.IndexByte(s, '{')
+	arrStart := strings.IndexByte(s, '[')
+	var open, closeCh byte
+	var start int
+	switch {
+	case objStart < 0 && arrStart < 0:
+		return ""
+	case objStart < 0:
+		open, closeCh, start = '[', ']', arrStart
+	case arrStart < 0:
+		open, closeCh, start = '{', '}', objStart
+	case objStart < arrStart:
+		open, closeCh, start = '{', '}', objStart
+	default:
+		open, closeCh, start = '[', ']', arrStart
+	}
+
+	depth := 0
+	inString := false
+	escaped := false
+	for i := start; i < len(s); i++ {
+		c := s[i]
+		if inString {
+			switch {
+			case escaped:
+				escaped = false
+			case c == '\\':
+				escaped = true
+			case c == '"':
+				inString = false
+			}
+			continue
+		}
+		switch c {
+		case '"':
+			inString = true
+		case open:
+			depth++
+		case closeCh:
+			depth--
+			if depth == 0 {
+				return strings.TrimSpace(s[start : i+1])
+			}
+		}
+	}
+	return ""
+}
+
+// CleanInvalidUTF8 移除字符串中的非法 UTF-8 字符和 \x00
+func CleanInvalidUTF8(s string) string {
+	var b strings.Builder
+	b.Grow(len(s))
+
+	for i := 0; i < len(s); {
+		r, size := utf8.DecodeRuneInString(s[i:])
+		if r == utf8.RuneError && size == 1 {
+			// 非法 UTF-8 字节，跳过
+			i++
+			continue
+		}
+		if r == 0 {
+			// NULL 字符 \x00，跳过
+			i += size
+			continue
+		}
+		b.WriteRune(r)
+		i += size
+	}
+
+	return b.String()
+}
+
+const (
+	pipelineLogValueMaxRune = 300
+	defaultPipelineStage    = "PIPELINE"
+	defaultPipelineAction   = "info"
+	pipelineLogPrefix       = "[PIPELINE]"
+	pipelineTruncateEll     = "..."
+)
+
+// PipelineLog builds a structured pipeline log string.
+func PipelineLog(stage, action string, fields map[string]interface{}) string {
+	if stage == "" {
+		stage = defaultPipelineStage
+	}
+	if action == "" {
+		action = defaultPipelineAction
+	}
+
+	builder := strings.Builder{}
+	builder.Grow(128)
+	builder.WriteString(pipelineLogPrefix)
+	builder.WriteString(" stage=")
+	builder.WriteString(stage)
+	builder.WriteString(" action=")
+	builder.WriteString(action)
+
+	if len(fields) > 0 {
+		keys := make([]string, 0, len(fields))
+		for k := range fields {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		for _, key := range keys {
+			builder.WriteString(" ")
+			builder.WriteString(key)
+			builder.WriteString("=")
+			builder.WriteString(secutils.SanitizeForLog(formatPipelineLogValue(fields[key])))
+		}
+	}
+	return builder.String()
+}
+
+// PipelineInfo logs pipeline info level entries.
+func PipelineInfo(ctx context.Context, stage, action string, fields map[string]interface{}) {
+	logger.GetLogger(ctx).Info(PipelineLog(stage, action, fields))
+}
+
+// PipelineWarn logs pipeline warning level entries.
+func PipelineWarn(ctx context.Context, stage, action string, fields map[string]interface{}) {
+	logger.GetLogger(ctx).Warn(PipelineLog(stage, action, fields))
+}
+
+// PipelineError logs pipeline error level entries.
+func PipelineError(ctx context.Context, stage, action string, fields map[string]interface{}) {
+	logger.GetLogger(ctx).Error(PipelineLog(stage, action, fields))
+}
+
+func formatPipelineLogValue(value interface{}) string {
+	switch v := value.(type) {
+	case string:
+		return strconv.Quote(truncatePipelineValue(v))
+	case fmt.Stringer:
+		return strconv.Quote(truncatePipelineValue(v.String()))
+	case json.RawMessage:
+		bytes, _ := v.MarshalJSON()
+		return string(bytes)
+	default:
+		return fmt.Sprintf("%v", v)
+	}
+}
+
+func truncatePipelineValue(content string) string {
+	content = strings.ReplaceAll(content, "\n", "\\n")
+	runes := []rune(content)
+	if len(runes) <= pipelineLogValueMaxRune {
+		return content
+	}
+	return string(runes[:pipelineLogValueMaxRune]) + pipelineTruncateEll
+}
+
+func TruncateForLog(content string) string {
+	return truncatePipelineValue(content)
+}

internal/common/tools_test.go

@@ -0,0 +1,67 @@
+package common
+
+import (
+	"testing"
+	"unicode/utf8"
+)
+
+func TestParseLLMJsonResponse(t *testing.T) {
+	type payload struct {
+		Key string `json:"key"`
+	}
+
+	tests := []struct {
+		name    string
+		content string
+		want    string
+		wantErr bool
+	}{
+		{"direct json", `{"key":"value"}`, "value", false},
+		{"fenced json", "```json\n{\"key\":\"fenced\"}\n```", "fenced", false},
+		{"fenced no lang", "```\n{\"key\":\"plain\"}\n```", "plain", false},
+		{"prose around fence", "Here you go:\n```json\n{\"key\":\"wrapped\"}\n```\nThanks", "wrapped", false},
+		{"prose around bare json", `Sure: {"key":"bare"} hope that helps`, "bare", false},
+		// Trailing bracket-like prose must not truncate the payload.
+		{"trailing citation", `{"key":"cited"} — see section [1].`, "cited", false},
+		// Braces inside a string literal must not close the object early.
+		{"braces in string", `Result: {"key":"a}b"} done`, "a}b", false},
+		{"not json", "just some text", "", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var got payload
+			err := ParseLLMJsonResponse(tt.content, &got)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("err = %v, wantErr %v", err, tt.wantErr)
+			}
+			if !tt.wantErr && got.Key != tt.want {
+				t.Errorf("Key = %q, want %q", got.Key, tt.want)
+			}
+		})
+	}
+}
+
+func TestCleanInvalidUTF8(t *testing.T) {
+	input := "valid " + string([]byte{0xef, 0xbc, 0x2e}) + "\x00tail"
+
+	got := CleanInvalidUTF8(input)
+	if !utf8.ValidString(got) {
+		t.Fatalf("CleanInvalidUTF8 returned invalid UTF-8: % x", []byte(got))
+	}
+	if want := "valid .tail"; got != want {
+		t.Fatalf("CleanInvalidUTF8() = %q, want %q", got, want)
+	}
+}
+
+func BenchmarkParseLLMJsonResponse_Fenced(b *testing.B) {
+	const content = "```json\n{\"name\":\"Acme\",\"slug\":\"entity/acme\",\"aliases\":[\"A\",\"B\"]}\n```"
+	type payload struct {
+		Name string `json:"name"`
+	}
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		var p payload
+		_ = ParseLLMJsonResponse(content, &p)
+	}
+}