WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/models/limiter/governor.go

@@ -0,0 +1,174 @@
+package limiter
+
+import (
+	"context"
+	"sort"
+	"sync"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// The concurrency governor is process-wide, shared by every model-client layer
+// that fronts a provider (chat, vlm). Keeping the singleton here — rather than
+// inside one client package — lets all of them gate against the same limiter
+// and per-model limit without importing each other. Wired once at startup (see
+// container.registerModelConcurrencyLimiter) via SetGovernor.
+var (
+	governorMu sync.RWMutex
+	governor   ModelConcurrencyLimiter
+	governorN  int
+)
+
+// SetGovernor installs the process-wide background concurrency governor and the
+// default per-model limit. Passing a nil limiter or a non-positive limit
+// disables governance (all calls pass through). Safe to call at startup.
+func SetGovernor(l ModelConcurrencyLimiter, limit int) {
+	governorMu.Lock()
+	defer governorMu.Unlock()
+	governor = l
+	governorN = limit
+}
+
+// SetGlobalLimit updates ONLY the process-wide default per-model limit,
+// leaving the installed limiter backend intact. Used by the system-settings
+// runtime bridge so an operator can retune model.max_concurrency without a
+// restart. A non-positive value disables the default (models that carry their
+// own MaxConcurrency still honour it).
+func SetGlobalLimit(limit int) {
+	governorMu.Lock()
+	defer governorMu.Unlock()
+	governorN = limit
+}
+
+// Gate acquires a per-model concurrency slot using the process-wide default
+// limit. Equivalent to GateN(ctx, modelID, 0).
+func Gate(ctx context.Context, modelID string) func() {
+	return GateN(ctx, modelID, 0)
+}
+
+// GateN acquires a per-model concurrency slot when the call is a background task
+// (see types.IsBackgroundTask) and a governor is installed. modelLimit is the
+// model's own configured cap; a value <= 0 means "fall back to the process-wide
+// default" (governorN). It returns a release func that is ALWAYS safe to call:
+// on the passthrough / fail-open paths it is a cheap no-op. The gate never
+// blocks a call permanently — a limiter/Redis outage or a cancelled context
+// fails open.
+func GateN(ctx context.Context, modelID string, modelLimit int) func() {
+	return GateNamedN(ctx, modelID, "", modelLimit)
+}
+
+func GateNamedN(ctx context.Context, modelID, modelName string, modelLimit int) func() {
+	governorMu.RLock()
+	l, defaultLimit := governor, governorN
+	governorMu.RUnlock()
+
+	limit := modelLimit
+	if limit <= 0 {
+		limit = defaultLimit
+	}
+	if l == nil || limit <= 0 || !types.IsBackgroundTask(ctx) {
+		return noop
+	}
+	if named, ok := l.(interface{ SetModelName(string, string) }); ok {
+		named.SetModelName(modelID, modelName)
+	}
+	release, err := l.Acquire(ctx, modelID, limit)
+	if err != nil || release == nil {
+		return noop
+	}
+	return release
+}
+
+// RuntimeStats returns the semaphores observed by this process. Redis-backed
+// limiters report cluster-wide active holders; waiting callers remain local to
+// this instance by design.
+func RuntimeStats(ctx context.Context) ([]RuntimeStat, bool, error) {
+	governorMu.RLock()
+	l := governor
+	governorMu.RUnlock()
+	inspector, ok := l.(runtimeInspectable)
+	if !ok || inspector == nil {
+		return []RuntimeStat{}, false, nil
+	}
+	stats, err := inspector.RuntimeStats(ctx)
+	if stats == nil {
+		stats = []RuntimeStat{}
+	}
+	return stats, true, err
+}
+
+// localLimiter is an in-process (single-node) counting semaphore keyed by
+// model ID. It is the Lite-mode counterpart to the Redis limiter: Lite runs a
+// single process with no Redis, so a shared distributed semaphore is neither
+// available nor needed — but background ingestion can still burst the whole
+// worker pool against one provider, so we still cap concurrency locally.
+type localLimiter struct {
+	mu      sync.Mutex
+	sems    map[string]chan struct{}
+	tracked map[string]*trackedSemaphore
+}
+
+// NewLocalLimiter builds an in-process per-key concurrency limiter.
+func NewLocalLimiter() ModelConcurrencyLimiter {
+	return &localLimiter{sems: make(map[string]chan struct{}), tracked: make(map[string]*trackedSemaphore)}
+}
+
+func (l *localLimiter) Acquire(ctx context.Context, key string, limit int) (func(), error) {
+	if l == nil || limit <= 0 || key == "" {
+		return noop, nil
+	}
+
+	l.mu.Lock()
+	sem, ok := l.sems[key]
+	tracked := l.tracked[key]
+	if tracked == nil {
+		tracked = &trackedSemaphore{}
+		l.tracked[key] = tracked
+	}
+	tracked.limit.Store(int64(limit))
+	if !ok {
+		// Capacity is fixed at first use for a key; the limit is a
+		// process-wide constant, so it never changes across acquires.
+		sem = make(chan struct{}, limit)
+		l.sems[key] = sem
+	}
+	l.mu.Unlock()
+	tracked.waiting.Add(1)
+	defer tracked.waiting.Add(-1)
+
+	select {
+	case sem <- struct{}{}:
+		var once sync.Once
+		return func() { once.Do(func() { <-sem }) }, nil
+	case <-ctx.Done():
+		// Fail open on cancellation, mirroring the Redis limiter.
+		return noop, nil
+	}
+}
+
+func (l *localLimiter) RuntimeStats(_ context.Context) ([]RuntimeStat, error) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	stats := make([]RuntimeStat, 0, len(l.sems))
+	for modelID, sem := range l.sems {
+		tracked := l.tracked[modelID]
+		name, _ := tracked.name.Load().(string)
+		stats = append(stats, RuntimeStat{ModelID: modelID, Name: name, Active: int64(len(sem)), Waiting: tracked.waiting.Load(), Limit: int(tracked.limit.Load())})
+	}
+	sort.Slice(stats, func(i, j int) bool { return stats[i].ModelID < stats[j].ModelID })
+	return stats, nil
+}
+
+func (l *localLimiter) SetModelName(modelID, name string) {
+	if modelID == "" || name == "" {
+		return
+	}
+	l.mu.Lock()
+	tracked := l.tracked[modelID]
+	if tracked == nil {
+		tracked = &trackedSemaphore{}
+		l.tracked[modelID] = tracked
+	}
+	l.mu.Unlock()
+	tracked.name.Store(name)
+}

internal/models/limiter/governor_test.go

@@ -0,0 +1,133 @@
+package limiter
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func withBackground() context.Context {
+	return types.WithBackgroundTask(context.Background())
+}
+
+// TestGateOnlyGovernsBackground verifies interactive calls always pass through
+// (no-op release) and only background calls consult the installed limiter.
+func TestGateOnlyGovernsBackground(t *testing.T) {
+	t.Cleanup(func() { SetGovernor(nil, 0) })
+	SetGovernor(NewLocalLimiter(), 1)
+
+	// Interactive (non-background) ctx: never gated, even at limit 1.
+	rel1 := Gate(context.Background(), "m")
+	rel2 := Gate(context.Background(), "m")
+	rel1()
+	rel2()
+
+	// Background ctx: gated. First slot taken; second would block, so a
+	// cancelled second acquire must fail open (return a usable release).
+	held := Gate(withBackground(), "m")
+	defer held()
+
+	ctx, cancel := context.WithTimeout(withBackground(), 30*time.Millisecond)
+	defer cancel()
+	rel := Gate(ctx, "m")
+	if rel == nil {
+		t.Fatal("cancelled background gate must fail open with a usable release")
+	}
+	rel()
+}
+
+// TestGateDisabledWhenNoGovernor verifies Gate is a passthrough when no
+// governor is installed.
+func TestGateDisabledWhenNoGovernor(t *testing.T) {
+	SetGovernor(nil, 0)
+	if rel := Gate(withBackground(), "m"); rel == nil {
+		t.Fatal("gate with no governor must return a usable release")
+	} else {
+		rel()
+	}
+}
+
+// TestLocalLimiterCaps verifies the in-process limiter enforces the cap and
+// releases slots.
+func TestLocalLimiterCaps(t *testing.T) {
+	l := NewLocalLimiter()
+	ctx := context.Background()
+
+	r1, _ := l.Acquire(ctx, "k", 2)
+	r2, _ := l.Acquire(ctx, "k", 2)
+
+	// Third acquire must block while full.
+	done := make(chan func(), 1)
+	go func() {
+		r3, _ := l.Acquire(ctx, "k", 2)
+		done <- r3
+	}()
+	select {
+	case <-done:
+		t.Fatal("third acquire should block while at limit")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	r1() // free a slot; the waiter proceeds
+	select {
+	case r3 := <-done:
+		r3()
+	case <-time.After(time.Second):
+		t.Fatal("waiter should proceed after a slot frees")
+	}
+	r2()
+}
+
+// TestLocalLimiterIndependentKeys verifies per-key budgets are independent.
+func TestLocalLimiterIndependentKeys(t *testing.T) {
+	l := NewLocalLimiter()
+	ctx := context.Background()
+	ra, _ := l.Acquire(ctx, "a", 1)
+	rb, _ := l.Acquire(ctx, "b", 1) // different key: must not block
+	ra()
+	rb()
+}
+
+// TestLocalLimiterFailOpen verifies degraded inputs pass through.
+func TestLocalLimiterFailOpen(t *testing.T) {
+	l := NewLocalLimiter()
+	ctx := context.Background()
+	if r, err := l.Acquire(ctx, "", 4); err != nil || r == nil {
+		t.Fatal("empty key should fail open")
+	} else {
+		r()
+	}
+	if r, err := l.Acquire(ctx, "k", 0); err != nil || r == nil {
+		t.Fatal("limit<=0 should fail open")
+	} else {
+		r()
+	}
+}
+
+// TestLocalLimiterReleaseIdempotent verifies double release doesn't
+// over-free a slot.
+func TestLocalLimiterReleaseIdempotent(t *testing.T) {
+	l := NewLocalLimiter()
+	ctx := context.Background()
+	r1, _ := l.Acquire(ctx, "k", 1)
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() { defer wg.Done(); r1() }()
+	go func() { defer wg.Done(); r1() }()
+	wg.Wait()
+
+	// After idempotent release the single slot is free again.
+	r2, _ := l.Acquire(ctx, "k", 1)
+	done := make(chan struct{})
+	go func() { r3, _ := l.Acquire(ctx, "k", 1); r3(); close(done) }()
+	select {
+	case <-done:
+		t.Fatal("a double-release must not leave the slot permanently free")
+	case <-time.After(50 * time.Millisecond):
+	}
+	r2()
+	<-done
+}

internal/models/limiter/limiter.go

@@ -0,0 +1,239 @@
+// Package limiter provides a distributed, per-key concurrency governor for
+// outbound model-provider calls. The shared finite resource is the model
+// provider (its request/concurrency budget), so concurrency is capped at the
+// model-client layer — keyed by model ID — rather than at the asynq queue layer
+// (queue weights are scheduling priority, not throttling).
+//
+// The Redis implementation is a self-healing distributed semaphore built on a
+// sorted set: each held slot is a ZSET member (unique token) scored by its
+// lease expiry. Acquire atomically prunes expired leases, counts live holders,
+// and admits a new one only while under the limit. A background heartbeat
+// refreshes the lease so long calls keep their slot; a crashed holder's lease
+// simply expires and is reclaimed. Every backend error fails OPEN (the call is
+// allowed) so a limiter/Redis outage can never halt model traffic.
+package limiter
+
+import (
+	"context"
+	"sort"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/logger"
+	"github.com/google/uuid"
+	"github.com/redis/go-redis/v9"
+)
+
+// ModelConcurrencyLimiter caps the number of concurrent in-flight calls per
+// key (typically a model ID) across all processes sharing the same backend.
+type ModelConcurrencyLimiter interface {
+	// Acquire blocks until a slot for key is available or ctx is done. It
+	// returns a release func that MUST be invoked to free the slot. On any
+	// backend error (or ctx cancellation) it fails open: release is a no-op and
+	// err is nil, so callers proceed without a slot rather than dropping the
+	// call.
+	Acquire(ctx context.Context, key string, limit int) (release func(), err error)
+}
+
+// RuntimeStat is a point-in-time view of a model semaphore. Active is
+// cluster-wide for the Redis backend and process-local in Lite mode. Waiting is
+// deliberately process-local: waiters block in application processes and are
+// not represented in Redis.
+type RuntimeStat struct {
+	ModelID string `json:"model_id"`
+	Name    string `json:"name"`
+	Active  int64  `json:"active"`
+	Waiting int64  `json:"waiting"`
+	Limit   int    `json:"limit"`
+}
+
+type runtimeInspectable interface {
+	RuntimeStats(context.Context) ([]RuntimeStat, error)
+}
+
+type trackedSemaphore struct {
+	limit   atomic.Int64
+	waiting atomic.Int64
+	name    atomic.Value // string
+}
+
+// noop is the release returned on the fail-open / passthrough paths.
+func noop() {}
+
+const (
+	// defaultLeaseTTL is the crash-recovery window, not a request timeout.
+	// Live calls refresh their lease every ttl/3, so even a very long provider
+	// request keeps its slot. Keeping this short prevents an app/container
+	// restart from leaving an entire model budget apparently occupied for
+	// minutes while the replacement workers are blocked behind dead holders.
+	defaultLeaseTTL = 30 * time.Second
+	// defaultPollInterval is how often a waiting acquirer re-checks for a free
+	// slot. Small enough to stay responsive, large enough to avoid hammering
+	// Redis under contention.
+	defaultPollInterval = 200 * time.Millisecond
+	// keyPrefix namespaces the semaphore ZSETs in Redis.
+	keyPrefix = "weknora:modelsem:"
+)
+
+// acquireScript atomically prunes expired leases, counts live holders, and
+// admits the caller (adding its token scored by lease expiry) only while the
+// live count is below the limit. Returns 1 on admission, 0 when full.
+//
+//	KEYS[1] = semaphore ZSET key
+//	ARGV[1] = now (unix ms)
+//	ARGV[2] = limit
+//	ARGV[3] = caller token
+//	ARGV[4] = lease TTL (ms)
+var acquireScript = redis.NewScript(`
+redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1])
+local count = redis.call('ZCARD', KEYS[1])
+if count < tonumber(ARGV[2]) then
+    redis.call('ZADD', KEYS[1], ARGV[1] + ARGV[4], ARGV[3])
+    redis.call('PEXPIRE', KEYS[1], ARGV[4] * 2)
+    return 1
+end
+return 0
+`)
+
+type redisLimiter struct {
+	rdb          *redis.Client
+	ttl          time.Duration
+	pollInterval time.Duration
+	tracked      sync.Map // model ID -> *trackedSemaphore
+}
+
+// NewRedisLimiter builds a distributed limiter backed by rdb. A nil client
+// yields a limiter that always fails open.
+func NewRedisLimiter(rdb *redis.Client) ModelConcurrencyLimiter {
+	return &redisLimiter{
+		rdb:          rdb,
+		ttl:          defaultLeaseTTL,
+		pollInterval: defaultPollInterval,
+	}
+}
+
+func (l *redisLimiter) Acquire(ctx context.Context, key string, limit int) (func(), error) {
+	if l == nil || l.rdb == nil || limit <= 0 || key == "" {
+		return noop, nil
+	}
+
+	zkey := keyPrefix + key
+	entry, _ := l.tracked.LoadOrStore(key, &trackedSemaphore{})
+	tracked := entry.(*trackedSemaphore)
+	tracked.limit.Store(int64(limit))
+	tracked.waiting.Add(1)
+	defer tracked.waiting.Add(-1)
+	token := uuid.NewString()
+	ttlMs := l.ttl.Milliseconds()
+
+	// Reuse a single timer across poll iterations rather than allocating a new
+	// one via time.After each loop: under sustained contention a waiter can
+	// spin thousands of times, and every time.After timer lives until it fires.
+	// Start it stopped so the first Reset below arms it cleanly.
+	timer := time.NewTimer(0)
+	if !timer.Stop() {
+		<-timer.C
+	}
+	defer timer.Stop()
+
+	for {
+		now := time.Now().UnixMilli()
+		res, err := acquireScript.Run(ctx, l.rdb, []string{zkey},
+			now, limit, token, ttlMs).Int()
+		if err != nil {
+			// Fail open: a limiter outage must never block model traffic.
+			logger.Warnf(ctx, "[ModelLimiter] acquire failed for key=%s, failing open: %v", key, err)
+			return noop, nil
+		}
+		if res == 1 {
+			return l.hold(zkey, token), nil
+		}
+
+		timer.Reset(l.pollInterval)
+		select {
+		case <-ctx.Done():
+			// Fail open on cancellation too: let the inner call observe the
+			// cancelled context and return its own error, rather than us
+			// synthesising one here.
+			return noop, nil
+		case <-timer.C:
+		}
+	}
+}
+
+func (l *redisLimiter) RuntimeStats(ctx context.Context) ([]RuntimeStat, error) {
+	stats := make([]RuntimeStat, 0)
+	if l == nil || l.rdb == nil {
+		return stats, nil
+	}
+	var firstErr error
+	now := time.Now().UnixMilli()
+	l.tracked.Range(func(rawKey, rawValue any) bool {
+		modelID := rawKey.(string)
+		tracked := rawValue.(*trackedSemaphore)
+		active, err := l.rdb.ZCount(ctx, keyPrefix+modelID, strconv.FormatInt(now+1, 10), "+inf").Result()
+		if err != nil {
+			if firstErr == nil {
+				firstErr = err
+			}
+			return true
+		}
+		name, _ := tracked.name.Load().(string)
+		stats = append(stats, RuntimeStat{ModelID: modelID, Name: name, Active: active, Waiting: tracked.waiting.Load(), Limit: int(tracked.limit.Load())})
+		return true
+	})
+	sort.Slice(stats, func(i, j int) bool { return stats[i].ModelID < stats[j].ModelID })
+	return stats, firstErr
+}
+
+func (l *redisLimiter) SetModelName(modelID, name string) {
+	if modelID == "" || name == "" {
+		return
+	}
+	entry, _ := l.tracked.LoadOrStore(modelID, &trackedSemaphore{})
+	entry.(*trackedSemaphore).name.Store(name)
+}
+
+// hold starts a heartbeat that refreshes the lease and returns an idempotent
+// release that stops the heartbeat and drops the slot.
+func (l *redisLimiter) hold(zkey, token string) func() {
+	stop := make(chan struct{})
+	go func() {
+		t := time.NewTicker(l.ttl / 3)
+		defer t.Stop()
+		for {
+			select {
+			case <-stop:
+				return
+			case <-t.C:
+				now := time.Now().UnixMilli()
+				// Detached ctx: the heartbeat must outlive request ctx up to
+				// release. Best-effort; a failed refresh just risks early
+				// reclamation, which the limit already tolerates.
+				//
+				// Refresh BOTH the member lease score AND the ZSET key's own
+				// TTL. The acquire script only PEXPIREs the key on admission,
+				// so a semaphore that stays saturated with no slot turnover
+				// would otherwise let the whole key expire after ttl*2 —
+				// dropping every live lease and admitting over the limit. The
+				// heartbeat pushes the key TTL out in lockstep with the lease.
+				bg := context.Background()
+				_ = l.rdb.ZAdd(bg, zkey, redis.Z{
+					Score:  float64(now + l.ttl.Milliseconds()),
+					Member: token,
+				}).Err()
+				_ = l.rdb.PExpire(bg, zkey, l.ttl*2).Err()
+			}
+		}
+	}()
+
+	var once sync.Once
+	return func() {
+		once.Do(func() {
+			close(stop)
+			_ = l.rdb.ZRem(context.Background(), zkey, token).Err()
+		})
+	}
+}

internal/models/limiter/limiter_test.go

@@ -0,0 +1,175 @@
+package limiter
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/alicebob/miniredis/v2"
+	"github.com/redis/go-redis/v9"
+)
+
+func newTestLimiter(t *testing.T, ttl, poll time.Duration) (*redisLimiter, *miniredis.Miniredis, *redis.Client) {
+	t.Helper()
+	s, err := miniredis.Run()
+	if err != nil {
+		t.Fatalf("start miniredis: %v", err)
+	}
+	t.Cleanup(s.Close)
+	rdb := redis.NewClient(&redis.Options{Addr: s.Addr()})
+	t.Cleanup(func() { _ = rdb.Close() })
+	return &redisLimiter{rdb: rdb, ttl: ttl, pollInterval: poll}, s, rdb
+}
+
+func zcard(t *testing.T, rdb *redis.Client, key string) int64 {
+	t.Helper()
+	n, err := rdb.ZCard(context.Background(), keyPrefix+key).Result()
+	if err != nil {
+		t.Fatalf("zcard: %v", err)
+	}
+	return n
+}
+
+// TestRedisLimiterCapsConcurrency verifies the live-holder count never exceeds
+// the limit and that a freed slot lets a waiting acquirer through.
+func TestRedisLimiterCapsConcurrency(t *testing.T) {
+	lim, _, rdb := newTestLimiter(t, time.Minute, 10*time.Millisecond)
+	ctx := context.Background()
+	const key = "m1"
+
+	r1, err := lim.Acquire(ctx, key, 2)
+	if err != nil {
+		t.Fatalf("acquire 1: %v", err)
+	}
+	r2, err := lim.Acquire(ctx, key, 2)
+	if err != nil {
+		t.Fatalf("acquire 2: %v", err)
+	}
+	if got := zcard(t, rdb, key); got != 2 {
+		t.Fatalf("expected 2 live holders, got %d", got)
+	}
+
+	// The third acquire must block while the semaphore is full.
+	done := make(chan func(), 1)
+	go func() {
+		r3, _ := lim.Acquire(ctx, key, 2)
+		done <- r3
+	}()
+
+	select {
+	case <-done:
+		t.Fatal("third acquire should block while at limit")
+	case <-time.After(100 * time.Millisecond):
+	}
+	if got := zcard(t, rdb, key); got != 2 {
+		t.Fatalf("count must not exceed limit while blocked, got %d", got)
+	}
+
+	// Freeing a slot lets the waiter in.
+	r1()
+	var r3 func()
+	select {
+	case r3 = <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("third acquire should proceed after a slot frees")
+	}
+	if got := zcard(t, rdb, key); got != 2 {
+		t.Fatalf("expected 2 live holders after handoff, got %d", got)
+	}
+
+	r2()
+	r3()
+	if got := zcard(t, rdb, key); got != 0 {
+		t.Fatalf("expected 0 live holders after release, got %d", got)
+	}
+}
+
+// TestRedisLimiterReclaimsExpiredLease verifies a crashed holder's stale lease
+// (no heartbeat) is pruned so its slot is reclaimed by a new acquirer.
+func TestRedisLimiterReclaimsExpiredLease(t *testing.T) {
+	lim, _, rdb := newTestLimiter(t, time.Minute, 10*time.Millisecond)
+	ctx := context.Background()
+	const key = "m2"
+
+	// Simulate a dead holder: a member whose lease already expired.
+	expired := float64(time.Now().Add(-time.Second).UnixMilli())
+	if err := rdb.ZAdd(ctx, keyPrefix+key, redis.Z{Score: expired, Member: "dead"}).Err(); err != nil {
+		t.Fatalf("seed dead holder: %v", err)
+	}
+
+	// limit=1 would be full if the dead lease still counted; pruning reclaims it.
+	release, err := lim.Acquire(ctx, key, 1)
+	if err != nil {
+		t.Fatalf("acquire after expiry: %v", err)
+	}
+	if got := zcard(t, rdb, key); got != 1 {
+		t.Fatalf("expected only the live holder, got %d", got)
+	}
+	if exists, _ := rdb.ZScore(ctx, keyPrefix+key, "dead").Result(); exists != 0 {
+		t.Fatal("expired lease should have been pruned")
+	}
+	release()
+}
+
+// TestRedisLimiterIndependentKeys verifies budgets are per key.
+func TestRedisLimiterIndependentKeys(t *testing.T) {
+	lim, _, _ := newTestLimiter(t, time.Minute, 10*time.Millisecond)
+	ctx := context.Background()
+
+	ra, err := lim.Acquire(ctx, "a", 1)
+	if err != nil {
+		t.Fatalf("acquire a: %v", err)
+	}
+	// A different key has its own budget, so this must not block.
+	rb, err := lim.Acquire(ctx, "b", 1)
+	if err != nil {
+		t.Fatalf("acquire b: %v", err)
+	}
+	ra()
+	rb()
+}
+
+// TestRedisLimiterFailsOpen verifies every degraded path allows the call
+// instead of blocking model traffic.
+func TestRedisLimiterFailsOpen(t *testing.T) {
+	ctx := context.Background()
+
+	// Nil client.
+	lim := &redisLimiter{rdb: nil, ttl: time.Minute, pollInterval: 10 * time.Millisecond}
+	if release, err := lim.Acquire(ctx, "k", 4); err != nil || release == nil {
+		t.Fatalf("nil client should fail open, got release==nil=%v err=%v", release == nil, err)
+	}
+
+	// Non-positive limit is a no-op governor.
+	lim2, _, _ := newTestLimiter(t, time.Minute, 10*time.Millisecond)
+	if release, err := lim2.Acquire(ctx, "k", 0); err != nil || release == nil {
+		t.Fatalf("limit<=0 should fail open, got release==nil=%v err=%v", release == nil, err)
+	}
+
+	// Backend error (server down) must fail open too.
+	lim3, s, _ := newTestLimiter(t, time.Minute, 10*time.Millisecond)
+	s.Close()
+	if release, err := lim3.Acquire(ctx, "k", 4); err != nil || release == nil {
+		t.Fatalf("backend error should fail open, got release==nil=%v err=%v", release == nil, err)
+	}
+}
+
+// TestRedisLimiterCancelledContextFailsOpen verifies a waiter whose context is
+// cancelled while blocked returns a usable (no-op) release rather than erroring.
+func TestRedisLimiterCancelledContextFailsOpen(t *testing.T) {
+	lim, _, _ := newTestLimiter(t, time.Minute, 10*time.Millisecond)
+
+	r1, err := lim.Acquire(context.Background(), "m3", 1)
+	if err != nil {
+		t.Fatalf("acquire: %v", err)
+	}
+	defer r1()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
+	defer cancel()
+	release, err := lim.Acquire(ctx, "m3", 1)
+	if err != nil || release == nil {
+		t.Fatalf("cancelled wait should fail open, got release==nil=%v err=%v", release == nil, err)
+	}
+	release()
+}