WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/stream/factory.go

@@ -0,0 +1,37 @@
+package stream
+
+import (
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// 流管理器类型
+const (
+	TypeMemory = "memory"
+	TypeRedis  = "redis"
+)
+
+// NewStreamManager 创建流管理器
+func NewStreamManager() (interfaces.StreamManager, error) {
+	switch os.Getenv("STREAM_MANAGER_TYPE") {
+	case TypeRedis:
+		db, err := strconv.Atoi(os.Getenv("REDIS_DB"))
+		if err != nil {
+			db = 0
+		}
+		ttl := time.Hour // 默认1小时
+		return NewRedisStreamManager(
+			os.Getenv("REDIS_ADDR"),
+			os.Getenv("REDIS_USERNAME"),
+			os.Getenv("REDIS_PASSWORD"),
+			db,
+			os.Getenv("REDIS_PREFIX"),
+			ttl,
+		)
+	default:
+		return NewMemoryStreamManager(), nil
+	}
+}

internal/stream/memory_manager.go

@@ -0,0 +1,118 @@
+package stream
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// memoryStreamData holds stream events in memory
+type memoryStreamData struct {
+	events      []interfaces.StreamEvent
+	lastUpdated time.Time
+	mu          sync.RWMutex
+}
+
+// MemoryStreamManager implements StreamManager using in-memory storage
+type MemoryStreamManager struct {
+	// Map: sessionID -> messageID -> stream data
+	streams map[string]map[string]*memoryStreamData
+	mu      sync.RWMutex
+}
+
+// NewMemoryStreamManager creates a new in-memory stream manager
+func NewMemoryStreamManager() *MemoryStreamManager {
+	return &MemoryStreamManager{
+		streams: make(map[string]map[string]*memoryStreamData),
+	}
+}
+
+// getOrCreateStream gets or creates stream data
+func (m *MemoryStreamManager) getOrCreateStream(sessionID, messageID string) *memoryStreamData {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if _, exists := m.streams[sessionID]; !exists {
+		m.streams[sessionID] = make(map[string]*memoryStreamData)
+	}
+
+	if _, exists := m.streams[sessionID][messageID]; !exists {
+		m.streams[sessionID][messageID] = &memoryStreamData{
+			events:      make([]interfaces.StreamEvent, 0),
+			lastUpdated: time.Now(),
+		}
+	}
+
+	return m.streams[sessionID][messageID]
+}
+
+// getStream gets existing stream data (returns nil if not found)
+func (m *MemoryStreamManager) getStream(sessionID, messageID string) *memoryStreamData {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	if sessionMap, exists := m.streams[sessionID]; exists {
+		return sessionMap[messageID]
+	}
+	return nil
+}
+
+// AppendEvent appends a single event to the stream
+func (m *MemoryStreamManager) AppendEvent(
+	ctx context.Context,
+	sessionID, messageID string,
+	event interfaces.StreamEvent,
+) error {
+	stream := m.getOrCreateStream(sessionID, messageID)
+
+	stream.mu.Lock()
+	defer stream.mu.Unlock()
+
+	// Set timestamp if not already set
+	if event.Timestamp.IsZero() {
+		event.Timestamp = time.Now()
+	}
+
+	// Append event
+	stream.events = append(stream.events, event)
+	stream.lastUpdated = time.Now()
+
+	return nil
+}
+
+// GetEvents gets events starting from offset
+// Returns: events slice, next offset, error
+func (m *MemoryStreamManager) GetEvents(
+	ctx context.Context,
+	sessionID, messageID string,
+	fromOffset int,
+) ([]interfaces.StreamEvent, int, error) {
+	stream := m.getStream(sessionID, messageID)
+	if stream == nil {
+		// Stream doesn't exist yet
+		return []interfaces.StreamEvent{}, fromOffset, nil
+	}
+
+	stream.mu.RLock()
+	defer stream.mu.RUnlock()
+
+	// Check if offset is beyond current events
+	if fromOffset >= len(stream.events) {
+		return []interfaces.StreamEvent{}, fromOffset, nil
+	}
+
+	// Get events from offset to end
+	events := stream.events[fromOffset:]
+	nextOffset := len(stream.events)
+
+	// Return copy of events to avoid race conditions
+	eventsCopy := make([]interfaces.StreamEvent, len(events))
+	copy(eventsCopy, events)
+
+	return eventsCopy, nextOffset, nil
+}
+
+// Ensure MemoryStreamManager implements StreamManager interface
+var _ interfaces.StreamManager = (*MemoryStreamManager)(nil)

internal/stream/redis_manager.go

@@ -0,0 +1,139 @@
+package stream
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/common"
+	"github.com/Tencent/WeKnora/internal/types/interfaces"
+	"github.com/redis/go-redis/v9"
+)
+
+// RedisStreamManager implements StreamManager using Redis Lists for append-only event streaming
+type RedisStreamManager struct {
+	client *redis.Client
+	ttl    time.Duration // TTL for stream data in Redis
+	prefix string        // Redis key prefix
+}
+
+// NewRedisStreamManager creates a new Redis-based stream manager
+func NewRedisStreamManager(redisAddr, redisUsername, redisPassword string,
+	redisDB int, prefix string, ttl time.Duration,
+) (*RedisStreamManager, error) {
+	client := redis.NewClient(&redis.Options{
+		Addr:      redisAddr,
+		Username:  redisUsername,
+		Password:  redisPassword,
+		DB:        redisDB,
+		TLSConfig: common.RedisTLSConfig(),
+	})
+
+	// Verify connection
+	_, err := client.Ping(context.Background()).Result()
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to Redis: %w", err)
+	}
+
+	if ttl == 0 {
+		ttl = 24 * time.Hour // Default TTL: 24 hours
+	}
+
+	if prefix == "" {
+		prefix = "stream:events" // Default prefix
+	}
+
+	return &RedisStreamManager{
+		client: client,
+		ttl:    ttl,
+		prefix: prefix,
+	}, nil
+}
+
+// buildKey builds the Redis key for event list
+func (r *RedisStreamManager) buildKey(sessionID, messageID string) string {
+	return fmt.Sprintf("%s:%s:%s", r.prefix, sessionID, messageID)
+}
+
+// AppendEvent appends a single event to the stream using Redis RPush
+func (r *RedisStreamManager) AppendEvent(
+	ctx context.Context,
+	sessionID, messageID string,
+	event interfaces.StreamEvent,
+) error {
+	key := r.buildKey(sessionID, messageID)
+
+	// Set timestamp if not already set
+	if event.Timestamp.IsZero() {
+		event.Timestamp = time.Now()
+	}
+
+	// Serialize event to JSON
+	eventJSON, err := json.Marshal(event)
+	if err != nil {
+		return fmt.Errorf("failed to marshal event: %w", err)
+	}
+
+	// Append to Redis list with RPush (O(1) operation)
+	if err := r.client.RPush(ctx, key, eventJSON).Err(); err != nil {
+		return fmt.Errorf("failed to append event to Redis: %w", err)
+	}
+
+	// Set/refresh TTL on the key
+	if err := r.client.Expire(ctx, key, r.ttl).Err(); err != nil {
+		return fmt.Errorf("failed to set TTL: %w", err)
+	}
+
+	return nil
+}
+
+// GetEvents gets events starting from offset using Redis LRange
+// Returns: events slice, next offset, error
+func (r *RedisStreamManager) GetEvents(
+	ctx context.Context,
+	sessionID, messageID string,
+	fromOffset int,
+) ([]interfaces.StreamEvent, int, error) {
+	key := r.buildKey(sessionID, messageID)
+
+	// Get all events from offset to end using LRange
+	// LRange is inclusive, so fromOffset to -1 gets all remaining elements
+	results, err := r.client.LRange(ctx, key, int64(fromOffset), -1).Result()
+	if err != nil {
+		if err == redis.Nil {
+			// Key doesn't exist - return empty slice
+			return []interfaces.StreamEvent{}, fromOffset, nil
+		}
+		return nil, fromOffset, fmt.Errorf("failed to get events from Redis: %w", err)
+	}
+
+	// No new events
+	if len(results) == 0 {
+		return []interfaces.StreamEvent{}, fromOffset, nil
+	}
+
+	// Unmarshal events
+	events := make([]interfaces.StreamEvent, 0, len(results))
+	for _, result := range results {
+		var event interfaces.StreamEvent
+		if err := json.Unmarshal([]byte(result), &event); err != nil {
+			// Log error but continue with other events
+			continue
+		}
+		events = append(events, event)
+	}
+
+	// Calculate next offset
+	nextOffset := fromOffset + len(results)
+
+	return events, nextOffset, nil
+}
+
+// Close closes the Redis connection
+func (r *RedisStreamManager) Close() error {
+	return r.client.Close()
+}
+
+// Ensure RedisStreamManager implements StreamManager interface
+var _ interfaces.StreamManager = (*RedisStreamManager)(nil)