WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/tracing/langfuse/asynq.go

@@ -0,0 +1,207 @@
+package langfuse
+
+import (
+	"context"
+	"encoding/json"
+	"strconv"
+
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/hibiken/asynq"
+	"go.opentelemetry.io/otel/propagation"
+	oteltrace "go.opentelemetry.io/otel/trace"
+)
+
+// InjectTracing stamps the current W3C traceparent (plus a best-effort
+// user/session label) from ctx onto the given payload, provided the payload
+// embeds types.TracingContext (and thus implements LangfuseTracingCarrier).
+//
+// The traceparent is produced by propagator.Inject from the active OTel span
+// context (the HTTP root span opened by GinMiddleware). The asynq worker
+// re-extracts it so worker-side spans are children of the same trace —
+// giving LiteFuse one stitched tree across the HTTP request and the async
+// processing. This also makes a sop3 run's traceparent propagate through to
+// any asynq jobs WeKnora enqueues while serving sop3's agent-chat call.
+//
+// Safe to call unconditionally: when Langfuse is disabled or no span is
+// present on ctx, it writes a zero-valued TracingContext — which round-trips
+// through JSON as absent fields and costs nothing.
+func InjectTracing(ctx context.Context, carrier types.LangfuseTracingCarrier) {
+	if carrier == nil {
+		return
+	}
+	mgr := GetManager()
+	if !mgr.Enabled() {
+		return
+	}
+	tc := types.TracingContext{}
+	c := propagation.MapCarrier{}
+	propagator.Inject(ctx, c)
+	tc.LangfuseTraceparent = c["traceparent"]
+	// Backward-compat: keep LangfuseTraceID = the W3C trace id for any legacy
+	// reader. LangfuseParentObservationID is no longer used by the OTLP path.
+	if trace, ok := TraceFromContext(ctx); ok && trace != nil {
+		tc.LangfuseTraceID = trace.ID
+	}
+	tc.LangfuseUserID = userIDFromCtx(ctx)
+	tc.LangfuseSessionID = sessionIDFromCtx(ctx)
+	carrier.SetLangfuseTracing(tc)
+}
+
+// peekTracingContext pulls just the Langfuse tracing fields out of a raw
+// asynq payload. It's deliberately lax: every payload type in the project
+// is a JSON object at the top level, and absent/mismatched fields decode to
+// the zero value. If unmarshalling fails entirely (e.g. the payload isn't
+// JSON at all) we return a zero TracingContext and let the main handler
+// deal with its own error — we never want an observability bug to mask a
+// real task failure.
+func peekTracingContext(payload []byte) types.TracingContext {
+	if len(payload) == 0 {
+		return types.TracingContext{}
+	}
+	var tc types.TracingContext
+	_ = json.Unmarshal(payload, &tc)
+	return tc
+}
+
+// AsynqMiddleware is the worker-side counterpart of GinMiddleware. It:
+//
+//  1. Extracts the W3C traceparent stamped onto the task payload by
+//     InjectTracing and resumes the originating trace (so the Langfuse UI
+//     stitches the HTTP request and the async processing into one tree). For
+//     scheduled jobs with no upstream trace it opens a standalone trace named
+//     after the task type.
+//
+//  2. Opens a SPAN around the handler execution so every child generation
+//     (embedding / VLM / chat / rerank / ASR) auto-attaches to it.
+//
+//  3. Enriches the span with asynq's own metadata: task id, queue, retry
+//     count, payload size.
+//
+// When the manager is disabled it degrades to a pass-through; failure of the
+// Langfuse path never blocks task execution. Register once via mux.Use.
+func AsynqMiddleware() asynq.MiddlewareFunc {
+	return func(next asynq.Handler) asynq.Handler {
+		return asynq.HandlerFunc(func(ctx context.Context, task *asynq.Task) error {
+			mgr := GetManager()
+			if !mgr.Enabled() {
+				return next.ProcessTask(ctx, task)
+			}
+
+			tc := peekTracingContext(task.Payload())
+			taskID, _ := asynq.GetTaskID(ctx)
+			retryCount, _ := asynq.GetRetryCount(ctx)
+			maxRetry, _ := asynq.GetMaxRetry(ctx)
+			queueName, _ := asynq.GetQueueName(ctx)
+
+			meta := map[string]interface{}{
+				"task_type":     task.Type(),
+				"task_id":       taskID,
+				"queue":         queueName,
+				"retry":         retryCount,
+				"max_retry":     maxRetry,
+				"payload_bytes": len(task.Payload()),
+			}
+
+			// If the upstream enqueuer stamped a traceparent, resume that trace
+			// (worker spans become children of the HTTP trace). Otherwise start
+			// a standalone trace named after the task type.
+			var trace *Trace
+			shouldFinishTrace := false
+			if tc.LangfuseTraceparent != "" {
+				ctx = propagator.Extract(ctx, propagation.MapCarrier{"traceparent": tc.LangfuseTraceparent})
+				if sc := oteltrace.SpanContextFromContext(ctx); sc.IsValid() {
+					ctx = withTrace(ctx, &Trace{ID: sc.TraceID().String(), manager: mgr})
+				}
+			} else {
+				ctx, trace = mgr.StartTrace(ctx, TraceOptions{
+					Name:      "asynq." + task.Type(),
+					UserID:    firstNonEmptyString(tc.LangfuseUserID, userIDFromCtx(ctx)),
+					SessionID: firstNonEmptyString(tc.LangfuseSessionID, sessionIDFromCtx(ctx)),
+					Metadata:  meta,
+					Tags:      []string{"asynq", task.Type()},
+				})
+				shouldFinishTrace = true
+			}
+
+			ctx, span := mgr.StartSpan(ctx, SpanOptions{
+				Name:     "asynq." + task.Type(),
+				Input:    spanInputFromPayload(task.Payload()),
+				Metadata: meta,
+			})
+
+			err := next.ProcessTask(ctx, task)
+
+			outcome := "success"
+			if err != nil {
+				outcome = "error"
+			}
+			span.Finish(map[string]interface{}{
+				"outcome": outcome,
+			}, map[string]interface{}{
+				"outcome": outcome,
+			}, err)
+
+			if shouldFinishTrace {
+				trace.Finish(map[string]interface{}{
+					"outcome": outcome,
+				}, map[string]interface{}{
+					"task_type": task.Type(),
+					"outcome":   outcome,
+				})
+			}
+
+			return err
+		})
+	}
+}
+
+// spanInputFromPayload surfaces a compact, human-readable summary of the
+// task payload for the Langfuse "Input" pane. We deliberately do NOT send
+// the full JSON blob because manual/text-ingest payloads can be many
+// kilobytes and FAQ import payloads embed the full entry list. Instead we
+// preview the first ~1KB verbatim.
+func spanInputFromPayload(payload []byte) interface{} {
+	const preview = 1024
+	if len(payload) == 0 {
+		return nil
+	}
+	if len(payload) <= preview {
+		return string(payload)
+	}
+	return map[string]interface{}{
+		"preview": string(payload[:preview]) + "...",
+		"bytes":   len(payload),
+	}
+}
+
+// userIDFromCtx mirrors middleware.extractUserID but accepts a raw context
+// (no gin.Context) so both HTTP and asynq paths share the same fallback
+// logic: explicit UserID → tenant:<id> → empty.
+func userIDFromCtx(ctx context.Context) string {
+	if v, ok := ctx.Value(types.UserIDContextKey).(string); ok && v != "" {
+		return v
+	}
+	if v, ok := ctx.Value(types.TenantIDContextKey).(uint64); ok && v != 0 {
+		return "tenant:" + strconv.FormatUint(v, 10)
+	}
+	return ""
+}
+
+// sessionIDFromCtx pulls a best-effort "session" label. For HTTP chat this
+// is already set by GinMiddleware; for async work we fall back to the
+// request id so retries of the same logical task group together.
+func sessionIDFromCtx(ctx context.Context) string {
+	if v, ok := types.RequestIDFromContext(ctx); ok && v != "" {
+		return v
+	}
+	return ""
+}
+
+func firstNonEmptyString(vals ...string) string {
+	for _, v := range vals {
+		if v != "" {
+			return v
+		}
+	}
+	return ""
+}

internal/tracing/langfuse/asynq_test.go

@@ -0,0 +1,123 @@
+package langfuse
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/hibiken/asynq"
+)
+
+// dummyPayload is a minimal payload that embeds TracingContext, mirroring
+// how real asynq payloads opt into trace propagation.
+type dummyPayload struct {
+	types.TracingContext
+	KnowledgeID string `json:"knowledge_id"`
+}
+
+// TestInjectTracing_DisabledIsZero verifies InjectTracing is a no-op when
+// Langfuse is disabled: no panics, no trace fields written.
+func TestInjectTracing_DisabledIsZero(t *testing.T) {
+	_, _ = Init(Config{Enabled: false})
+
+	p := &dummyPayload{KnowledgeID: "k1"}
+	InjectTracing(context.Background(), p)
+	if p.LangfuseTraceparent != "" || p.LangfuseTraceID != "" {
+		t.Fatalf("expected no tracing fields on disabled manager, got %+v", p.TracingContext)
+	}
+}
+
+// TestInjectTracing_PopulatesTraceparent checks that when a trace is active
+// on the context, a W3C traceparent is stamped onto the payload (so the
+// asynq worker can resume the same trace).
+func TestInjectTracing_PopulatesTraceparent(t *testing.T) {
+	m, _ := newTestManager(t)
+
+	ctx, trace := m.StartTrace(context.Background(), TraceOptions{Name: "parent"})
+	p := &dummyPayload{KnowledgeID: "k1"}
+	InjectTracing(ctx, p)
+
+	if p.LangfuseTraceparent == "" {
+		t.Fatal("expected LangfuseTraceparent to be populated")
+	}
+	// The traceparent carries the trace id; trace.ID is the OTel trace id.
+	if !strings.HasPrefix(p.LangfuseTraceparent, "00-"+trace.ID) {
+		t.Errorf("traceparent %q does not carry trace id %s", p.LangfuseTraceparent, trace.ID)
+	}
+	if p.LangfuseTraceID != trace.ID {
+		t.Errorf("LangfuseTraceID = %q, want %q", p.LangfuseTraceID, trace.ID)
+	}
+}
+
+// TestAsynqMiddleware_TraceparentPropagation is the cross-process correlation
+// core test: InjectTracing stamps a traceparent onto the payload; the worker
+// middleware re-extracts it, and the worker span inherits the upstream trace
+// id — stitching the HTTP trace and the async job into one LiteFuse tree.
+func TestAsynqMiddleware_TraceparentPropagation(t *testing.T) {
+	m, exp := newTestManager(t)
+
+	// Upstream caller opens a span and injects a traceparent onto the payload.
+	upstreamCtx, upstreamSpan := m.Tracer().Start(context.Background(), "upstream-http")
+	remoteTraceID := upstreamSpan.SpanContext().TraceID()
+	payload := &dummyPayload{KnowledgeID: "k1"}
+	InjectTracing(upstreamCtx, payload)
+	if payload.LangfuseTraceparent == "" {
+		t.Fatal("InjectTracing did not stamp a traceparent")
+	}
+	raw, _ := json.Marshal(payload)
+
+	mw := AsynqMiddleware()(asynq.HandlerFunc(func(context.Context, *asynq.Task) error { return nil }))
+	if err := mw.ProcessTask(context.Background(), asynq.NewTask("test:type", raw)); err != nil {
+		t.Fatalf("handler err: %v", err)
+	}
+
+	for _, s := range exp.GetSpans() {
+		if s.Name != "asynq.test:type" {
+			continue
+		}
+		if s.SpanContext.TraceID() != remoteTraceID {
+			t.Errorf("worker span trace id = %s, want upstream %s (traceparent not propagated)",
+				s.SpanContext.TraceID(), remoteTraceID)
+		}
+		return
+	}
+	t.Fatal("asynq worker span not exported")
+}
+
+// TestAsynqMiddleware_StandaloneTrace asserts that when the payload carries
+// NO upstream traceparent (e.g. a scheduled job), the middleware opens a
+// standalone trace named after the task type.
+func TestAsynqMiddleware_StandaloneTrace(t *testing.T) {
+	_, exp := newTestManager(t)
+
+	payload := &dummyPayload{KnowledgeID: "kX"}
+	raw, _ := json.Marshal(payload)
+
+	mw := AsynqMiddleware()(asynq.HandlerFunc(func(context.Context, *asynq.Task) error { return nil }))
+	if err := mw.ProcessTask(context.Background(), asynq.NewTask("scheduled:ping", raw)); err != nil {
+		t.Fatalf("handler err: %v", err)
+	}
+
+	// The standalone run opens a root trace span ("asynq.scheduled:ping",
+	// type=trace) plus a worker span with the same name (type=span).
+	var sawRoot, sawSpan bool
+	for _, s := range exp.GetSpans() {
+		if s.Name != "asynq.scheduled:ping" {
+			continue
+		}
+		switch spanType(s) {
+		case obsTypeTrace:
+			sawRoot = true
+		case obsTypeSpan:
+			sawSpan = true
+		}
+	}
+	if !sawRoot {
+		t.Error("standalone run should open a root trace span named asynq.scheduled:ping")
+	}
+	if !sawSpan {
+		t.Error("standalone run should open a worker span")
+	}
+}

internal/tracing/langfuse/config.go

@@ -0,0 +1,153 @@
+// Package langfuse implements WeKnora's Langfuse/LiteFuse LLM tracing client on
+// top of the OpenTelemetry Go SDK. It records chat/embedding/rerank/VLM/ASR
+// generations plus the surrounding agent/HTTP/asynq spans as OTLP/HTTP spans
+// to a Langfuse v3+ or LiteFuse backend (POST /api/public/otel/v1/traces).
+//
+// The integration is fully opt-in: when disabled (the default), every public
+// entry point is a cheap no-op, so callers can wire them unconditionally. A
+// W3C traceparent propagated from upstream callers (e.g. sop3) is inherited so
+// WeKnora's trace shares the upstream trace id — correlating runs across
+// services in the LiteFuse UI.
+package langfuse
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+// Config holds the runtime configuration for the Langfuse client.
+//
+// In practice users enable Langfuse purely through environment variables —
+// Host / PublicKey / SecretKey — which matches every other Langfuse SDK and
+// keeps WeKnora's YAML config free of secrets.
+type Config struct {
+	// Enabled is the master switch. If false the entire package is a no-op.
+	Enabled bool
+	// Host is the Langfuse base URL, e.g. https://cloud.langfuse.com or
+	// https://us.cloud.langfuse.com or a self-hosted address.
+	Host string
+	// PublicKey / SecretKey are the project credentials used for Basic Auth.
+	PublicKey string
+	SecretKey string
+	// FlushAt flushes the queued events once the buffer reaches this size.
+	FlushAt int
+	// FlushInterval is the maximum time between automatic flushes.
+	FlushInterval time.Duration
+	// QueueSize bounds the in-memory buffer to avoid unbounded growth if the
+	// Langfuse endpoint is unreachable.
+	QueueSize int
+	// RequestTimeout is the HTTP timeout for a single ingestion batch.
+	RequestTimeout time.Duration
+	// Release / Environment are attached to every trace for filtering in the
+	// Langfuse UI (e.g. release="v0.4.2", environment="production").
+	Release     string
+	Environment string
+	// SampleRate (0..1) controls trace sampling. 0 means "use 1.0".
+	SampleRate float64
+	// Debug enables verbose logging of batch send errors.
+	Debug bool
+
+	// testExporter, when non-nil, replaces the real OTLP exporter. Tests use
+	// it to inject an in-memory span exporter (tracetest.InMemoryExporter) so
+	// they can assert on exported spans deterministically without an HTTP
+	// server. Init registers it with a SimpleSpanProcessor (synchronous
+	// export on span End) when set. Unexported — production never sets it.
+	testExporter sdktrace.SpanExporter
+}
+
+// LoadConfigFromEnv builds a Config by reading the LANGFUSE_* environment
+// variables, mirroring the official Python / JS SDK conventions.
+func LoadConfigFromEnv() Config {
+	cfg := Config{
+		Host:           firstNonEmpty(os.Getenv("LANGFUSE_HOST"), "https://cloud.langfuse.com"),
+		PublicKey:      strings.TrimSpace(os.Getenv("LANGFUSE_PUBLIC_KEY")),
+		SecretKey:      strings.TrimSpace(os.Getenv("LANGFUSE_SECRET_KEY")),
+		Release:        strings.TrimSpace(os.Getenv("LANGFUSE_RELEASE")),
+		Environment:    strings.TrimSpace(os.Getenv("LANGFUSE_ENVIRONMENT")),
+		FlushAt:        15,
+		FlushInterval:  3 * time.Second,
+		QueueSize:      2048,
+		RequestTimeout: 10 * time.Second,
+		SampleRate:     1.0,
+	}
+
+	if v := strings.TrimSpace(os.Getenv("LANGFUSE_ENABLED")); v != "" {
+		cfg.Enabled = parseBool(v)
+	} else if cfg.PublicKey != "" && cfg.SecretKey != "" {
+		// Auto-enable when credentials are present — matches the Python SDK.
+		cfg.Enabled = true
+	}
+
+	if v := strings.TrimSpace(os.Getenv("LANGFUSE_FLUSH_AT")); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			cfg.FlushAt = n
+		}
+	}
+	if v := strings.TrimSpace(os.Getenv("LANGFUSE_FLUSH_INTERVAL")); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.FlushInterval = d
+		} else if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			cfg.FlushInterval = time.Duration(n) * time.Second
+		}
+	}
+	if v := strings.TrimSpace(os.Getenv("LANGFUSE_QUEUE_SIZE")); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			cfg.QueueSize = n
+		}
+	}
+	if v := strings.TrimSpace(os.Getenv("LANGFUSE_REQUEST_TIMEOUT")); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.RequestTimeout = d
+		}
+	}
+	if v := strings.TrimSpace(os.Getenv("LANGFUSE_SAMPLE_RATE")); v != "" {
+		if f, err := strconv.ParseFloat(v, 64); err == nil && f >= 0 && f <= 1 {
+			cfg.SampleRate = f
+		}
+	}
+	if v := strings.TrimSpace(os.Getenv("LANGFUSE_DEBUG")); v != "" {
+		cfg.Debug = parseBool(v)
+	}
+
+	if cfg.SampleRate == 0 {
+		cfg.SampleRate = 1.0
+	}
+
+	return cfg
+}
+
+// Validate verifies required fields are present when Langfuse is enabled.
+func (c Config) Validate() error {
+	if !c.Enabled {
+		return nil
+	}
+	if strings.TrimSpace(c.Host) == "" {
+		return fmt.Errorf("langfuse: host is required when enabled")
+	}
+	if c.PublicKey == "" || c.SecretKey == "" {
+		return fmt.Errorf("langfuse: public_key and secret_key are required when enabled")
+	}
+	return nil
+}
+
+func firstNonEmpty(values ...string) string {
+	for _, v := range values {
+		if s := strings.TrimSpace(v); s != "" {
+			return s
+		}
+	}
+	return ""
+}
+
+func parseBool(v string) bool {
+	switch strings.ToLower(strings.TrimSpace(v)) {
+	case "1", "true", "t", "yes", "y", "on":
+		return true
+	}
+	return false
+}

internal/tracing/langfuse/config_test.go

@@ -0,0 +1,84 @@
+package langfuse
+
+import (
+	"testing"
+	"time"
+)
+
+func TestLoadConfigFromEnv_AutoEnablesWithCredentials(t *testing.T) {
+	t.Setenv("LANGFUSE_PUBLIC_KEY", "pk-test")
+	t.Setenv("LANGFUSE_SECRET_KEY", "sk-test")
+	t.Setenv("LANGFUSE_HOST", "https://example.langfuse.com")
+
+	cfg := LoadConfigFromEnv()
+
+	if !cfg.Enabled {
+		t.Fatalf("expected Enabled=true when both keys are set, got false")
+	}
+	if cfg.Host != "https://example.langfuse.com" {
+		t.Errorf("unexpected host: %q", cfg.Host)
+	}
+	if cfg.SampleRate != 1.0 {
+		t.Errorf("expected default SampleRate=1.0, got %v", cfg.SampleRate)
+	}
+}
+
+func TestLoadConfigFromEnv_DisabledWithoutKeys(t *testing.T) {
+	t.Setenv("LANGFUSE_PUBLIC_KEY", "")
+	t.Setenv("LANGFUSE_SECRET_KEY", "")
+	t.Setenv("LANGFUSE_ENABLED", "")
+
+	cfg := LoadConfigFromEnv()
+	if cfg.Enabled {
+		t.Fatalf("expected Enabled=false when no keys set")
+	}
+}
+
+func TestLoadConfigFromEnv_ExplicitDisableOverridesKeys(t *testing.T) {
+	t.Setenv("LANGFUSE_PUBLIC_KEY", "pk")
+	t.Setenv("LANGFUSE_SECRET_KEY", "sk")
+	t.Setenv("LANGFUSE_ENABLED", "false")
+
+	cfg := LoadConfigFromEnv()
+	if cfg.Enabled {
+		t.Fatalf("expected Enabled=false when LANGFUSE_ENABLED=false")
+	}
+}
+
+func TestLoadConfigFromEnv_FlushIntervalAcceptsSecondsAndDuration(t *testing.T) {
+	t.Setenv("LANGFUSE_PUBLIC_KEY", "pk")
+	t.Setenv("LANGFUSE_SECRET_KEY", "sk")
+
+	t.Setenv("LANGFUSE_FLUSH_INTERVAL", "500ms")
+	cfg := LoadConfigFromEnv()
+	if cfg.FlushInterval != 500*time.Millisecond {
+		t.Errorf("expected 500ms, got %v", cfg.FlushInterval)
+	}
+
+	t.Setenv("LANGFUSE_FLUSH_INTERVAL", "7")
+	cfg = LoadConfigFromEnv()
+	if cfg.FlushInterval != 7*time.Second {
+		t.Errorf("expected 7s (bare integer), got %v", cfg.FlushInterval)
+	}
+}
+
+func TestConfigValidate(t *testing.T) {
+	cases := []struct {
+		name    string
+		cfg     Config
+		wantErr bool
+	}{
+		{"disabled is always valid", Config{Enabled: false}, false},
+		{"enabled without host fails", Config{Enabled: true, PublicKey: "pk", SecretKey: "sk"}, true},
+		{"enabled without keys fails", Config{Enabled: true, Host: "https://x"}, true},
+		{"enabled with all fields passes", Config{Enabled: true, Host: "https://x", PublicKey: "pk", SecretKey: "sk"}, false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.cfg.Validate()
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("err=%v wantErr=%v", err, tc.wantErr)
+			}
+		})
+	}
+}

internal/tracing/langfuse/context.go

@@ -0,0 +1,85 @@
+package langfuse
+
+import (
+	"context"
+
+	"github.com/Tencent/WeKnora/internal/types"
+	"go.opentelemetry.io/otel/propagation"
+	oteltrace "go.opentelemetry.io/otel/trace"
+)
+
+// traceCtxKey is the exported context key defined in types/const.go. It lives
+// there (not inside this package) so that logger.CloneContext — which rebuilds
+// a stripped-down context on every request — can preserve the Langfuse trace
+// without importing this package. If we kept the key private here, every
+// CloneContext call would drop the trace and downstream LLM wrappers would
+// each auto-create their own shallow trace, fragmenting a single HTTP request
+// into many unrelated traces in the Langfuse UI.
+//
+// Span parenting is NOT carried by this key: it flows through the standard
+// OpenTelemetry context (trace.SpanFromContext), which tracer.Start wires up
+// automatically. The *Trace on this key is only for handlers/middleware that
+// need to set trace-level input/output.
+var traceCtxKey = types.LangfuseTraceContextKey
+
+// withTrace stores a *Trace on the context so downstream LLM wrappers can
+// attach their generations to it.
+func withTrace(ctx context.Context, t *Trace) context.Context {
+	if t == nil || ctx == nil {
+		return ctx
+	}
+	return context.WithValue(ctx, traceCtxKey, t)
+}
+
+// traceFromCtx retrieves the active trace, if any.
+func traceFromCtx(ctx context.Context) (*Trace, bool) {
+	if ctx == nil {
+		return nil, false
+	}
+	t, ok := ctx.Value(traceCtxKey).(*Trace)
+	return t, ok && t != nil
+}
+
+// TraceFromContext is the public accessor used by HTTP middlewares and
+// handlers that want to set the trace input/output on the active trace.
+func TraceFromContext(ctx context.Context) (*Trace, bool) {
+	return traceFromCtx(ctx)
+}
+
+// TraceparentFromContext returns the W3C traceparent for the active span, or
+// empty when Langfuse is disabled or ctx carries no span. Used to stamp the
+// originating chat trace onto derived work (follow-up suggestions) that may
+// run after the HTTP handler returns, or on a later request.
+func TraceparentFromContext(ctx context.Context) string {
+	mgr := GetManager()
+	if ctx == nil || !mgr.Enabled() {
+		return ""
+	}
+	c := propagation.MapCarrier{}
+	propagator.Inject(ctx, c)
+	return c["traceparent"]
+}
+
+// AttachTraceparent resumes the originating trace from a W3C traceparent
+// previously captured by TraceparentFromContext / InjectTracing. When ctx
+// already has a *Trace (same-request background work), it is left unchanged
+// so we do not replace a live local parent with a remote one.
+//
+// This is the in-process counterpart of AsynqMiddleware's extract path: a
+// later HTTP request (e.g. POST .../suggestions) has no GinMiddleware trace,
+// and without this StartGeneration would auto-create an orphan root named
+// after the LLM call.
+func AttachTraceparent(ctx context.Context, traceparent string) context.Context {
+	mgr := GetManager()
+	if ctx == nil || traceparent == "" || !mgr.Enabled() {
+		return ctx
+	}
+	if _, ok := traceFromCtx(ctx); ok {
+		return ctx
+	}
+	ctx = propagator.Extract(ctx, propagation.MapCarrier{"traceparent": traceparent})
+	if sc := oteltrace.SpanContextFromContext(ctx); sc.IsValid() {
+		ctx = withTrace(ctx, &Trace{ID: sc.TraceID().String(), manager: mgr})
+	}
+	return ctx
+}

internal/tracing/langfuse/events.go

@@ -0,0 +1,63 @@
+package langfuse
+
+import (
+	"time"
+)
+
+// TokenUsage captures the input/output/total token counts reported by the
+// underlying model, in Langfuse's canonical schema. Set as the value of the
+// langfuse.observation.usage_details span attribute (JSON-serialized).
+type TokenUsage struct {
+	Input      int    `json:"input,omitempty"`
+	Output     int    `json:"output,omitempty"`
+	Total      int    `json:"total,omitempty"`
+	CacheRead  int    `json:"cache_read_input_tokens,omitempty"`
+	CacheWrite int    `json:"cache_creation_input_tokens,omitempty"`
+	CacheMiss  int    `json:"cache_miss_input_tokens,omitempty"`
+	Unit       string `json:"unit,omitempty"`
+}
+
+// Langfuse / OpenTelemetry semantic-convention attribute keys mirrored from
+// the official langfuse-python v4 SDK (_client/attributes.py). These are set
+// as span attributes on the OTLP wire; LiteFuse (and Langfuse v3+) index
+// traces/generations off them.
+const (
+	attrObsType            = "langfuse.observation.type"
+	attrObsInput           = "langfuse.observation.input"
+	attrObsOutput          = "langfuse.observation.output"
+	attrObsMetadata        = "langfuse.observation.metadata"
+	attrObsModel           = "langfuse.observation.model.name"
+	attrObsModelParams     = "langfuse.observation.model.parameters"
+	attrObsUsageDetails    = "langfuse.observation.usage_details"
+	attrObsCompletionStart = "langfuse.observation.completion_start_time"
+	attrTraceName          = "langfuse.trace.name"
+	attrTraceInput         = "langfuse.trace.input"
+	attrTraceOutput        = "langfuse.trace.output"
+	attrTraceMetadata      = "langfuse.trace.metadata"
+	attrTraceTags          = "langfuse.trace.tags"
+	attrUserID             = "user.id"
+	attrSessionID          = "session.id"
+	attrEnvironment        = "langfuse.environment"
+	attrRelease            = "langfuse.release"
+	attrLangfusePubKey     = "langfuse.public.key"
+
+	// The LiteFuse/Langfuse v3 OTel gate keys the "events_full"
+	// direct-write path on the instrumentation scope name. langfuse-python
+	// v4 uses "langfuse-sdk"; LiteFuse's getSdkInfoFromResourceSpans only
+	// requires the scope name to contain "langfuse" for server-side SDK
+	// classification. The actual gate is the x-langfuse-ingestion-version:4
+	// HTTP header (see exporter.go).
+	langfuseScopeName    = "langfuse-sdk"
+	langfuseScopeVersion = "4.0.0"
+)
+
+// Observation types carried by the langfuse.observation.type attribute.
+const (
+	obsTypeTrace      = "trace"
+	obsTypeSpan       = "span"
+	obsTypeGeneration = "generation"
+)
+
+func isoTime(t time.Time) string {
+	return t.UTC().Format("2006-01-02T15:04:05.000Z")
+}

internal/tracing/langfuse/exporter.go

@@ -0,0 +1,41 @@
+package langfuse
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"strings"
+
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+// newExporter builds an OTLP/HTTP trace exporter pointed at the
+// Langfuse v3+ / LiteFuse OTel endpoint (POST /api/public/otel/v1/traces).
+//
+// Auth is HTTP Basic (public_key:secret_key). The x-langfuse-ingestion-version
+// header is the gate opt-in that LiteFuse/Langfuse v3 require for the OTel
+// direct-write path (verified against directWriteHelpers.ts — without it the
+// server returns 400 "requires Python SDK >= 4.0.0"). The x-langfuse-sdk-*
+// markers are sent for parity; WeKnora is a Go client, not the Python SDK.
+func newExporter(ctx context.Context, cfg Config) (sdktrace.SpanExporter, error) {
+	endpoint := strings.TrimRight(cfg.Host, "/") + "/api/public/otel/v1/traces"
+	creds := cfg.PublicKey + ":" + cfg.SecretKey
+	opts := []otlptracehttp.Option{
+		otlptracehttp.WithEndpointURL(endpoint),
+		otlptracehttp.WithHeaders(map[string]string{
+			"Authorization":                "Basic " + base64.StdEncoding.EncodeToString([]byte(creds)),
+			"x-langfuse-ingestion-version": "4",
+			"x-langfuse-sdk-name":          "python",
+			"x-langfuse-sdk-version":       langfuseScopeVersion,
+		}),
+	}
+	if cfg.RequestTimeout > 0 {
+		opts = append(opts, otlptracehttp.WithTimeout(cfg.RequestTimeout))
+	}
+	exp, err := otlptracehttp.New(ctx, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("langfuse: build otlp exporter: %w", err)
+	}
+	return exp, nil
+}

internal/tracing/langfuse/manager.go

@@ -0,0 +1,146 @@
+package langfuse
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+
+	"github.com/Tencent/WeKnora/internal/logger"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/trace"
+	"go.opentelemetry.io/otel/trace/noop"
+)
+
+// propagator is the W3C TraceContext propagator used to extract/inject
+// traceparent across process boundaries (HTTP requests from sop3, asynq
+// payloads). It is a package-level value rather than the global OTel
+// propagator so tests remain isolated and Init never mutates global state.
+var propagator = propagation.TraceContext{}
+
+// Manager is the public façade of the langfuse package. A singleton is
+// installed via Init(); callers should treat a nil *Manager as "disabled"
+// and still invoke methods — every public method tolerates a nil receiver.
+//
+// Internally the manager owns an OpenTelemetry TracerProvider backed by an
+// OTLP/HTTP exporter pointing at the Langfuse v3+ / LiteFuse OTel endpoint.
+// The handles (*Trace / *Span / *Generation) wrap OTel spans; spans are
+// buffered by the BatchSpanProcessor and exported complete on End, so there
+// is no per-flush-batch duplication of root spans (the bug the legacy
+// hand-rolled translator had on long traces spanning multiple flushes).
+type Manager struct {
+	cfg Config
+
+	tp     *sdktrace.TracerProvider
+	tracer trace.Tracer
+
+	closed atomic.Bool
+}
+
+var (
+	globalMu sync.RWMutex
+	global   *Manager
+)
+
+// Init builds a Manager from cfg and installs it as the package-wide
+// singleton. When cfg.Enabled is false this returns a disabled manager that
+// behaves as a no-op for every public method.
+func Init(cfg Config) (*Manager, error) {
+	if err := cfg.Validate(); err != nil {
+		return nil, err
+	}
+	m := &Manager{cfg: cfg}
+	if cfg.Enabled {
+		resAttrs := []attribute.KeyValue{
+			attribute.String("service.name", "weknora"),
+			attribute.String(attrLangfusePubKey, cfg.PublicKey),
+		}
+		if cfg.Environment != "" {
+			resAttrs = append(resAttrs, attribute.String(attrEnvironment, cfg.Environment))
+		}
+		if cfg.Release != "" {
+			resAttrs = append(resAttrs, attribute.String(attrRelease, cfg.Release))
+		}
+		res, err := resource.New(context.Background(), resource.WithAttributes(resAttrs...))
+		if err != nil {
+			return nil, err
+		}
+		var sp sdktrace.SpanProcessor
+		if cfg.testExporter != nil {
+			// Test mode: synchronous export on span End (deterministic).
+			sp = sdktrace.NewSimpleSpanProcessor(cfg.testExporter)
+		} else {
+			exp, err := newExporter(context.Background(), cfg)
+			if err != nil {
+				return nil, err
+			}
+			sp = sdktrace.NewBatchSpanProcessor(exp,
+				sdktrace.WithBatchTimeout(cfg.FlushInterval),
+				sdktrace.WithMaxExportBatchSize(cfg.FlushAt),
+				sdktrace.WithMaxQueueSize(cfg.QueueSize),
+			)
+		}
+		m.tp = sdktrace.NewTracerProvider(
+			sdktrace.WithResource(res),
+			sdktrace.WithSpanProcessor(sp),
+			sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRate))),
+		)
+		m.tracer = m.tp.Tracer(langfuseScopeName,
+			trace.WithInstrumentationVersion(langfuseScopeVersion),
+			trace.WithInstrumentationAttributes(attribute.String("public_key", cfg.PublicKey)),
+		)
+		// Extraction/injection in this package use the package-level
+		// `propagator` directly, so we deliberately do NOT call
+		// otel.SetTextMapPropagator here — mutating global OTel state could
+		// interfere with any other OTel instrumentation in the process.
+	}
+
+	globalMu.Lock()
+	global = m
+	globalMu.Unlock()
+
+	if cfg.Enabled {
+		logger.Infof(context.Background(),
+			"[Langfuse] enabled host=%s flush_at=%d flush_interval=%s sample_rate=%.2f (OTLP/OTel SDK)",
+			cfg.Host, cfg.FlushAt, cfg.FlushInterval, cfg.SampleRate,
+		)
+	}
+	return m, nil
+}
+
+// GetManager returns the installed singleton, or nil if Init has not been
+// called. Callers must tolerate a nil return.
+func GetManager() *Manager {
+	globalMu.RLock()
+	defer globalMu.RUnlock()
+	return global
+}
+
+// Enabled reports whether the manager would actually emit spans.
+func (m *Manager) Enabled() bool {
+	return m != nil && m.cfg.Enabled && !m.closed.Load() && m.tp != nil
+}
+
+// Tracer exposes the OTel tracer so middleware can create spans directly
+// when needed (e.g. extracting a remote traceparent). Returns a no-op tracer
+// when disabled.
+func (m *Manager) Tracer() trace.Tracer {
+	if !m.Enabled() {
+		return noop.NewTracerProvider().Tracer(langfuseScopeName)
+	}
+	return m.tracer
+}
+
+// Shutdown flushes pending spans and releases the exporter. Safe to call
+// multiple times.
+func (m *Manager) Shutdown(ctx context.Context) error {
+	if m == nil || !m.cfg.Enabled || m.tp == nil {
+		return nil
+	}
+	if !m.closed.CompareAndSwap(false, true) {
+		return nil
+	}
+	return m.tp.Shutdown(ctx)
+}

internal/tracing/langfuse/manager_test.go

@@ -0,0 +1,28 @@
+package langfuse
+
+import (
+	"context"
+	"testing"
+)
+
+// TestManager_DisabledIsNoop verifies that when the manager is disabled the
+// public API is safe to call and produces no side effects (no spans exported,
+// no panics).
+func TestManager_DisabledIsNoop(t *testing.T) {
+	m, err := Init(Config{Enabled: false})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if m.Enabled() {
+		t.Fatal("expected disabled")
+	}
+	ctx, trace := m.StartTrace(context.Background(), TraceOptions{Name: "x"})
+	trace.Finish(nil, nil)
+
+	_, gen := m.StartGeneration(ctx, GenerationOptions{Name: "g", Model: "m"})
+	gen.Finish(nil, nil, nil)
+
+	if err := m.Shutdown(context.Background()); err != nil {
+		t.Fatalf("shutdown: %v", err)
+	}
+}

internal/tracing/langfuse/memory_obs.go

@@ -0,0 +1,71 @@
+package langfuse
+
+import (
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// SummarizeMemoryRecallOutput builds Langfuse output for a memory.recall span.
+func SummarizeMemoryRecallOutput(meta map[string]interface{}, items []*types.MemoryItem) map[string]interface{} {
+	out := make(map[string]interface{}, len(meta)+3)
+	for k, v := range meta {
+		out[k] = v
+	}
+	recalled, truncated := summarizeMemoryItems(items, defaultHitPreviewLimit)
+	out["recalled_items"] = recalled
+	if truncated > 0 {
+		out["recalled_items_truncated"] = truncated
+	}
+	return out
+}
+
+func summarizeMemoryItems(items []*types.MemoryItem, limit int) ([]map[string]interface{}, int) {
+	if limit <= 0 {
+		limit = defaultHitPreviewLimit
+	}
+	if len(items) == 0 {
+		return nil, 0
+	}
+	n := len(items)
+	truncated := 0
+	if n > limit {
+		truncated = n - limit
+		n = limit
+	}
+	out := make([]map[string]interface{}, 0, n)
+	for i := 0; i < n; i++ {
+		item := items[i]
+		if item == nil {
+			continue
+		}
+		row := map[string]interface{}{
+			"id":         item.ID,
+			"kind":       item.Kind,
+			"topic":      TruncateRunes(item.Topic, 80),
+			"importance": item.Importance,
+			"preview":    TruncateRunes(item.Content, 160),
+		}
+		out = append(out, row)
+	}
+	return out, truncated
+}
+
+// SummarizeRetrievalContextOutput builds Langfuse output for retrieval conditioning.
+func SummarizeRetrievalContextOutput(
+	background string,
+	interests, documents []string,
+	items []*types.MemoryItem,
+) map[string]interface{} {
+	conditioned, _ := summarizeMemoryItems(items, defaultHitPreviewLimit)
+	out := map[string]interface{}{
+		"background":        TruncateRunes(background, 240),
+		"interest_count":    len(interests),
+		"document_count":    len(documents),
+		"interests":         interests,
+		"documents":         documents,
+		"conditioned_items": conditioned,
+	}
+	if out["background"] == "" {
+		delete(out, "background")
+	}
+	return out
+}

internal/tracing/langfuse/memory_obs_test.go

@@ -0,0 +1,24 @@
+package langfuse
+
+import (
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestSummarizeMemoryRecallOutput_includesMetaAndItems(t *testing.T) {
+	out := SummarizeMemoryRecallOutput(map[string]interface{}{
+		"matched": 1,
+		"mode":    "lexical_only",
+	}, []*types.MemoryItem{
+		{ID: "m1", Kind: types.MemoryKindFact, Topic: "数据库", Content: "生产用 PostgreSQL 17"},
+	})
+
+	if out["matched"] != 1 || out["mode"] != "lexical_only" {
+		t.Fatalf("meta not preserved: %#v", out)
+	}
+	items := out["recalled_items"].([]map[string]interface{})
+	if len(items) != 1 || items[0]["id"] != "m1" {
+		t.Fatalf("unexpected recalled_items: %#v", items)
+	}
+}

internal/tracing/langfuse/middleware.go

@@ -0,0 +1,159 @@
+package langfuse
+
+import (
+	"context"
+	"strconv"
+	"strings"
+
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/gin-gonic/gin"
+	"go.opentelemetry.io/otel/propagation"
+)
+
+// GinMiddleware returns a Gin handler that opens a Langfuse trace for each
+// incoming request that hits a traced path. The trace is auto-finished when
+// the handler chain returns; individual LLM calls inside the handler attach
+// their generations to this trace via the request context.
+//
+// Only paths matching shouldTrace are traced — static assets, health checks
+// and polling endpoints are noisy and uninteresting.
+func GinMiddleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		mgr := GetManager()
+		if !mgr.Enabled() || !shouldTrace(c) {
+			c.Next()
+			return
+		}
+
+		ctx := c.Request.Context()
+		// Extract a W3C traceparent from the incoming request so the WeKnora
+		// trace inherits the upstream caller's trace id. This is what lets a
+		// sop3 pipeline run (identified by its W3C trace_id) and the WeKnora
+		// agent-chat call it triggers land under the same trace in LiteFuse.
+		// When no traceparent is present (human UI calls, other clients) the
+		// root span starts a fresh trace as before.
+		ctx = propagator.Extract(ctx, propagation.HeaderCarrier(c.Request.Header))
+		userID := extractUserID(ctx)
+		sessionID := extractSessionID(c)
+
+		opts := TraceOptions{
+			Name:      c.Request.Method + " " + c.FullPath(),
+			UserID:    userID,
+			SessionID: sessionID,
+			Metadata: map[string]interface{}{
+				"http.method": c.Request.Method,
+				"http.path":   c.FullPath(),
+				"http.query":  c.Request.URL.RawQuery,
+			},
+			Tags: []string{"http", strings.ToLower(c.Request.Method)},
+		}
+		if rid, ok := types.RequestIDFromContext(ctx); ok {
+			opts.Metadata["request_id"] = rid
+		}
+
+		newCtx, trace := mgr.StartTrace(ctx, opts)
+		c.Request = c.Request.WithContext(newCtx)
+
+		c.Next()
+
+		trace.Finish(map[string]interface{}{
+			"status":        c.Writer.Status(),
+			"response.size": c.Writer.Size(),
+		}, nil)
+	}
+}
+
+// shouldTrace restricts tracing to endpoints where LLM work (or the asynq
+// jobs that will run LLM work) originates. Everything else — auth, list,
+// config fetches, static assets, health checks — is skipped to keep the
+// Langfuse dashboard's signal-to-noise ratio high.
+//
+// The list below is grouped by purpose:
+//   - online inference: knowledge-chat / agent-chat / knowledge-search /
+//     generate_title are the existing chat/retrieval surface.
+//   - ingestion: POST on knowledge-bases/:id/knowledge (file / url / manual)
+//     and reparse / move / copy all kick off asynq jobs that later run
+//     embedding/VLM/chat calls. Tracing the HTTP side means the Langfuse UI
+//     shows a parent trace whose children are the worker spans.
+//   - batch ops: FAQ import + knowledge batch delete also enqueue jobs.
+//   - model/setup diagnostics: initialization endpoints exercise live
+//     models; evaluation runs arbitrary chat pipelines.
+//   - wiki: auto-fix kicks off wiki ingest, which calls embedding.
+//
+// Read-only listing / GET endpoints are deliberately excluded; they never
+// trigger LLM work and would only add noise.
+func shouldTrace(c *gin.Context) bool {
+	path := c.FullPath()
+	if path == "" {
+		return false
+	}
+	method := c.Request.Method
+	// Online inference
+	switch {
+	case strings.HasPrefix(path, "/api/v1/knowledge-chat"),
+		strings.HasPrefix(path, "/api/v1/agent-chat"),
+		strings.HasPrefix(path, "/api/v1/knowledge-search"),
+		strings.HasPrefix(path, "/api/v1/sessions") && strings.Contains(path, "generate_title"),
+		strings.HasPrefix(path, "/api/v1/initialization/remote/check"),
+		strings.HasPrefix(path, "/api/v1/initialization/embedding/test"),
+		strings.HasPrefix(path, "/api/v1/initialization/rerank/check"),
+		strings.HasPrefix(path, "/api/v1/initialization/asr/check"),
+		strings.HasPrefix(path, "/api/v1/initialization/multimodal/test"),
+		strings.HasPrefix(path, "/api/v1/initialization/extract/"),
+		strings.HasPrefix(path, "/api/v1/evaluation"):
+		return true
+	}
+	// Ingestion (all POST/PUT that enqueue LLM-backed async work)
+	if method == "POST" || method == "PUT" {
+		switch {
+		// Per-knowledge-base ingestion surface.
+		case strings.Contains(path, "/knowledge-bases/") && strings.Contains(path, "/knowledge/"):
+			return true
+		// Knowledge-level mutations that trigger re-processing.
+		case strings.HasPrefix(path, "/api/v1/knowledge/") &&
+			(strings.HasSuffix(path, "/reparse") ||
+				strings.HasSuffix(path, "/move") ||
+				strings.Contains(path, "/manual/")):
+			return true
+		// Knowledge base copy (clones an entire KB, fanning out documents).
+		case path == "/api/v1/knowledge-bases/copy":
+			return true
+		// FAQ bulk import.
+		case strings.Contains(path, "/faq/entries") ||
+			strings.Contains(path, "/faq/entry") ||
+			strings.Contains(path, "/faq/import"):
+			return true
+		// Wiki auto-fix enqueues wiki ingest.
+		case strings.Contains(path, "/wiki/auto-fix") ||
+			strings.Contains(path, "/wiki/rebuild-links"):
+			return true
+		// Chunk-level mutations that rerun embeddings on update.
+		case strings.HasPrefix(path, "/api/v1/chunks/") && method == "PUT":
+			return true
+		// Manual data source sync triggers asynq sync.
+		case strings.Contains(path, "/datasource/") && strings.HasSuffix(path, "/sync"):
+			return true
+		}
+	}
+	return false
+}
+
+func extractUserID(ctx context.Context) string {
+	if v, ok := ctx.Value(types.UserIDContextKey).(string); ok && v != "" {
+		return v
+	}
+	if v, ok := ctx.Value(types.TenantIDContextKey).(uint64); ok && v != 0 {
+		return "tenant:" + strconv.FormatUint(v, 10)
+	}
+	return ""
+}
+
+func extractSessionID(c *gin.Context) string {
+	if v := c.Param("session_id"); v != "" {
+		return v
+	}
+	if v := c.Param("id"); v != "" && strings.Contains(c.FullPath(), "/sessions/") {
+		return v
+	}
+	return ""
+}

internal/tracing/langfuse/retrieval_obs.go

@@ -0,0 +1,204 @@
+package langfuse
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+const defaultHitPreviewLimit = 25
+
+// TruncateRunes shortens s to at most maxRunes runes for trace payloads.
+func TruncateRunes(s string, maxRunes int) string {
+	if maxRunes <= 0 {
+		return ""
+	}
+	r := []rune(s)
+	if len(r) <= maxRunes {
+		return s
+	}
+	return string(r[:maxRunes]) + "..."
+}
+
+// SummarizeRetrieveOutput builds Langfuse output for the retrieve span.
+func SummarizeRetrieveOutput(results []*types.RetrieveResult) map[string]interface{} {
+	out := map[string]interface{}{
+		"total_hits":   0,
+		"vector_hits":  0,
+		"keyword_hits": 0,
+		"group_count":  len(results),
+		"by_retriever": []map[string]interface{}{},
+		"top_hits":     []map[string]interface{}{},
+	}
+	if len(results) == 0 {
+		return out
+	}
+
+	byRetriever := make([]map[string]interface{}, 0, len(results))
+	var all []*types.IndexWithScore
+	for _, rr := range results {
+		if rr == nil {
+			continue
+		}
+		count := len(rr.Results)
+		out["total_hits"] = out["total_hits"].(int) + count
+		if rr.RetrieverType == types.VectorRetrieverType {
+			out["vector_hits"] = out["vector_hits"].(int) + count
+		} else {
+			out["keyword_hits"] = out["keyword_hits"].(int) + count
+		}
+		byRetriever = append(byRetriever, map[string]interface{}{
+			"engine":    string(rr.RetrieverEngineType),
+			"retriever": string(rr.RetrieverType),
+			"count":     count,
+		})
+		all = append(all, rr.Results...)
+	}
+	out["by_retriever"] = byRetriever
+	out["top_hits"] = summarizeIndexHits(all, defaultHitPreviewLimit)
+	return out
+}
+
+// SummarizeSearchResults builds a compact ranked preview for rerank spans.
+func SummarizeSearchResults(results []*types.SearchResult, limit int) map[string]interface{} {
+	if limit <= 0 {
+		limit = defaultHitPreviewLimit
+	}
+	out := map[string]interface{}{
+		"count":    len(results),
+		"top_hits": []map[string]interface{}{},
+	}
+	if len(results) == 0 {
+		return out
+	}
+
+	sorted := append([]*types.SearchResult(nil), results...)
+	sort.Slice(sorted, func(i, j int) bool {
+		if sorted[i].Score != sorted[j].Score {
+			return sorted[i].Score > sorted[j].Score
+		}
+		return sorted[i].ID < sorted[j].ID
+	})
+
+	hits := make([]map[string]interface{}, 0, minInt(limit, len(sorted)))
+	for i, sr := range sorted {
+		if i >= limit {
+			break
+		}
+		item := map[string]interface{}{
+			"rank":            i + 1,
+			"chunk_id":        sr.ID,
+			"knowledge_id":    sr.KnowledgeID,
+			"knowledge_title": sr.KnowledgeTitle,
+			"composite_score": fmt.Sprintf("%.4f", sr.Score),
+			"match_type":      sr.MatchType,
+			"chunk_type":      sr.ChunkType,
+			"preview":         TruncateRunes(sr.Content, 160),
+		}
+		if sr.Metadata != nil {
+			if base, ok := sr.Metadata["base_score"]; ok {
+				item["retrieval_score"] = base
+			}
+			if model, ok := sr.Metadata["model_score"]; ok {
+				item["model_score"] = model
+			}
+			if boosted, ok := sr.Metadata["faq_boosted"]; ok {
+				item["faq_boosted"] = boosted
+			}
+			if orig, ok := sr.Metadata["faq_original_score"]; ok {
+				item["faq_original_score"] = orig
+			}
+		}
+		hits = append(hits, item)
+	}
+	out["top_hits"] = hits
+	if len(sorted) > limit {
+		out["truncated"] = len(sorted) - limit
+	}
+	return out
+}
+
+// SummarizeRankScores builds rerank model score rows for Langfuse output.
+func SummarizeRankScores(
+	results []map[string]interface{},
+	limit int,
+) []map[string]interface{} {
+	if limit <= 0 {
+		limit = defaultHitPreviewLimit
+	}
+	if len(results) <= limit {
+		return results
+	}
+	out := make([]map[string]interface{}, limit)
+	copy(out, results[:limit])
+	return out
+}
+
+func summarizeIndexHits(hits []*types.IndexWithScore, limit int) []map[string]interface{} {
+	if len(hits) == 0 {
+		return nil
+	}
+	sorted := append([]*types.IndexWithScore(nil), hits...)
+	sort.Slice(sorted, func(i, j int) bool {
+		if sorted[i].Score != sorted[j].Score {
+			return sorted[i].Score > sorted[j].Score
+		}
+		return sorted[i].ChunkID < sorted[j].ChunkID
+	})
+
+	n := minInt(limit, len(sorted))
+	out := make([]map[string]interface{}, 0, n)
+	for i := 0; i < n; i++ {
+		hit := sorted[i]
+		out = append(out, map[string]interface{}{
+			"rank":              i + 1,
+			"chunk_id":          hit.ChunkID,
+			"knowledge_id":      hit.KnowledgeID,
+			"knowledge_base_id": hit.KnowledgeBaseID,
+			"score":             fmt.Sprintf("%.4f", hit.Score),
+			"match_type":        hit.MatchType,
+			"preview":           TruncateRunes(hit.Content, 160),
+		})
+	}
+	return out
+}
+
+// SummarizePassagePreviews builds rerank passage previews aligned with candidates.
+func SummarizePassagePreviews(
+	candidates []*types.SearchResult,
+	passages []string,
+	limit int,
+) []map[string]interface{} {
+	if limit <= 0 {
+		limit = defaultHitPreviewLimit
+	}
+	n := len(candidates)
+	if len(passages) < n {
+		n = len(passages)
+	}
+	if limit < n {
+		n = limit
+	}
+	out := make([]map[string]interface{}, 0, n)
+	for i := 0; i < n; i++ {
+		sr := candidates[i]
+		out = append(out, map[string]interface{}{
+			"index":           i,
+			"chunk_id":        sr.ID,
+			"knowledge_id":    sr.KnowledgeID,
+			"knowledge_title": sr.KnowledgeTitle,
+			"retrieval_score": fmt.Sprintf("%.4f", sr.Score),
+			"match_type":      sr.MatchType,
+			"preview":         TruncateRunes(passages[i], 160),
+		})
+	}
+	return out
+}
+
+func minInt(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}

internal/tracing/langfuse/retrieval_obs_test.go

@@ -0,0 +1,49 @@
+package langfuse
+
+import (
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestSummarizeRetrieveOutput_countsAndPreviews(t *testing.T) {
+	out := SummarizeRetrieveOutput([]*types.RetrieveResult{
+		{
+			RetrieverEngineType: types.PostgresRetrieverEngineType,
+			RetrieverType:       types.VectorRetrieverType,
+			Results: []*types.IndexWithScore{
+				{ChunkID: "c1", KnowledgeID: "k1", Score: 0.9, Content: "alpha"},
+				{ChunkID: "c2", KnowledgeID: "k1", Score: 0.5, Content: "beta"},
+			},
+		},
+		{
+			RetrieverEngineType: types.PostgresRetrieverEngineType,
+			RetrieverType:       types.KeywordsRetrieverType,
+			Results: []*types.IndexWithScore{
+				{ChunkID: "c3", KnowledgeID: "k2", Score: 0.7, Content: "gamma"},
+			},
+		},
+	})
+
+	if out["total_hits"] != 3 {
+		t.Fatalf("total_hits = %v, want 3", out["total_hits"])
+	}
+	if out["vector_hits"] != 2 || out["keyword_hits"] != 1 {
+		t.Fatalf("vector/keyword hits = %v/%v", out["vector_hits"], out["keyword_hits"])
+	}
+	hits := out["top_hits"].([]map[string]interface{})
+	if len(hits) != 3 || hits[0]["chunk_id"] != "c1" {
+		t.Fatalf("unexpected top_hits: %#v", hits)
+	}
+}
+
+func TestSummarizeSearchResults_sortsByScore(t *testing.T) {
+	out := SummarizeSearchResults([]*types.SearchResult{
+		{ID: "low", Score: 0.2, Content: "low"},
+		{ID: "high", Score: 0.9, Content: "high"},
+	}, 10)
+	hits := out["top_hits"].([]map[string]interface{})
+	if len(hits) != 2 || hits[0]["chunk_id"] != "high" {
+		t.Fatalf("unexpected hits: %#v", hits)
+	}
+}

internal/tracing/langfuse/tracer.go

@@ -0,0 +1,356 @@
+package langfuse
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/logger"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// Trace represents an active root observation. A Trace is conceptually one
+// "request" (e.g. a chat turn). Generations and spans attached to it roll up
+// as children in the Langfuse UI. It wraps an OpenTelemetry root span; its
+// ID is the OTel trace id (W3C 32-hex), which — when the request carried a
+// traceparent header — is the upstream caller's trace id (sop3 correlation).
+type Trace struct {
+	ID      string
+	span    trace.Span
+	manager *Manager
+	// metadata holds the metadata set at StartTrace so Finish can merge (not
+	// overwrite) the finish-time metadata into it before serializing.
+	metadata map[string]interface{}
+}
+
+// Generation represents a single model invocation (LLM / embedding / VLM / ASR).
+type Generation struct {
+	ID      string
+	span    trace.Span
+	manager *Manager
+	model   string
+	name    string
+	// autoTrace is a non-nil root trace this generation implicitly opened
+	// because ctx carried none; Finish must End it so the root is exported.
+	autoTrace *Trace
+}
+
+// Span represents a logical unit of work that isn't itself an LLM call — for
+// example an asynq task execution, a pipeline stage, or a document-processing
+// step. Generations and nested spans attach as children via the OTel span
+// context (parenting is automatic through trace.SpanFromContext).
+type Span struct {
+	ID      string
+	span    trace.Span
+	manager *Manager
+	name    string
+	// metadata holds the metadata set at StartSpan so Finish can merge (not
+	// overwrite) the finish-time metadata into it before serializing.
+	metadata map[string]interface{}
+	// autoTrace is a non-nil root trace this span implicitly opened because
+	// ctx carried none; Finish must End it so the root is exported.
+	autoTrace *Trace
+}
+
+// TraceOptions configures a new trace.
+type TraceOptions struct {
+	Name        string
+	UserID      string
+	SessionID   string
+	Input       interface{}
+	Metadata    map[string]interface{}
+	Tags        []string
+	Environment string
+	Release     string
+}
+
+// GenerationOptions configures a new generation observation.
+type GenerationOptions struct {
+	Name            string
+	Model           string
+	Input           interface{}
+	Metadata        map[string]interface{}
+	ModelParameters map[string]interface{}
+}
+
+// SpanOptions configures a new SPAN observation.
+type SpanOptions struct {
+	Name     string
+	Input    interface{}
+	Metadata map[string]interface{}
+}
+
+// StartTrace opens a root span. When ctx carries a remote SpanContext (from a
+// W3C traceparent extracted by GinMiddleware), the root span inherits the
+// upstream trace id — this is what makes a sop3 run and its WeKnora call land
+// under the same trace in LiteFuse. The returned *Trace is non-nil even when
+// disabled (methods are no-ops), so callers don't need nil checks.
+func (m *Manager) StartTrace(ctx context.Context, opts TraceOptions) (context.Context, *Trace) {
+	if !m.Enabled() {
+		return ctx, &Trace{manager: m}
+	}
+	name := opts.Name
+	attrs := []attribute.KeyValue{attribute.String(attrObsType, obsTypeTrace)}
+	if opts.Name != "" {
+		attrs = append(attrs, attribute.String(attrTraceName, opts.Name))
+	}
+	if opts.UserID != "" {
+		attrs = append(attrs, attribute.String(attrUserID, opts.UserID))
+	}
+	if opts.SessionID != "" {
+		attrs = append(attrs, attribute.String(attrSessionID, opts.SessionID))
+	}
+	env := opts.Environment
+	if env == "" {
+		env = m.cfg.Environment
+	}
+	if env != "" {
+		attrs = append(attrs, attribute.String(attrEnvironment, env))
+	}
+	rel := opts.Release
+	if rel == "" {
+		rel = m.cfg.Release
+	}
+	if rel != "" {
+		attrs = append(attrs, attribute.String(attrRelease, rel))
+	}
+	attrs = append(attrs, jsonAttr(attrTraceInput, opts.Input))
+	attrs = append(attrs, jsonAttr(attrTraceMetadata, opts.Metadata))
+	if len(opts.Tags) > 0 {
+		attrs = append(attrs, jsonAttr(attrTraceTags, opts.Tags))
+	}
+	ctx, span := m.tracer.Start(ctx, name, trace.WithTimestamp(time.Now()), trace.WithAttributes(attrs...))
+	t := &Trace{ID: span.SpanContext().TraceID().String(), span: span, manager: m, metadata: opts.Metadata}
+	return withTrace(ctx, t), t
+}
+
+// Finish updates the trace with its final output and merges any finish-time
+// metadata into the metadata set at StartTrace. Safe to call on a disabled
+// trace (no-op). Finish keys are merged on top of the open-time correlation
+// fields (request_id, http.method, etc.) rather than overwriting them, so
+// both the open's correlation and the finish outcome survive.
+func (t *Trace) Finish(output interface{}, metadata map[string]interface{}) {
+	if t == nil || t.manager == nil || !t.manager.Enabled() || t.span == nil {
+		return
+	}
+	attrs := []attribute.KeyValue{jsonAttr(attrTraceOutput, output)}
+	if merged := mergeMetadata(t.metadata, metadata); merged != nil {
+		attrs = append(attrs, jsonAttr(attrTraceMetadata, merged))
+	}
+	t.span.SetAttributes(attrs...)
+	t.span.End()
+}
+
+// ResumeTrace reconstructs a *Trace handle from an externally-provided W3C
+// trace id (and optional parent span id), without creating a new root span —
+// the originating process (e.g. an HTTP request that already opened a trace)
+// owns the root. Used to graft async work onto an existing trace: it sets a
+// remote SpanContext on ctx so any child span/generation started under it
+// inherits the upstream trace id. When traceID is empty the returned *Trace
+// is nil, signalling the caller should fall back to StartTrace.
+func (m *Manager) ResumeTrace(ctx context.Context, traceID, parentSpanID string) (context.Context, *Trace) {
+	if m == nil || !m.Enabled() || traceID == "" {
+		return ctx, nil
+	}
+	tid, err := trace.TraceIDFromHex(traceID)
+	if err != nil {
+		// Not a W3C 32-hex trace id (legacy UUID, etc.); cannot resume.
+		return ctx, nil
+	}
+	var sid trace.SpanID
+	if parentSpanID != "" {
+		if s, err := trace.SpanIDFromHex(parentSpanID); err == nil {
+			sid = s
+		}
+	}
+	sc := trace.NewSpanContext(trace.SpanContextConfig{
+		TraceID:    tid,
+		SpanID:     sid,
+		TraceFlags: trace.FlagsSampled,
+		Remote:     true,
+	})
+	ctx = trace.ContextWithRemoteSpanContext(ctx, sc)
+	t := &Trace{ID: traceID, manager: m}
+	return withTrace(ctx, t), t
+}
+
+// reestablishParentSpan re-injects the active trace's root span as the OTel
+// parent when ctx carries a *Trace but no active OTel span. This happens when
+// a context rebuild drops the OTel span while the *Trace handle survives on
+// the exported key (e.g. a background goroutine derived from a non-request
+// context, or a CloneContext that predates/missed the span fix). Without
+// this, child spans (e.g. a summary generation) start a fresh root and orphan
+// off the HTTP trace.
+func (m *Manager) reestablishParentSpan(ctx context.Context) context.Context {
+	if !m.Enabled() {
+		return ctx
+	}
+	if sp := trace.SpanFromContext(ctx); sp.IsRecording() {
+		return ctx // already has an active span
+	}
+	if t, ok := traceFromCtx(ctx); ok && t != nil && t.span != nil {
+		return trace.ContextWithSpan(ctx, t.span)
+	}
+	return ctx
+}
+
+// StartSpan opens a child span under the trace/span carried by ctx. When no
+// trace is present, OTel creates a fresh root (mirroring StartGeneration's
+// auto-trace behaviour). Returns a ctx whose active span is this span.
+func (m *Manager) StartSpan(ctx context.Context, opts SpanOptions) (context.Context, *Span) {
+	if !m.Enabled() {
+		return ctx, &Span{manager: m}
+	}
+	ctx = m.reestablishParentSpan(ctx)
+	var autoTrace *Trace
+	if _, ok := traceFromCtx(ctx); !ok {
+		// No active trace: open a shallow root so the span isn't orphaned.
+		// Hold the handle so Finish can End it — otherwise the root span is
+		// never exported and this span's parent points at a missing span.
+		ctx, autoTrace = m.StartTrace(ctx, TraceOptions{Name: opts.Name})
+	}
+	attrs := []attribute.KeyValue{
+		attribute.String(attrObsType, obsTypeSpan),
+		jsonAttr(attrObsInput, opts.Input),
+		jsonAttr(attrObsMetadata, opts.Metadata),
+	}
+	ctx, span := m.tracer.Start(ctx, opts.Name, trace.WithTimestamp(time.Now()), trace.WithAttributes(attrs...))
+	return ctx, &Span{
+		ID:        span.SpanContext().SpanID().String(),
+		span:      span,
+		manager:   m,
+		name:      opts.Name,
+		metadata:  opts.Metadata,
+		autoTrace: autoTrace,
+	}
+}
+
+// Finish updates a span with its final output, extra metadata and any error.
+// A non-nil err marks the span as ERROR. Finish-time metadata is merged on top
+// of the metadata set at StartSpan (finish keys win) rather than discarded, so
+// fields only known at completion (outcome, duration_ms, tool_calls, …) are
+// reported. If this span implicitly opened a root trace, that root is ended
+// last so it is exported.
+func (s *Span) Finish(output interface{}, metadata map[string]interface{}, err error) {
+	if s == nil || s.manager == nil || !s.manager.Enabled() || s.span == nil {
+		return
+	}
+	attrs := []attribute.KeyValue{jsonAttr(attrObsOutput, output)}
+	if merged := mergeMetadata(s.metadata, metadata); merged != nil {
+		attrs = append(attrs, jsonAttr(attrObsMetadata, merged))
+	}
+	s.span.SetAttributes(attrs...)
+	if err != nil {
+		s.span.RecordError(err)
+		s.span.SetStatus(codes.Error, err.Error())
+	}
+	s.span.End()
+	if s.autoTrace != nil {
+		s.autoTrace.Finish(nil, nil)
+	}
+}
+
+// StartGeneration opens a generation observation under the trace carried by
+// ctx (or a newly auto-created trace). If a parent span is present on ctx,
+// the generation attaches under it via the OTel span context.
+func (m *Manager) StartGeneration(ctx context.Context, opts GenerationOptions) (context.Context, *Generation) {
+	if !m.Enabled() {
+		return ctx, &Generation{manager: m, model: opts.Model, name: opts.Name}
+	}
+	ctx = m.reestablishParentSpan(ctx)
+	var autoTrace *Trace
+	if _, ok := traceFromCtx(ctx); !ok {
+		// No active trace: open a root so the generation isn't orphaned, and
+		// hold the handle so Finish can End it (otherwise the root span never
+		// gets exported and this generation's parent points at nothing).
+		ctx, autoTrace = m.StartTrace(ctx, TraceOptions{Name: opts.Name})
+	}
+	attrs := []attribute.KeyValue{
+		attribute.String(attrObsType, obsTypeGeneration),
+		attribute.String(attrObsModel, opts.Model),
+		jsonAttr(attrObsInput, opts.Input),
+		jsonAttr(attrObsMetadata, opts.Metadata),
+		jsonAttr(attrObsModelParams, opts.ModelParameters),
+	}
+	ctx, span := m.tracer.Start(ctx, opts.Name, trace.WithTimestamp(time.Now()), trace.WithAttributes(attrs...))
+	g := &Generation{
+		ID:        span.SpanContext().SpanID().String(),
+		span:      span,
+		manager:   m,
+		model:     opts.Model,
+		name:      opts.Name,
+		autoTrace: autoTrace,
+	}
+	return ctx, g
+}
+
+// Finish updates a generation with its final output, token usage and any
+// error. A non-nil err marks the observation as ERROR.
+func (g *Generation) Finish(output interface{}, usage *TokenUsage, err error) {
+	if g == nil || g.manager == nil || !g.manager.Enabled() || g.span == nil {
+		return
+	}
+	attrs := []attribute.KeyValue{jsonAttr(attrObsOutput, output)}
+	if usage != nil {
+		attrs = append(attrs, jsonAttr(attrObsUsageDetails, usage))
+	}
+	g.span.SetAttributes(attrs...)
+	if err != nil {
+		g.span.RecordError(err)
+		g.span.SetStatus(codes.Error, err.Error())
+	}
+	g.span.End()
+	if g.autoTrace != nil {
+		g.autoTrace.Finish(nil, nil)
+	}
+}
+
+// MarkCompletionStart records the time at which the first token was received
+// in a streaming generation. Langfuse surfaces this as time-to-first-token.
+func (g *Generation) MarkCompletionStart(t time.Time) {
+	if g == nil || g.manager == nil || !g.manager.Enabled() || g.span == nil {
+		return
+	}
+	g.span.SetAttributes(attribute.String(attrObsCompletionStart, isoTime(t)))
+}
+
+// mergeMetadata combines the metadata captured when an observation opened
+// with the metadata supplied at Finish. Finish keys win on conflict (they
+// reflect the final outcome), while open-time keys (correlation fields such
+// as request_id / http.method) are preserved. Returns nil when both inputs
+// are empty so callers can skip writing an empty attribute.
+func mergeMetadata(start, finish map[string]interface{}) map[string]interface{} {
+	if len(start) == 0 && len(finish) == 0 {
+		return nil
+	}
+	merged := make(map[string]interface{}, len(start)+len(finish))
+	for k, v := range start {
+		merged[k] = v
+	}
+	for k, v := range finish {
+		merged[k] = v
+	}
+	return merged
+}
+
+// jsonAttr serializes v to a compact JSON string and wraps it as a string
+// OTel attribute — matching how langfuse-python stores structured fields
+// (input/output/metadata/usage) on spans. nil/zero values return an empty
+// KeyValue (harmless on SetAttributes).
+func jsonAttr(key string, v interface{}) attribute.KeyValue {
+	if v == nil {
+		return attribute.KeyValue{Key: attribute.Key(key)}
+	}
+	b, err := json.Marshal(v)
+	if err != nil {
+		logger.Warnf(context.Background(), "[Langfuse] marshal attr %s failed: %v", key, err)
+		return attribute.KeyValue{Key: attribute.Key(key)}
+	}
+	if len(b) == 0 || string(b) == "null" {
+		// Optional structured fields are often unset; omit rather than warn.
+		return attribute.KeyValue{Key: attribute.Key(key)}
+	}
+	return attribute.String(key, string(b))
+}

internal/tracing/langfuse/tracer_test.go

@@ -0,0 +1,393 @@
+package langfuse
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+)
+
+// newTestManager builds a Manager wired to an in-memory span exporter via the
+// SimpleSpanProcessor (synchronous export on span End), so tests can assert on
+// exported spans deterministically without an HTTP server.
+func newTestManager(t *testing.T) (*Manager, *tracetest.InMemoryExporter) {
+	t.Helper()
+	exp := tracetest.NewInMemoryExporter()
+	m, err := Init(Config{
+		Enabled:        true,
+		Host:           "http://test",
+		PublicKey:      "pk",
+		SecretKey:      "sk",
+		FlushAt:        1,
+		FlushInterval:  1 * time.Second,
+		QueueSize:      32,
+		RequestTimeout: 2 * time.Second,
+		SampleRate:     1.0,
+		testExporter:   exp,
+	})
+	if err != nil {
+		t.Fatalf("init: %v", err)
+	}
+	t.Cleanup(func() { _ = m.Shutdown(context.Background()) })
+	return m, exp
+}
+
+// spanAttr returns the string value of a span attribute, or "" if absent.
+func spanAttr(attrs []attribute.KeyValue, key string) string {
+	for _, kv := range attrs {
+		if string(kv.Key) == key {
+			if v, ok := kv.Value.AsInterface().(string); ok {
+				return v
+			}
+		}
+	}
+	return ""
+}
+
+// spanType returns the langfuse.observation.type of a span.
+func spanType(s tracetest.SpanStub) string { return spanAttr(s.Attributes, attrObsType) }
+
+// TestSpan_NestedHierarchy verifies nested StartSpan calls produce a
+// trace → span → span → generation tree with correct OTel parent linking
+// (parenting is automatic through trace.SpanFromContext, no manual ids).
+func TestSpan_NestedHierarchy(t *testing.T) {
+	m, exp := newTestManager(t)
+
+	ctx, trace := m.StartTrace(context.Background(), TraceOptions{Name: "root"})
+	ctx, outer := m.StartSpan(ctx, SpanOptions{Name: "outer"})
+	ctx, inner := m.StartSpan(ctx, SpanOptions{Name: "inner"})
+	_, gen := m.StartGeneration(ctx, GenerationOptions{Name: "llm", Model: "m"})
+
+	gen.Finish("out", &TokenUsage{Input: 1, Output: 2, Total: 3}, nil)
+	inner.Finish("inner-out", nil, nil)
+	outer.Finish("outer-out", nil, nil)
+	trace.Finish("root-out", nil)
+
+	spans := exp.GetSpans()
+	if len(spans) != 4 {
+		t.Fatalf("expected 4 spans, got %d", len(spans))
+	}
+	byName := map[string]tracetest.SpanStub{}
+	for _, s := range spans {
+		byName[s.Name] = s
+	}
+	root, outerS, innerS, genS := byName["root"], byName["outer"], byName["inner"], byName["llm"]
+	if root.Name == "" || outerS.Name == "" || innerS.Name == "" || genS.Name == "" {
+		t.Fatalf("missing expected spans: %+v", byName)
+	}
+	// All spans share the trace id.
+	if outerS.SpanContext.TraceID() != root.SpanContext.TraceID() ||
+		innerS.SpanContext.TraceID() != root.SpanContext.TraceID() ||
+		genS.SpanContext.TraceID() != root.SpanContext.TraceID() {
+		t.Errorf("all spans must share the root trace id")
+	}
+	// Parent chain: outer → root, inner → outer, gen → inner.
+	if outerS.Parent.SpanID() != root.SpanContext.SpanID() {
+		t.Errorf("outer parent = %s, want root %s", outerS.Parent.SpanID(), root.SpanContext.SpanID())
+	}
+	if innerS.Parent.SpanID() != outerS.SpanContext.SpanID() {
+		t.Errorf("inner parent = %s, want outer %s", innerS.Parent.SpanID(), outerS.SpanContext.SpanID())
+	}
+	if genS.Parent.SpanID() != innerS.SpanContext.SpanID() {
+		t.Errorf("gen parent = %s, want inner %s", genS.Parent.SpanID(), innerS.SpanContext.SpanID())
+	}
+	// Observation types.
+	if spanType(root) != obsTypeTrace {
+		t.Errorf("root type = %q, want %q", spanType(root), obsTypeTrace)
+	}
+	if spanType(outerS) != obsTypeSpan || spanType(innerS) != obsTypeSpan {
+		t.Errorf("outer/inner type not span: %q %q", spanType(outerS), spanType(innerS))
+	}
+	if spanType(genS) != obsTypeGeneration {
+		t.Errorf("gen type = %q, want %q", spanType(genS), obsTypeGeneration)
+	}
+}
+
+// TestSpan_FinishWithError records an error status on the span so failures in
+// asynq handlers surface as red observations in Langfuse.
+func TestSpan_FinishWithError(t *testing.T) {
+	m, exp := newTestManager(t)
+
+	ctx, _ := m.StartTrace(context.Background(), TraceOptions{Name: "root"})
+	_, span := m.StartSpan(ctx, SpanOptions{Name: "boom"})
+	span.Finish(nil, nil, errors.New("kaboom"))
+
+	for _, s := range exp.GetSpans() {
+		if s.Name != "boom" {
+			continue
+		}
+		if s.Status.Code != codes.Error {
+			t.Errorf("span status = %v, want Error", s.Status.Code)
+		}
+		if s.Status.Description != "kaboom" {
+			t.Errorf("status description = %q, want kaboom", s.Status.Description)
+		}
+		return
+	}
+	t.Fatal("boom span not exported")
+}
+
+// TestManager_FullRoundTrip asserts a generation carries the model name and
+// usage_details attribute (JSON with token counts), and shares the trace id
+// of the root span.
+func TestManager_FullRoundTrip(t *testing.T) {
+	m, exp := newTestManager(t)
+
+	ctx, trace := m.StartTrace(context.Background(), TraceOptions{Name: "test.trace", UserID: "user-42"})
+	_, gen := m.StartGeneration(ctx, GenerationOptions{
+		Name:  "chat.completion",
+		Model: "gpt-test",
+		Input: []map[string]string{{"role": "user", "content": "hi"}},
+	})
+	gen.Finish("hello", &TokenUsage{Input: 10, Output: 20, Total: 30, Unit: "TOKENS"}, nil)
+	trace.Finish("hello", nil)
+
+	for _, s := range exp.GetSpans() {
+		if s.Name != "chat.completion" {
+			continue
+		}
+		if spanAttr(s.Attributes, attrObsModel) != "gpt-test" {
+			t.Errorf("model = %q, want gpt-test", spanAttr(s.Attributes, attrObsModel))
+		}
+		usage := spanAttr(s.Attributes, attrObsUsageDetails)
+		if !strings.Contains(usage, `"total":30`) {
+			t.Errorf("usage_details = %q, want total:30", usage)
+		}
+		if s.SpanContext.TraceID().String() != trace.ID {
+			t.Errorf("gen trace id = %s, want %s", s.SpanContext.TraceID(), trace.ID)
+		}
+		return
+	}
+	t.Fatal("generation span not exported")
+}
+
+// TestSpan_FinishMetadataMerged verifies that metadata supplied at Finish is
+// merged into (not discarded, nor overwriting) the metadata set at StartSpan.
+// Regression guard: several call sites only know key fields (outcome,
+// duration_ms, …) at completion and rely on Finish metadata being reported.
+func TestSpan_FinishMetadataMerged(t *testing.T) {
+	m, exp := newTestManager(t)
+
+	ctx, tr := m.StartTrace(context.Background(), TraceOptions{Name: "root"})
+	_, span := m.StartSpan(ctx, SpanOptions{
+		Name:     "work",
+		Metadata: map[string]interface{}{"stage": "ingest", "task_type": "manual"},
+	})
+	span.Finish("out", map[string]interface{}{"outcome": "success", "duration_ms": 42}, nil)
+	tr.Finish(nil, nil)
+
+	for _, s := range exp.GetSpans() {
+		if s.Name != "work" {
+			continue
+		}
+		meta := spanAttr(s.Attributes, attrObsMetadata)
+		for _, want := range []string{`"stage":"ingest"`, `"task_type":"manual"`, `"outcome":"success"`, `"duration_ms":42`} {
+			if !strings.Contains(meta, want) {
+				t.Errorf("span metadata %q missing %q", meta, want)
+			}
+		}
+		return
+	}
+	t.Fatal("work span not exported")
+}
+
+// TestTrace_FinishMetadataMerged verifies the same merge behaviour on a trace:
+// open-time correlation fields (request_id) survive alongside finish outcome.
+func TestTrace_FinishMetadataMerged(t *testing.T) {
+	m, exp := newTestManager(t)
+
+	_, tr := m.StartTrace(context.Background(), TraceOptions{
+		Name:     "root",
+		Metadata: map[string]interface{}{"request_id": "req-1"},
+	})
+	tr.Finish("done", map[string]interface{}{"status": 200})
+
+	for _, s := range exp.GetSpans() {
+		if s.Name != "root" {
+			continue
+		}
+		meta := spanAttr(s.Attributes, attrTraceMetadata)
+		if !strings.Contains(meta, `"request_id":"req-1"`) || !strings.Contains(meta, `"status":200`) {
+			t.Errorf("trace metadata %q missing merged fields", meta)
+		}
+		return
+	}
+	t.Fatal("root span not exported")
+}
+
+// TestStartGeneration_AutoRootExported guards the regression where a
+// generation started with no active trace auto-opened a root span that was
+// never ended and therefore never exported — leaving the generation's parent
+// dangling. After the fix the auto root is exported and shares the trace id.
+func TestStartGeneration_AutoRootExported(t *testing.T) {
+	m, exp := newTestManager(t)
+
+	_, gen := m.StartGeneration(context.Background(), GenerationOptions{Name: "llm", Model: "m"})
+	gen.Finish("out", nil, nil)
+
+	var root, generation tracetest.SpanStub
+	for _, s := range exp.GetSpans() {
+		switch spanType(s) {
+		case obsTypeTrace:
+			root = s
+		case obsTypeGeneration:
+			generation = s
+		}
+	}
+	if root.Name == "" {
+		t.Fatal("auto-created root trace span was not exported")
+	}
+	if generation.Name == "" {
+		t.Fatal("generation span was not exported")
+	}
+	if generation.SpanContext.TraceID() != root.SpanContext.TraceID() {
+		t.Errorf("generation trace id %s != root trace id %s", generation.SpanContext.TraceID(), root.SpanContext.TraceID())
+	}
+	if generation.Parent.SpanID() != root.SpanContext.SpanID() {
+		t.Errorf("generation parent %s != root span %s (dangling parent)", generation.Parent.SpanID(), root.SpanContext.SpanID())
+	}
+}
+
+// TestStartSpan_AutoRootExported is the span counterpart of the above.
+func TestStartSpan_AutoRootExported(t *testing.T) {
+	m, exp := newTestManager(t)
+
+	_, span := m.StartSpan(context.Background(), SpanOptions{Name: "orphan"})
+	span.Finish("out", nil, nil)
+
+	var sawRoot, sawSpan bool
+	for _, s := range exp.GetSpans() {
+		switch spanType(s) {
+		case obsTypeTrace:
+			sawRoot = true
+		case obsTypeSpan:
+			sawSpan = true
+		}
+	}
+	if !sawRoot {
+		t.Error("auto-created root trace span was not exported")
+	}
+	if !sawSpan {
+		t.Error("span was not exported")
+	}
+}
+
+// TestTraceparentPropagation is the sop3 correlation core test: an incoming
+// W3C traceparent (as injected by an upstream caller like sop3) is extracted,
+// and the WeKnora root span inherits the upstream trace id — so in LiteFuse
+// the WeKnora trace and the upstream caller's trace are the same trace.
+func TestTraceparentPropagation(t *testing.T) {
+	m, exp := newTestManager(t)
+
+	// Simulate an upstream caller carrying a traceparent.
+	upstreamCtx, upstreamSpan := m.Tracer().Start(context.Background(), "upstream-caller")
+	remoteTraceID := upstreamSpan.SpanContext().TraceID()
+	carrier := propagation.MapCarrier{}
+	propagator.Inject(upstreamCtx, carrier)
+	if carrier["traceparent"] == "" {
+		t.Fatal("no traceparent injected")
+	}
+
+	// The HTTP middleware extracts the traceparent into the request context.
+	httpCtx := propagator.Extract(context.Background(), carrier)
+	_, trace := m.StartTrace(httpCtx, TraceOptions{Name: "weknora-root"})
+	trace.Finish(nil, nil)
+
+	for _, s := range exp.GetSpans() {
+		if s.Name != "weknora-root" {
+			continue
+		}
+		if s.SpanContext.TraceID() != remoteTraceID {
+			t.Errorf("weknora root trace id = %s, want upstream %s (traceparent not inherited)",
+				s.SpanContext.TraceID(), remoteTraceID)
+		}
+		return
+	}
+	t.Fatal("weknora-root span not exported")
+}
+
+// TestAttachTraceparent_FollowUpJoinsOriginatingTrace is the follow-up
+// suggestion regression: generation often runs on a later HTTP request
+// (POST .../suggestions) after the chat handler has already ended its root
+// span. Without AttachTraceparent, StartGeneration auto-opens an orphan
+// root named after the LLM call. With it, the follow-up span and generation
+// inherit the originating chat trace id.
+func TestAttachTraceparent_FollowUpJoinsOriginatingTrace(t *testing.T) {
+	m, exp := newTestManager(t)
+
+	httpCtx, httpTrace := m.StartTrace(context.Background(), TraceOptions{Name: "POST /api/v1/agent-chat/:session_id"})
+	traceparent := TraceparentFromContext(httpCtx)
+	if traceparent == "" {
+		t.Fatal("expected a traceparent on the HTTP trace")
+	}
+	httpTrace.Finish(nil, nil)
+
+	// Separate request: no *Trace, no OTel span — this is POST /suggestions.
+	followCtx := AttachTraceparent(context.Background(), traceparent)
+	followCtx, span := m.StartSpan(followCtx, SpanOptions{Name: "follow_up.suggestions"})
+	_, gen := m.StartGeneration(followCtx, GenerationOptions{Name: "chat.completion", Model: "m"})
+	gen.Finish("qs", nil, nil)
+	span.Finish(nil, nil, nil)
+
+	var followSpan, generation tracetest.SpanStub
+	var autoRoot bool
+	for _, s := range exp.GetSpans() {
+		switch {
+		case s.Name == "follow_up.suggestions" && spanType(s) == obsTypeSpan:
+			followSpan = s
+		case s.Name == "chat.completion" && spanType(s) == obsTypeGeneration:
+			generation = s
+		case s.Name == "chat.completion" && spanType(s) == obsTypeTrace:
+			autoRoot = true
+		}
+	}
+	if followSpan.Name == "" {
+		t.Fatal("follow_up.suggestions span not exported")
+	}
+	if generation.Name == "" {
+		t.Fatal("chat.completion generation not exported")
+	}
+	if followSpan.SpanContext.TraceID().String() != httpTrace.ID {
+		t.Errorf("follow-up span trace id = %s, want HTTP %s", followSpan.SpanContext.TraceID(), httpTrace.ID)
+	}
+	if generation.SpanContext.TraceID().String() != httpTrace.ID {
+		t.Errorf("generation trace id = %s, want HTTP %s", generation.SpanContext.TraceID(), httpTrace.ID)
+	}
+	if autoRoot {
+		t.Error("StartGeneration opened an orphan chat.completion root; traceparent was not attached")
+	}
+}
+
+// TestAttachTraceparent_LeavesExistingTrace ensures same-request background
+// work (completeAssistantMessage) is not re-parented onto a stale remote
+// span when ctx already carries a live *Trace.
+func TestAttachTraceparent_LeavesExistingTrace(t *testing.T) {
+	m, exp := newTestManager(t)
+
+	otherCtx, other := m.StartTrace(context.Background(), TraceOptions{Name: "other"})
+	otherParent := TraceparentFromContext(otherCtx)
+	other.Finish(nil, nil)
+
+	ctx, live := m.StartTrace(context.Background(), TraceOptions{Name: "live"})
+	ctx = AttachTraceparent(ctx, otherParent)
+	_, gen := m.StartGeneration(ctx, GenerationOptions{Name: "chat.completion", Model: "m"})
+	gen.Finish("out", nil, nil)
+	live.Finish(nil, nil)
+
+	for _, s := range exp.GetSpans() {
+		if spanType(s) != obsTypeGeneration {
+			continue
+		}
+		if s.SpanContext.TraceID().String() != live.ID {
+			t.Errorf("generation joined foreign trace %s, want live %s", s.SpanContext.TraceID(), live.ID)
+		}
+		return
+	}
+	t.Fatal("generation span not exported")
+}