WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

cli/internal/format/bare.go

@@ -0,0 +1,82 @@
+package format
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+)
+
+// WriteJSON serializes v as one-line JSON to w. Bare-data contract: list
+// commands emit their array directly, single-resource commands emit their
+// object directly. The shape is whatever the producing command marshals.
+func WriteJSON(w io.Writer, v any) error {
+	enc := json.NewEncoder(w)
+	enc.SetEscapeHTML(false)
+	return enc.Encode(v)
+}
+
+// WriteJSONFiltered serializes v to w with optional field restriction and
+// optional jq evaluation.
+//
+//   - len(fields) == 0 → no field filter
+//   - jqExpr == ""     → no jq filter
+//
+// Field filter rules:
+//
+//   - v marshals to a top-level array  → each [*] object is restricted to
+//     the named keys
+//   - v marshals to a top-level object → the object is restricted to the
+//     named keys
+//   - v marshals to a scalar           → unchanged
+//
+// Unknown field names are silently dropped so users can pass an aspirational
+// field set across heterogenous list outputs.
+func WriteJSONFiltered(w io.Writer, v any, fields []string, jqExpr string) error {
+	if len(fields) == 0 && jqExpr == "" {
+		return WriteJSON(w, v)
+	}
+	raw, err := marshalJSON(v)
+	if err != nil {
+		return err
+	}
+	if len(fields) > 0 {
+		raw, err = applyBareFieldFilter(raw, fields)
+		if err != nil {
+			return err
+		}
+	}
+	if jqExpr != "" {
+		return writeJQ(w, raw, jqExpr)
+	}
+	_, err = w.Write(raw)
+	return err
+}
+
+// marshalJSON encodes v to a newline-terminated byte slice using the same
+// encoder settings as WriteJSON (HTML escaping disabled).
+func marshalJSON(v any) ([]byte, error) {
+	var buf bytes.Buffer
+	enc := json.NewEncoder(&buf)
+	enc.SetEscapeHTML(false)
+	if err := enc.Encode(v); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+// applyBareFieldFilter dispatches on the JSON shape of raw and restricts
+// elements / object keys to the named fields.
+func applyBareFieldFilter(raw []byte, fields []string) ([]byte, error) {
+	trimmed := bytes.TrimSpace(raw)
+	if len(trimmed) == 0 {
+		return raw, nil
+	}
+	switch trimmed[0] {
+	case '[':
+		return filterArrayItems(raw, fields)
+	case '{':
+		return filterObjectKeys(raw, fields)
+	default:
+		return raw, nil
+	}
+}

cli/internal/format/bare_test.go

@@ -0,0 +1,140 @@
+package format_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/Tencent/WeKnora/cli/internal/format"
+)
+
+func TestWriteJSON_BareArray(t *testing.T) {
+	buf := &bytes.Buffer{}
+	if err := format.WriteJSON(buf, []map[string]string{
+		{"id": "1", "name": "alpha"},
+		{"id": "2", "name": "beta"},
+	}); err != nil {
+		t.Fatalf("err = %v", err)
+	}
+	if !bytes.HasPrefix(buf.Bytes(), []byte("[")) {
+		t.Errorf("expected bare JSON array, got %q", buf.String())
+	}
+}
+
+func TestWriteJSON_BareObject(t *testing.T) {
+	buf := &bytes.Buffer{}
+	if err := format.WriteJSON(buf, map[string]any{"id": "kb_x", "name": "Engineering"}); err != nil {
+		t.Fatalf("err = %v", err)
+	}
+	if !bytes.HasPrefix(buf.Bytes(), []byte("{")) {
+		t.Errorf("expected bare JSON object, got %q", buf.String())
+	}
+	if bytes.Contains(buf.Bytes(), []byte(`"ok":`)) || bytes.Contains(buf.Bytes(), []byte(`"data":`)) {
+		t.Errorf("bare output must not carry envelope keys: %s", buf.String())
+	}
+}
+
+func TestWriteJSONFiltered_FieldsOnArray(t *testing.T) {
+	buf := &bytes.Buffer{}
+	items := []map[string]any{
+		{"id": "1", "name": "alpha", "kb_id": "kb_x", "updated_at": "2026-01-01"},
+		{"id": "2", "name": "beta", "kb_id": "kb_x", "updated_at": "2026-01-02"},
+	}
+	if err := format.WriteJSONFiltered(buf, items, []string{"id", "name"}, ""); err != nil {
+		t.Fatalf("err = %v", err)
+	}
+	var got []map[string]string
+	if err := json.Unmarshal(buf.Bytes(), &got); err != nil {
+		t.Fatalf("parse: %v\n%s", err, buf.String())
+	}
+	if len(got) != 2 {
+		t.Fatalf("items len = %d, want 2", len(got))
+	}
+	for i, item := range got {
+		if _, has := item["kb_id"]; has {
+			t.Errorf("item[%d] should not have kb_id: %v", i, item)
+		}
+		if item["id"] == "" || item["name"] == "" {
+			t.Errorf("item[%d] missing kept fields: %v", i, item)
+		}
+	}
+}
+
+func TestWriteJSONFiltered_FieldsOnObject(t *testing.T) {
+	buf := &bytes.Buffer{}
+	obj := map[string]any{"id": "kb_x", "name": "Engineering", "owner": "alice"}
+	if err := format.WriteJSONFiltered(buf, obj, []string{"id", "name"}, ""); err != nil {
+		t.Fatalf("err = %v", err)
+	}
+	var got map[string]string
+	if err := json.Unmarshal(buf.Bytes(), &got); err != nil {
+		t.Fatalf("parse: %v\n%s", err, buf.String())
+	}
+	if _, has := got["owner"]; has {
+		t.Errorf("should not retain owner: %v", got)
+	}
+	if got["id"] != "kb_x" || got["name"] != "Engineering" {
+		t.Errorf("kept fields missing: %v", got)
+	}
+}
+
+func TestWriteJSONFiltered_UnknownFieldSilent(t *testing.T) {
+	buf := &bytes.Buffer{}
+	if err := format.WriteJSONFiltered(buf, map[string]any{"id": "1"}, []string{"id", "nonexistent"}, ""); err != nil {
+		t.Fatalf("err = %v", err)
+	}
+	var got map[string]any
+	if err := json.Unmarshal(buf.Bytes(), &got); err != nil {
+		t.Fatalf("parse: %v", err)
+	}
+	if got["id"] != "1" {
+		t.Errorf("id missing: %v", got)
+	}
+	if _, has := got["nonexistent"]; has {
+		t.Errorf("nonexistent should be silently dropped: %v", got)
+	}
+}
+
+func TestWriteJSONFiltered_JQOnly(t *testing.T) {
+	buf := &bytes.Buffer{}
+	items := []map[string]any{
+		{"id": "1", "name": "alpha"},
+		{"id": "2", "name": "beta"},
+	}
+	if err := format.WriteJSONFiltered(buf, items, nil, ".[].id"); err != nil {
+		t.Fatalf("err = %v", err)
+	}
+	// String / scalar results render without JSON quotes so scalar
+	// projections (e.g. `--jq '.[].id'`) pipe cleanly into shell tools.
+	lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n")
+	if len(lines) != 2 || lines[0] != "1" || lines[1] != "2" {
+		t.Errorf("jq output mismatch: %q", buf.String())
+	}
+}
+
+func TestWriteJSONFiltered_FieldsAndJQ(t *testing.T) {
+	buf := &bytes.Buffer{}
+	items := []map[string]any{
+		{"id": "1", "name": "alpha", "secret": "drop-me"},
+		{"id": "2", "name": "beta", "secret": "drop-me"},
+	}
+	// Field filter first → then jq selects from filtered shape.
+	if err := format.WriteJSONFiltered(buf, items, []string{"id"}, ".[].id"); err != nil {
+		t.Fatalf("err = %v", err)
+	}
+	out := buf.String()
+	if strings.Contains(out, "drop-me") {
+		t.Errorf("field filter must drop unrequested keys before jq: %q", out)
+	}
+}
+
+func TestWriteJSONFiltered_NilDataPassthrough(t *testing.T) {
+	buf := &bytes.Buffer{}
+	if err := format.WriteJSONFiltered(buf, nil, []string{"id"}, ""); err != nil {
+		t.Fatalf("err = %v", err)
+	}
+	if strings.TrimSpace(buf.String()) != "null" {
+		t.Errorf("nil should marshal to bare null, got %q", buf.String())
+	}
+}

cli/internal/format/dash.go

@@ -0,0 +1,11 @@
+package format
+
+// DashIfEmpty returns "-" for empty strings, otherwise s itself. Standard
+// rendering convention for human-readable table cells whose value is
+// optional (e.g. user email in `auth list` / `profile list`).
+func DashIfEmpty(s string) string {
+	if s == "" {
+		return "-"
+	}
+	return s
+}

cli/internal/format/filter.go

@@ -0,0 +1,98 @@
+package format
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/itchyny/gojq"
+)
+
+// filterArrayItems applies filterObjectKeys to each element of an array.
+// Non-object elements (e.g. an array of strings) are passed through.
+func filterArrayItems(arrayRaw json.RawMessage, fields []string) (json.RawMessage, error) {
+	var items []json.RawMessage
+	if err := json.Unmarshal(arrayRaw, &items); err != nil {
+		return nil, fmt.Errorf("field filter: parse array: %w", err)
+	}
+	for i, item := range items {
+		trimmed := bytes.TrimSpace(item)
+		if len(trimmed) == 0 || trimmed[0] != '{' {
+			continue
+		}
+		filtered, err := filterObjectKeys(item, fields)
+		if err != nil {
+			return nil, err
+		}
+		items[i] = filtered
+	}
+	return json.Marshal(items)
+}
+
+// filterObjectKeys produces a new object containing only the listed keys
+// that were present in the source.
+func filterObjectKeys(objRaw json.RawMessage, fields []string) (json.RawMessage, error) {
+	var src map[string]json.RawMessage
+	if err := json.Unmarshal(objRaw, &src); err != nil {
+		return nil, fmt.Errorf("field filter: parse object keys: %w", err)
+	}
+	dst := make(map[string]json.RawMessage, len(fields))
+	for _, k := range fields {
+		if v, ok := src[k]; ok {
+			dst[k] = v
+		}
+	}
+	return json.Marshal(dst)
+}
+
+// JQError marks a failure attributable to the user-supplied --jq expression
+// (parse or evaluation), as opposed to an internal serialization fault. The
+// cmdutil layer (which can't be imported here without a cycle) maps it to
+// input.invalid_argument so agents fix the expression rather than treating a
+// bad --jq as a CLI bug (internal.error / exit 1).
+type JQError struct{ Err error }
+
+func (e *JQError) Error() string { return e.Err.Error() }
+func (e *JQError) Unwrap() error { return e.Err }
+
+// writeJQ evaluates expr against raw and writes each result line by line to w.
+// String results render without quotes (so `--jq '.name'` yields shell-friendly
+// bare strings); non-string results use encoding/json.
+//
+// Expression-attributable failures (parse / eval) are wrapped in *JQError so
+// the caller can map them to a typed input.invalid_argument code. A failure to
+// parse `raw` itself is internal (our own envelope JSON) and stays plain.
+func writeJQ(w io.Writer, raw []byte, expr string) error {
+	query, err := gojq.Parse(expr)
+	if err != nil {
+		return &JQError{Err: fmt.Errorf("jq parse: %w", err)}
+	}
+	var input any
+	if err := json.Unmarshal(raw, &input); err != nil {
+		return fmt.Errorf("jq input parse: %w", err)
+	}
+	iter := query.Run(input)
+	for {
+		v, ok := iter.Next()
+		if !ok {
+			return nil
+		}
+		if e, ok := v.(error); ok {
+			return &JQError{Err: fmt.Errorf("jq eval: %w", e)}
+		}
+		if s, ok := v.(string); ok {
+			if _, err := fmt.Fprintln(w, s); err != nil {
+				return err
+			}
+			continue
+		}
+		out, err := json.Marshal(v)
+		if err != nil {
+			return err
+		}
+		if _, err := w.Write(append(out, '\n')); err != nil {
+			return err
+		}
+	}
+}

cli/internal/format/filter_test.go

@@ -0,0 +1,50 @@
+package format_test
+
+import (
+	"bytes"
+	"errors"
+	"testing"
+
+	"github.com/Tencent/WeKnora/cli/internal/format"
+)
+
+// A malformed --jq expression must surface as *format.JQError so the cmdutil
+// layer can map it to a typed input.invalid_argument (exit 5) instead of the
+// unclassified internal.error bucket (exit 1). Regression for the bad-jq
+// misclassification.
+func TestWriteJSONFiltered_BadJQExpr_IsJQError(t *testing.T) {
+	for _, expr := range []string{".data[", "this is not jq"} {
+		buf := &bytes.Buffer{}
+		err := format.WriteJSONFiltered(buf, map[string]any{"data": []any{}}, nil, expr)
+		if err == nil {
+			t.Fatalf("expr %q: expected error, got nil", expr)
+		}
+		var jqe *format.JQError
+		if !errors.As(err, &jqe) {
+			t.Errorf("expr %q: expected *format.JQError, got %T: %v", expr, err, err)
+		}
+	}
+}
+
+// A jq eval failure (valid parse, runtime error) is also expression-attributable
+// and must be a *JQError.
+func TestWriteJSONFiltered_JQEvalError_IsJQError(t *testing.T) {
+	buf := &bytes.Buffer{}
+	// ".foo.bar" against a non-object .foo raises an eval error.
+	err := format.WriteJSONFiltered(buf, map[string]any{"foo": 5}, nil, ".foo.bar")
+	if err == nil {
+		t.Fatal("expected eval error, got nil")
+	}
+	var jqe *format.JQError
+	if !errors.As(err, &jqe) {
+		t.Errorf("expected *format.JQError, got %T: %v", err, err)
+	}
+}
+
+// A valid expression must NOT error.
+func TestWriteJSONFiltered_ValidJQ_NoError(t *testing.T) {
+	buf := &bytes.Buffer{}
+	if err := format.WriteJSONFiltered(buf, map[string]any{"data": []any{"a", "b"}}, nil, ".data[]"); err != nil {
+		t.Fatalf("valid jq errored: %v", err)
+	}
+}

cli/internal/format/ndjson.go

@@ -0,0 +1,25 @@
+package format
+
+import (
+	"io"
+	"reflect"
+)
+
+// WriteNDJSON serializes v as NDJSON: one JSON value per line, '\n' terminated.
+// If v is a slice or array, each element gets its own line; otherwise v is
+// emitted as a single line. Per http://ndjson.org.
+//
+// HTML escaping is disabled (via WriteJSON) so '<', '>', '&' stay literal —
+// cleaner for agent stream-parsers that don't need HTML-safe output.
+func WriteNDJSON(w io.Writer, v any) error {
+	rv := reflect.ValueOf(v)
+	if rv.IsValid() && (rv.Kind() == reflect.Slice || rv.Kind() == reflect.Array) {
+		for i := 0; i < rv.Len(); i++ {
+			if err := WriteJSON(w, rv.Index(i).Interface()); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	return WriteJSON(w, v)
+}

cli/internal/format/ndjson_test.go

@@ -0,0 +1,67 @@
+package format_test
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/Tencent/WeKnora/cli/internal/format"
+)
+
+func TestWriteNDJSON_ArraySplitsPerElement(t *testing.T) {
+	var buf bytes.Buffer
+	arr := []map[string]any{{"id": "1"}, {"id": "2"}, {"id": "3"}}
+	if err := format.WriteNDJSON(&buf, arr); err != nil {
+		t.Fatalf("WriteNDJSON: %v", err)
+	}
+	want := `{"id":"1"}` + "\n" + `{"id":"2"}` + "\n" + `{"id":"3"}` + "\n"
+	if buf.String() != want {
+		t.Errorf("got %q want %q", buf.String(), want)
+	}
+}
+
+func TestWriteNDJSON_SingleRecord(t *testing.T) {
+	var buf bytes.Buffer
+	rec := map[string]string{"id": "abc"}
+	if err := format.WriteNDJSON(&buf, rec); err != nil {
+		t.Fatalf("WriteNDJSON: %v", err)
+	}
+	want := `{"id":"abc"}` + "\n"
+	if buf.String() != want {
+		t.Errorf("got %q want %q", buf.String(), want)
+	}
+}
+
+func TestWriteNDJSON_EmptySlice(t *testing.T) {
+	var buf bytes.Buffer
+	if err := format.WriteNDJSON(&buf, []string{}); err != nil {
+		t.Fatalf("WriteNDJSON: %v", err)
+	}
+	if buf.String() != "" {
+		t.Errorf("got %q want empty", buf.String())
+	}
+}
+
+func TestWriteNDJSON_TypedSlice(t *testing.T) {
+	type item struct {
+		ID string `json:"id"`
+	}
+	var buf bytes.Buffer
+	if err := format.WriteNDJSON(&buf, []item{{ID: "x"}, {ID: "y"}}); err != nil {
+		t.Fatalf("WriteNDJSON: %v", err)
+	}
+	want := `{"id":"x"}` + "\n" + `{"id":"y"}` + "\n"
+	if buf.String() != want {
+		t.Errorf("got %q want %q", buf.String(), want)
+	}
+}
+
+func TestWriteNDJSON_NoHTMLEscape(t *testing.T) {
+	var buf bytes.Buffer
+	if err := format.WriteNDJSON(&buf, map[string]string{"k": "a<b>c&d"}); err != nil {
+		t.Fatalf("WriteNDJSON: %v", err)
+	}
+	if !strings.Contains(buf.String(), "a<b>c&d") {
+		t.Errorf("expected unescaped HTML chars, got %q", buf.String())
+	}
+}

cli/internal/format/references.go

@@ -0,0 +1,36 @@
+package format
+
+import sdk "github.com/Tencent/WeKnora/client"
+
+// ReferenceIndex is the bounded citation pointer exposed by projected JSON,
+// text, and MCP output. ChunkID is the chunk the model cited; ParentChunkID is
+// retained when callers prefer to fetch the larger self-contained passage.
+// Full chunk text and retrieval metadata remain available only in raw NDJSON.
+type ReferenceIndex struct {
+	KBID          string `json:"kb_id,omitempty"`
+	ChunkID       string `json:"chunk_id"`
+	ParentChunkID string `json:"parent_chunk_id,omitempty"`
+}
+
+// IndexReferences projects full SDK search results into stable lookup keys.
+// It never mutates the SDK events, which keeps the raw NDJSON path lossless.
+// fallbackKBID is used by `chat`, whose single KB is known by the CLI even
+// when an older server omits knowledge_base_id from a reference.
+func IndexReferences(refs []*sdk.SearchResult, fallbackKBID string) []ReferenceIndex {
+	indexes := make([]ReferenceIndex, 0, len(refs))
+	for _, r := range refs {
+		if r == nil || r.ID == "" {
+			continue
+		}
+		kbID := r.KnowledgeBaseID
+		if kbID == "" {
+			kbID = fallbackKBID
+		}
+		indexes = append(indexes, ReferenceIndex{
+			KBID:          kbID,
+			ChunkID:       r.ID,
+			ParentChunkID: r.ParentChunkID,
+		})
+	}
+	return indexes
+}

cli/internal/format/references_test.go

@@ -0,0 +1,38 @@
+package format
+
+import (
+	"testing"
+
+	sdk "github.com/Tencent/WeKnora/client"
+)
+
+func TestIndexReferences_ProjectsLookupFieldsWithoutMutation(t *testing.T) {
+	refs := []*sdk.SearchResult{
+		{ID: "c1", Content: "bulky passage one", KnowledgeBaseID: "kb1", ParentChunkID: "p1", KnowledgeTitle: "Doc One", Score: 0.5},
+		{ID: "c2", Content: "bulky passage two", ParentChunkID: "p2"},
+		nil,
+	}
+	got := IndexReferences(refs, "fallback")
+
+	if len(got) != 2 {
+		t.Fatalf("indexes=%d, want 2", len(got))
+	}
+	if got[0].KBID != "kb1" || got[0].ChunkID != "c1" || got[0].ParentChunkID != "p1" {
+		t.Errorf("first index=%+v", got[0])
+	}
+	if got[1].KBID != "fallback" || got[1].ChunkID != "c2" {
+		t.Errorf("fallback index=%+v", got[1])
+	}
+	if refs[0].Content != "bulky passage one" || refs[1].Content != "bulky passage two" {
+		t.Errorf("source references mutated: %+v", refs)
+	}
+}
+
+func TestIndexReferences_NilSafe(t *testing.T) {
+	if got := IndexReferences(nil, ""); len(got) != 0 {
+		t.Errorf("nil indexes=%+v", got)
+	}
+	if got := IndexReferences([]*sdk.SearchResult{nil, nil}, ""); len(got) != 0 {
+		t.Errorf("nil-entry indexes=%+v", got)
+	}
+}