package wiki import ( "bytes" "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "os" "slices" "strings" "testing" "time" "unicode/utf8" "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" secutils "github.com/Tencent/WeKnora/internal/utils" ) func TestMain(m *testing.M) { os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost,open.feishu.cn,open.larksuite.com") secutils.ResetSSRFWhitelistForTest() os.Exit(m.Run()) } // ────────────────────────────────────────────────────────────────────── // Fake Feishu API server // ────────────────────────────────────────────────────────────────────── // fakeFeishu builds an httptest.Server that emulates the relevant Feishu APIs. // It returns the server and a core.Config pointing at it. func fakeFeishu(nodes []core.WikiNode) (*httptest.Server, *core.Config) { mux := http.NewServeMux() // --- auth --- mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.TokenResponse{ ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200, }) }) // --- wiki spaces --- mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.WikiSpaceListResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.WikiSpaceListData{ Items: []core.WikiSpace{ {SpaceID: "space1", Name: "Test Space", Description: "desc", Visibility: "public"}, }, }, }) }) // --- wiki nodes (top-level only for simplicity) --- mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.WikiNodeListResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.WikiNodeListData{ Items: nodes, }, }) }) // --- export task: create --- mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodPost { writeJSON(w, core.ExportTaskCreateResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.ExportTaskCreateData{Ticket: "ticket-123"}, }) return } // GET /open-apis/drive/v1/export_tasks/ticket-123 writeJSON(w, core.ExportTaskStatusResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.ExportTaskStatusData{ Result: core.ExportTaskResult{ FileToken: "ft-abc", FileSize: 100, JobStatus: 0, // success FileName: "exported.docx", }, }, }) }) // --- export task: status polling (pattern match with ticket) --- mux.HandleFunc("/open-apis/drive/v1/export_tasks/ticket-123", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.ExportTaskStatusResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.ExportTaskStatusData{ Result: core.ExportTaskResult{ FileToken: "ft-abc", FileSize: 100, JobStatus: 0, FileName: "exported.docx", }, }, }) }) // --- export file download --- mux.HandleFunc("/open-apis/drive/v1/export_tasks/file/ft-abc/download", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/octet-stream") w.Write([]byte("fake-docx-content")) }) // --- drive file download (for "file" type nodes) --- mux.HandleFunc("/open-apis/drive/v1/files/", func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/download") { w.Header().Set("Content-Type", "application/octet-stream") w.Write([]byte("fake-pdf-binary")) return } http.NotFound(w, r) }) ts := httptest.NewServer(mux) cfg := &core.Config{ AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL, } return ts, cfg } func fakeFeishuWithChildFailure(topNodes []core.WikiNode, failingParentToken string) (*httptest.Server, *core.Config) { return fakeFeishuHierarchy(topNodes, nil, failingParentToken) } func fakeFeishuHierarchy(topNodes []core.WikiNode, childNodes map[string][]core.WikiNode, failingParentToken string) (*httptest.Server, *core.Config) { mux := http.NewServeMux() nodeByToken := make(map[string]core.WikiNode) for _, node := range topNodes { node.SpaceID = "space1" nodeByToken[node.NodeToken] = node } for parentToken, nodes := range childNodes { for _, node := range nodes { node.SpaceID = "space1" if node.ParentNodeID == "" { node.ParentNodeID = parentToken } nodeByToken[node.NodeToken] = node } } mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.TokenResponse{ ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.WikiSpaceListResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.WikiSpaceListData{ Items: []core.WikiSpace{ {SpaceID: "space1", Name: "Test Space", Description: "desc", Visibility: "public"}, }, }, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) { parentToken := r.URL.Query().Get("parent_node_token") if parentToken != "" && parentToken == failingParentToken { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(`{"code":1663,"msg":"internal error"}`)) return } nodes := topNodes if parentToken != "" { nodes = childNodes[parentToken] for i := range nodes { if nodes[i].ParentNodeID == "" { nodes[i].ParentNodeID = parentToken } if nodes[i].SpaceID == "" { nodes[i].SpaceID = "space1" } } } writeJSON(w, core.WikiNodeListResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.WikiNodeListData{ Items: nodes, }, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces/get_node", func(w http.ResponseWriter, r *http.Request) { nodeToken := r.URL.Query().Get("token") node, ok := nodeByToken[nodeToken] if !ok { writeJSON(w, core.WikiNodeInfoResponse{ ApiResponse: core.ApiResponse{Code: 1663, Msg: "node not found"}, }) return } writeJSON(w, core.WikiNodeInfoResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.WikiNodeInfoData{Node: node}, }) }) mux.HandleFunc("/open-apis/drive/v1/files/", func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/download") { w.Header().Set("Content-Type", "application/octet-stream") _, _ = w.Write([]byte("fake-file-content")) return } http.NotFound(w, r) }) ts := httptest.NewServer(mux) cfg := &core.Config{ AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL, } return ts, cfg } func writeJSON(w http.ResponseWriter, v interface{}) { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(v) } func makeConfig(cfg *core.Config, resourceIDs []string) *types.DataSourceConfig { creds := map[string]interface{}{ "app_id": cfg.AppID, "app_secret": cfg.AppSecret, "base_url": cfg.BaseURL, } return &types.DataSourceConfig{ Type: types.ConnectorTypeFeishu, Credentials: creds, ResourceIDs: resourceIDs, } } // ────────────────────────────────────────────────────────────────────── // Helper function tests // ────────────────────────────────────────────────────────────────────── func TestIsSupportedDocType(t *testing.T) { tests := []struct { objType string expected bool }{ {"docx", true}, {"doc", true}, {"sheet", true}, {"bitable", true}, {"file", true}, {"mindnote", false}, {"slides", false}, {"unknown", false}, {"", false}, } for _, tt := range tests { t.Run(tt.objType, func(t *testing.T) { got := core.IsSupportedDocType(tt.objType) if got != tt.expected { t.Errorf("core.IsSupportedDocType(%q) = %v, want %v", tt.objType, got, tt.expected) } }) } } func TestSanitizeFileName(t *testing.T) { tests := []struct { input string expected string }{ {"hello", "hello"}, {"", "untitled"}, {"a/b\\c:d*e", "a_b_c_d_e"}, {"normal file.docx", "normal file.docx"}, {strings.Repeat("a", 300), strings.Repeat("a", 200)}, } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { got := core.SanitizeFileName(tt.input) if got != tt.expected { t.Errorf("core.SanitizeFileName(%q) = %q, want %q", tt.input, got, tt.expected) } }) } } func TestSanitizeFileName_TruncatesAtRuneBoundary(t *testing.T) { // Each 测 is 3 bytes; raw byte truncation at 200 would split a rune and // produce invalid UTF-8 that downstream filename validation rejects. long := strings.Repeat("测试", 100) got := core.SanitizeFileName(long) if !utf8.ValidString(got) { t.Fatalf("core.SanitizeFileName produced invalid UTF-8: %q", got) } if len(got) > 200 { t.Errorf("len = %d, want ≤ 200", len(got)) } if len(got) == 0 { t.Error("result is empty") } } func TestSanitizeFileName_PreservesExtensionOnTruncation(t *testing.T) { // A pathologically long (>200-byte) CJK attachment name ending in ".pdf". // Truncation must keep the ".pdf" suffix — downstream file-type validation // classifies by extension, so a chopped extension would reject the file. long := strings.Repeat("报告", 150) + ".pdf" // 150*6 bytes + ".pdf" got := core.SanitizeFileName(long) if !utf8.ValidString(got) { t.Fatalf("produced invalid UTF-8: %q", got) } if len(got) > 200 { t.Errorf("len = %d, want ≤ 200", len(got)) } if !strings.HasSuffix(got, ".pdf") { t.Errorf("extension dropped by truncation: %q must end in .pdf", got) } } func TestParseFeishuTimestamp(t *testing.T) { ts := core.ParseFeishuTimestamp("1711468800") // 2024-03-27 00:00:00 UTC if ts.IsZero() { t.Fatal("expected non-zero time") } if ts.Unix() != 1711468800 { t.Errorf("unexpected unix = %d", ts.Unix()) } if !core.ParseFeishuTimestamp("").IsZero() { t.Error("expected zero time for empty string") } if !core.ParseFeishuTimestamp("invalid").IsZero() { t.Error("expected zero time for invalid string") } } func TestParseFeishuConfig(t *testing.T) { t.Run("valid", func(t *testing.T) { cfg, err := core.ParseFeishuConfig(&types.DataSourceConfig{ Credentials: map[string]interface{}{ "app_id": "id1", "app_secret": "sec1", "base_url": "https://open.feishu.cn", }, }, core.RegionFeishu) if err != nil { t.Fatalf("unexpected error: %v", err) } if cfg.AppID != "id1" || cfg.AppSecret != "sec1" || cfg.BaseURL != "https://open.feishu.cn" { t.Errorf("unexpected config: %+v", cfg) } }) t.Run("nil config", func(t *testing.T) { _, err := core.ParseFeishuConfig(nil, core.RegionFeishu) if err == nil { t.Fatal("expected error for nil config") } }) t.Run("missing credentials", func(t *testing.T) { _, err := core.ParseFeishuConfig(&types.DataSourceConfig{ Credentials: map[string]interface{}{ "app_id": "id1", // missing app_secret }, }, core.RegionFeishu) if err == nil { t.Fatal("expected error for missing app_secret") } }) } // ────────────────────────────────────────────────────────────────────── // Connector interface tests // ────────────────────────────────────────────────────────────────────── func TestConnectorType(t *testing.T) { c := NewConnector(core.RegionFeishu) if c.Type() != types.ConnectorTypeFeishu { t.Errorf("Type() = %q, want %q", c.Type(), types.ConnectorTypeFeishu) } } func TestConnectorValidate(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() c := NewConnector(core.RegionFeishu) err := c.Validate(context.Background(), makeConfig(cfg, nil)) if err != nil { t.Fatalf("Validate() error: %v", err) } } func TestConnectorValidate_BadCredentials(t *testing.T) { c := NewConnector(core.RegionFeishu) err := c.Validate(context.Background(), &types.DataSourceConfig{ Credentials: map[string]interface{}{ "app_id": "bad", "app_secret": "bad", "base_url": "http://127.0.0.1:1", // will fail to connect }, }) if err == nil { t.Fatal("expected error for bad credentials") } } func TestConnectorListResources(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() c := NewConnector(core.RegionFeishu) resources, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "") if err != nil { t.Fatalf("ListResources() error: %v", err) } if len(resources) != 1 { t.Fatalf("expected 1 resource, got %d", len(resources)) } if resources[0].ExternalID != "space1" { t.Errorf("ExternalID = %q, want %q", resources[0].ExternalID, "space1") } if resources[0].Name != "Test Space" { t.Errorf("Name = %q, want %q", resources[0].Name, "Test Space") } if resources[0].Type != "wiki_space" { t.Errorf("Type = %q, want %q", resources[0].Type, "wiki_space") } if !resources[0].HasChildren { t.Errorf("HasChildren = false, want true") } } // TestConnectorListResources_LazyLoadsOneLevel verifies that ListResources loads // the wiki tree lazily — only the requested level — instead of recursing the whole // tree up front (Tencent/WeKnora#1672). func TestConnectorListResources_LazyLoadsOneLevel(t *testing.T) { topNodes := []core.WikiNode{ {NodeToken: "nt-root", ObjToken: "obj-root", ObjType: "docx", Title: "Root", HasChild: true, ObjEditTime: "100"}, {NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "docx", Title: "Peer", ObjEditTime: "200"}, } childNodes := map[string][]core.WikiNode{ "nt-root": { {NodeToken: "nt-child", ObjToken: "obj-child", ObjType: "docx", Title: "Child", ObjEditTime: "300"}, }, } ts, cfg := fakeFeishuHierarchy(topNodes, childNodes, "") defer ts.Close() c := NewConnector(core.RegionFeishu) // Root listing: only the space, no descendants. spaces, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "") if err != nil { t.Fatalf("ListResources(root) error: %v", err) } if len(spaces) != 1 || spaces[0].ExternalID != "space1" || !spaces[0].HasChildren { t.Fatalf("want single space with children, got %+v", spaces) } // Expanding the space returns only its top-level nodes. top, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "space1") if err != nil { t.Fatalf("ListResources(space1) error: %v", err) } if len(top) != 2 { t.Fatalf("want 2 top-level nodes, got %d: %+v", len(top), top) } byID := make(map[string]types.Resource) for _, r := range top { byID[r.ExternalID] = r } root := byID["space1:nt-root"] if root.ParentID != "space1" || !root.HasChildren { t.Fatalf("root node resource wrong: %+v", root) } // Expanding a node returns only its direct children. children, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "space1:nt-root") if err != nil { t.Fatalf("ListResources(space1:nt-root) error: %v", err) } if len(children) != 1 { t.Fatalf("want 1 child node, got %d: %+v", len(children), children) } child := children[0] if child.ExternalID != "space1:nt-child" || child.ParentID != "space1:nt-root" || child.Name != "Child" { t.Fatalf("child resource wrong: %+v", child) } if child.Metadata["space_id"] != "space1" || child.Metadata["node_token"] != "nt-child" { t.Fatalf("child metadata wrong: %+v", child.Metadata) } } // TestConnectorResolveResourceAncestors verifies that the ancestor chain of a // deeply nested selection is resolved (so an edit-mode picker can reveal it) // without listing the whole tree. func TestConnectorResolveResourceAncestors(t *testing.T) { topNodes := []core.WikiNode{ {NodeToken: "nt-root", ObjToken: "obj-root", ObjType: "docx", Title: "Root", HasChild: true}, } childNodes := map[string][]core.WikiNode{ "nt-root": { {NodeToken: "nt-child", ObjToken: "obj-child", ObjType: "docx", Title: "Child", HasChild: true}, }, "nt-child": { {NodeToken: "nt-grandchild", ObjToken: "obj-gc", ObjType: "docx", Title: "Grandchild"}, }, } ts, cfg := fakeFeishuHierarchy(topNodes, childNodes, "") defer ts.Close() c := NewConnector(core.RegionFeishu) // A deeply nested node resolves to its space plus every intermediate parent. ancestors, err := c.ResolveResourceAncestors( context.Background(), makeConfig(cfg, nil), []string{"space1:nt-grandchild"}, ) if err != nil { t.Fatalf("ResolveResourceAncestors error: %v", err) } got := make(map[string]bool) for _, a := range ancestors { got[a] = true } for _, want := range []string{"space1", "space1:nt-root", "space1:nt-child"} { if !got[want] { t.Fatalf("missing ancestor %q, got %+v", want, ancestors) } } if got["space1:nt-grandchild"] { t.Fatalf("selection itself must not be returned as its own ancestor: %+v", ancestors) } // A top-level node only needs its space loaded. ancestors, err = c.ResolveResourceAncestors( context.Background(), makeConfig(cfg, nil), []string{"space1:nt-root"}, ) if err != nil { t.Fatalf("ResolveResourceAncestors(top-level) error: %v", err) } if len(ancestors) != 1 || ancestors[0] != "space1" { t.Fatalf("want [space1] for a top-level node, got %+v", ancestors) } // A whole-space selection is already top-level: nothing to reveal. ancestors, err = c.ResolveResourceAncestors( context.Background(), makeConfig(cfg, nil), []string{"space1"}, ) if err != nil { t.Fatalf("ResolveResourceAncestors(space) error: %v", err) } if len(ancestors) != 0 { t.Fatalf("want no ancestors for a space selection, got %+v", ancestors) } } // ────────────────────────────────────────────────────────────────────── // FetchAll: test all supported doc types // ────────────────────────────────────────────────────────────────────── func TestFetchAll_DocxNode(t *testing.T) { nodes := []core.WikiNode{{ NodeToken: "nt1", ObjToken: "obj-docx-1", ObjType: "docx", Title: "My Document", NodeEditTime: "1711468800", }} ts, cfg := fakeFeishu(nodes) defer ts.Close() c := NewConnector(core.RegionFeishu) items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) } if len(items) != 1 { t.Fatalf("expected 1 item, got %d", len(items)) } item := items[0] if item.ExternalID != "nt1" { t.Errorf("ExternalID = %q, want %q", item.ExternalID, "nt1") } if item.Title != "My Document" { t.Errorf("Title = %q", item.Title) } if string(item.Content) != "fake-docx-content" { t.Errorf("Content = %q, want %q", string(item.Content), "fake-docx-content") } if item.FileName != "exported.docx" { t.Errorf("FileName = %q, want %q", item.FileName, "exported.docx") } if item.Metadata["obj_type"] != "docx" { t.Errorf("Metadata[obj_type] = %q", item.Metadata["obj_type"]) } if item.Metadata["channel"] != types.ChannelFeishu { t.Errorf("Metadata[channel] = %q", item.Metadata["channel"]) } } func TestFetchAll_SheetNode(t *testing.T) { nodes := []core.WikiNode{{ NodeToken: "nt-sheet", ObjToken: "obj-sheet-1", ObjType: "sheet", Title: "Sales Report", NodeEditTime: "1711468800", }} ts, cfg := fakeFeishu(nodes) defer ts.Close() c := NewConnector(core.RegionFeishu) items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) } if len(items) != 1 { t.Fatalf("expected 1 item, got %d", len(items)) } if items[0].Metadata["obj_type"] != "sheet" { t.Errorf("obj_type = %q, want sheet", items[0].Metadata["obj_type"]) } } func TestFetchAll_BitableNode(t *testing.T) { nodes := []core.WikiNode{{ NodeToken: "nt-bitable", ObjToken: "obj-bitable-1", ObjType: "bitable", Title: "Project Tracker", NodeEditTime: "1711468800", }} ts, cfg := fakeFeishu(nodes) defer ts.Close() c := NewConnector(core.RegionFeishu) items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) } if len(items) != 1 { t.Fatalf("expected 1 item, got %d", len(items)) } if items[0].Metadata["obj_type"] != "bitable" { t.Errorf("obj_type = %q, want bitable", items[0].Metadata["obj_type"]) } } func TestFetchAll_FileNode(t *testing.T) { nodes := []core.WikiNode{{ NodeToken: "nt-file", ObjToken: "obj-file-1", ObjType: "file", Title: "manual.pdf", NodeEditTime: "1711468800", }} ts, cfg := fakeFeishu(nodes) defer ts.Close() c := NewConnector(core.RegionFeishu) items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) } if len(items) != 1 { t.Fatalf("expected 1 item, got %d", len(items)) } item := items[0] if string(item.Content) != "fake-pdf-binary" { t.Errorf("Content = %q, want %q", string(item.Content), "fake-pdf-binary") } if item.FileName != "manual.pdf" { t.Errorf("FileName = %q, want %q", item.FileName, "manual.pdf") } if item.Metadata["obj_type"] != "file" { t.Errorf("obj_type = %q, want file", item.Metadata["obj_type"]) } } func TestFetchAll_SkipsMindnoteAndSlides(t *testing.T) { nodes := []core.WikiNode{ {NodeToken: "nt-mn", ObjToken: "obj-mn", ObjType: "mindnote", Title: "Brain Map"}, {NodeToken: "nt-sl", ObjToken: "obj-sl", ObjType: "slides", Title: "Presentation"}, } ts, cfg := fakeFeishu(nodes) defer ts.Close() c := NewConnector(core.RegionFeishu) items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) } // Both should be core.Skipped (nil returned by fetchNodeContent) if len(items) != 0 { t.Errorf("expected 0 items (mindnote+slides core.Skipped), got %d", len(items)) } } func TestFetchAll_MixedTypes(t *testing.T) { nodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", NodeEditTime: "1711468800"}, {NodeToken: "nt2", ObjToken: "obj2", ObjType: "sheet", Title: "Sheet", NodeEditTime: "1711468800"}, {NodeToken: "nt3", ObjToken: "obj3", ObjType: "file", Title: "report.pdf", NodeEditTime: "1711468800"}, {NodeToken: "nt4", ObjToken: "obj4", ObjType: "mindnote", Title: "Mind", NodeEditTime: "1711468800"}, {NodeToken: "nt5", ObjToken: "obj5", ObjType: "slides", Title: "Slides", NodeEditTime: "1711468800"}, {NodeToken: "nt6", ObjToken: "obj6", ObjType: "bitable", Title: "Table", NodeEditTime: "1711468800"}, } ts, cfg := fakeFeishu(nodes) defer ts.Close() c := NewConnector(core.RegionFeishu) items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) } // docx + sheet + file + bitable = 4 items; mindnote + slides = core.Skipped if len(items) != 4 { t.Errorf("expected 4 items, got %d", len(items)) for i, it := range items { t.Logf(" item[%d]: %s (obj_type=%s)", i, it.Title, it.Metadata["obj_type"]) } } } func TestFetchAll_LogsSummaryWithSkipBreakdown(t *testing.T) { nodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", NodeEditTime: "1711468800"}, {NodeToken: "nt4", ObjToken: "obj4", ObjType: "mindnote", Title: "Mind", NodeEditTime: "1711468800"}, {NodeToken: "nt5", ObjToken: "obj5", ObjType: "slides", Title: "Slides", NodeEditTime: "1711468800"}, } ts, cfg := fakeFeishu(nodes) defer ts.Close() var buf bytes.Buffer logger.SetOutput(&buf) defer logger.SetOutput(os.Stderr) c := NewConnector(core.RegionFeishu) if _, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}); err != nil { t.Fatalf("FetchAll() error: %v", err) } out := buf.String() for _, want := range []string{ "stream summary", "discovered=3", "fetched=1", "skipped_unsupported=2", "mindnote:1", "slides:1", } { if !strings.Contains(out, want) { t.Errorf("summary log missing %q; got:\n%s", want, out) } } } func TestFetchAll_ChildNodeListErrorReturnsPartialItems(t *testing.T) { nodes := []core.WikiNode{ {NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true}, {NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "file", Title: "Peer.pdf", NodeEditTime: "200"}, } ts, cfg := fakeFeishuWithChildFailure(nodes, "nt-parent") defer ts.Close() conn := NewConnector(core.RegionFeishu) items, err := conn.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll must not abort when one child listing fails: %v", err) } if len(items) != 3 { t.Fatalf("want 3 items (2 fetched + 1 failure placeholder), got %d: %+v", len(items), items) } var parent, peer, placeholder *types.FetchedItem for i := range items { switch { case items[i].ExternalID == "nt-parent" && len(items[i].Content) > 0: parent = &items[i] case items[i].ExternalID == "nt-peer" && len(items[i].Content) > 0: peer = &items[i] case items[i].Metadata["error"] != "": placeholder = &items[i] } } if parent == nil || peer == nil { t.Fatalf("expected both successfully listed nodes to be fetched, parent=%v peer=%v", parent != nil, peer != nil) } if placeholder == nil { t.Fatal("expected a failure placeholder for the child listing error") } if placeholder.ExternalID != "nt-parent" || placeholder.Title != "Parent.pdf" { t.Errorf("placeholder identity wrong: %+v", placeholder) } if placeholder.Metadata["channel"] != types.ChannelFeishu { t.Errorf("placeholder channel = %q", placeholder.Metadata["channel"]) } if placeholder.Metadata["node_token"] != "nt-parent" || placeholder.Metadata["space_id"] != "space1" { t.Errorf("placeholder missing traceability metadata: %+v", placeholder.Metadata) } if !strings.Contains(placeholder.Metadata["error"], "list children of nt-parent") { t.Errorf("placeholder error = %q", placeholder.Metadata["error"]) } } func TestFetchAll_WikiNodeResourceSyncsSelectedSubtree(t *testing.T) { topNodes := []core.WikiNode{ {NodeToken: "nt-root", ObjToken: "obj-root", ObjType: "file", Title: "Root.pdf", NodeEditTime: "100", HasChild: true}, {NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "file", Title: "Peer.pdf", NodeEditTime: "200"}, } childNodes := map[string][]core.WikiNode{ "nt-root": { {NodeToken: "nt-child", ObjToken: "obj-child", ObjType: "file", Title: "Child.pdf", NodeEditTime: "300"}, }, } ts, cfg := fakeFeishuHierarchy(topNodes, childNodes, "") defer ts.Close() resourceID := "space1:nt-root" conn := NewConnector(core.RegionFeishu) items, err := conn.FetchAll(context.Background(), makeConfig(cfg, []string{resourceID}), []string{resourceID}) if err != nil { t.Fatalf("FetchAll() error: %v", err) } if len(items) != 2 { t.Fatalf("want selected root and child only, got %d: %+v", len(items), items) } got := make(map[string]types.FetchedItem) for _, item := range items { got[item.ExternalID] = item } if _, ok := got["nt-root"]; !ok { t.Fatal("selected root node was not fetched") } if _, ok := got["nt-child"]; !ok { t.Fatal("child node was not fetched") } if _, ok := got["nt-peer"]; ok { t.Fatal("peer outside selected subtree must not be fetched") } if got["nt-root"].SourceResourceID != resourceID || got["nt-child"].SourceResourceID != resourceID { t.Fatalf("SourceResourceID should preserve selected resource id: %+v", got) } } // ────────────────────────────────────────────────────────────────────── // FetchIncremental tests // ────────────────────────────────────────────────────────────────────── func TestFetchIncremental_FirstSync(t *testing.T) { nodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"}, {NodeToken: "nt2", ObjToken: "obj2", ObjType: "file", Title: "file.pdf", NodeEditTime: "200"}, } ts, cfg := fakeFeishu(nodes) defer ts.Close() c := NewConnector(core.RegionFeishu) dsConfig := makeConfig(cfg, []string{"space1"}) // First sync with no cursor → all items should be fetched items, cursor, err := c.FetchIncremental(context.Background(), dsConfig, nil) if err != nil { t.Fatalf("FetchIncremental() error: %v", err) } if len(items) != 2 { t.Fatalf("expected 2 items on first sync, got %d", len(items)) } if cursor == nil { t.Fatal("expected non-nil cursor") } if cursor.LastSyncTime.IsZero() { t.Error("cursor.LastSyncTime should not be zero") } } func TestFetchIncremental_NoChanges(t *testing.T) { nodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"}, } ts, cfg := fakeFeishu(nodes) defer ts.Close() c := NewConnector(core.RegionFeishu) dsConfig := makeConfig(cfg, []string{"space1"}) // First sync _, cursor1, err := c.FetchIncremental(context.Background(), dsConfig, nil) if err != nil { t.Fatalf("first sync error: %v", err) } // Second sync with same edit times → should return 0 changed items items, _, err := c.FetchIncremental(context.Background(), dsConfig, cursor1) if err != nil { t.Fatalf("second sync error: %v", err) } if len(items) != 0 { t.Errorf("expected 0 items (no changes), got %d", len(items)) } } func TestFetchIncremental_DetectsDeleted(t *testing.T) { // First sync: 2 nodes allNodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"}, {NodeToken: "nt2", ObjToken: "obj2", ObjType: "docx", Title: "Doc2", NodeEditTime: "200"}, } ts, cfg := fakeFeishu(allNodes) c := NewConnector(core.RegionFeishu) dsConfig := makeConfig(cfg, []string{"space1"}) _, cursor1, err := c.FetchIncremental(context.Background(), dsConfig, nil) if err != nil { t.Fatalf("first sync error: %v", err) } ts.Close() // Second sync: only 1 node remains (nt2 was deleted) ts2, cfg2 := fakeFeishu([]core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"}, }) defer ts2.Close() dsConfig2 := makeConfig(cfg2, []string{"space1"}) items, _, err := c.FetchIncremental(context.Background(), dsConfig2, cursor1) if err != nil { t.Fatalf("second sync error: %v", err) } // Should have 1 deleted item for nt2 var deletedCount int for _, item := range items { if item.IsDeleted { deletedCount++ if item.ExternalID != "nt2" { t.Errorf("expected deleted ExternalID=nt2, got %q", item.ExternalID) } } } if deletedCount != 1 { t.Errorf("expected 1 deleted item, got %d", deletedCount) } } func TestFetchIncremental_NoResourceIDs(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() c := NewConnector(core.RegionFeishu) dsConfig := makeConfig(cfg, nil) // no resource IDs dsConfig.ResourceIDs = nil _, _, err := c.FetchIncremental(context.Background(), dsConfig, nil) if err == nil { t.Fatal("expected error for empty resource IDs") } if !strings.Contains(err.Error(), "no resource IDs") { t.Errorf("unexpected error: %v", err) } } func TestFetchIncremental_ChildNodeListErrorReturnsPartialItemsAndCursor(t *testing.T) { nodes := []core.WikiNode{ {NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true}, {NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "file", Title: "Peer.pdf", NodeEditTime: "200"}, } ts, cfg := fakeFeishuWithChildFailure(nodes, "nt-parent") defer ts.Close() conn := NewConnector(core.RegionFeishu) items, cursor, err := conn.FetchIncremental(context.Background(), makeConfig(cfg, []string{"space1"}), nil) if err != nil { t.Fatalf("FetchIncremental must not abort when one child listing fails: %v", err) } if cursor == nil { t.Fatal("expected cursor to be returned after partial sync") } if len(items) != 3 { t.Fatalf("want 3 items (2 fetched + 1 failure placeholder), got %d: %+v", len(items), items) } var placeholder *types.FetchedItem for i := range items { if items[i].Metadata["error"] != "" { placeholder = &items[i] break } } if placeholder == nil { t.Fatal("expected a failure placeholder for the child listing error") } if placeholder.Metadata["node_token"] != "nt-parent" { t.Errorf("placeholder node_token = %q", placeholder.Metadata["node_token"]) } } func TestFetchIncremental_ChildNodeListErrorDoesNotDeletePreviouslySeenChildren(t *testing.T) { firstNodes := []core.WikiNode{ {NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true}, } firstChildren := map[string][]core.WikiNode{ "nt-parent": { {NodeToken: "nt-child", ObjToken: "obj-child", ObjType: "file", Title: "Child.pdf", NodeEditTime: "150"}, }, } ts, cfg := fakeFeishuHierarchy(firstNodes, firstChildren, "") conn := NewConnector(core.RegionFeishu) firstItems, cursor, err := conn.FetchIncremental(context.Background(), makeConfig(cfg, []string{"space1"}), nil) if err != nil { t.Fatalf("first sync error: %v", err) } if len(firstItems) != 2 { t.Fatalf("want 2 first-sync items, got %d", len(firstItems)) } ts.Close() secondNodes := []core.WikiNode{ {NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true}, } ts2, cfg2 := fakeFeishuHierarchy(secondNodes, nil, "nt-parent") defer ts2.Close() items, nextCursor, err := conn.FetchIncremental(context.Background(), makeConfig(cfg2, []string{"space1"}), cursor) if err != nil { t.Fatalf("partial second sync error: %v", err) } for _, item := range items { if item.IsDeleted { t.Fatalf("partial child listing must not mark previously seen children as deleted: %+v", item) } } cursorBytes, _ := json.Marshal(nextCursor.ConnectorCursor) var restored core.FeishuCursor if err := json.Unmarshal(cursorBytes, &restored); err != nil { t.Fatalf("restore cursor: %v", err) } if restored.SpaceNodeTimes["space1"]["nt-child"] != "150" { t.Fatalf("partial sync should preserve previous child cursor entry, got %+v", restored.SpaceNodeTimes["space1"]) } } // ────────────────────────────────────────────────────────────────────── // core.Client tests // ────────────────────────────────────────────────────────────────────── func TestClientPing(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() client := core.NewClient(cfg) if err := client.Ping(context.Background()); err != nil { t.Fatalf("core.Ping() error: %v", err) } } func TestClientTokenCaching(t *testing.T) { callCount := 0 mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { callCount++ writeJSON(w, core.TokenResponse{ ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: fmt.Sprintf("token-%d", callCount), Expire: 7200, }) }) ts := httptest.NewServer(mux) defer ts.Close() client := core.NewClient(&core.Config{AppID: "a", AppSecret: "b", BaseURL: ts.URL}) // First call: fetches token t1, _ := client.GetTenantAccessToken(context.Background()) // Second call: should use cache t2, _ := client.GetTenantAccessToken(context.Background()) if t1 != t2 { t.Errorf("expected cached token, got different tokens: %q vs %q", t1, t2) } if callCount != 1 { t.Errorf("expected 1 auth call, got %d", callCount) } } func TestClientExportAndDownload(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() client := core.NewClient(cfg) data, fileName, err := client.ExportAndDownload(context.Background(), "obj-token-1", "docx") if err != nil { t.Fatalf("core.ExportAndDownload() error: %v", err) } if string(data) != "fake-docx-content" { t.Errorf("data = %q, want %q", string(data), "fake-docx-content") } if fileName != "exported.docx" { t.Errorf("fileName = %q, want %q", fileName, "exported.docx") } } func TestClientExportAndDownload_UnsupportedType(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() client := core.NewClient(cfg) _, _, err := client.ExportAndDownload(context.Background(), "obj-token-1", "mindnote") if err == nil { t.Fatal("expected error for unsupported type") } if !strings.Contains(err.Error(), "unsupported") { t.Errorf("unexpected error: %v", err) } } func TestClientDownloadDriveFile(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() client := core.NewClient(cfg) data, err := client.DownloadDriveFile(context.Background(), "file-token-abc") if err != nil { t.Fatalf("core.DownloadDriveFile() error: %v", err) } if string(data) != "fake-pdf-binary" { t.Errorf("data = %q, want %q", string(data), "fake-pdf-binary") } } func TestClientListWikiSpaces(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() client := core.NewClient(cfg) spaces, err := client.ListWikiSpaces(context.Background()) if err != nil { t.Fatalf("core.ListWikiSpaces() error: %v", err) } if len(spaces) != 1 { t.Fatalf("expected 1 space, got %d", len(spaces)) } if spaces[0].SpaceID != "space1" { t.Errorf("SpaceID = %q", spaces[0].SpaceID) } } // ────────────────────────────────────────────────────────────────────── // Type mapping tests // ────────────────────────────────────────────────────────────────────── func TestObjTypeToExportMappings(t *testing.T) { // Verify all exportable types have valid mappings exportable := []string{"docx", "doc", "sheet", "bitable"} for _, ot := range exportable { if _, ok := core.ObjTypeToExportFileExtension[ot]; !ok { t.Errorf("core.ObjTypeToExportFileExtension missing %q", ot) } if _, ok := core.ObjTypeToExportType[ot]; !ok { t.Errorf("core.ObjTypeToExportType missing %q", ot) } } // Verify non-exportable types do NOT have mappings nonExportable := []string{"file", "mindnote", "slides"} for _, ot := range nonExportable { if _, ok := core.ObjTypeToExportFileExtension[ot]; ok { t.Errorf("core.ObjTypeToExportFileExtension should NOT contain %q", ot) } } } func TestExportFileExtToSuffix(t *testing.T) { if core.ExportFileExtToSuffix[core.ExportTypeDocx] != ".docx" { t.Errorf("docx suffix = %q", core.ExportFileExtToSuffix[core.ExportTypeDocx]) } if core.ExportFileExtToSuffix[core.ExportTypeXlsx] != ".xlsx" { t.Errorf("xlsx suffix = %q", core.ExportFileExtToSuffix[core.ExportTypeXlsx]) } if core.ExportFileExtToSuffix[core.ExportTypePDF] != ".pdf" { t.Errorf("pdf suffix = %q", core.ExportFileExtToSuffix[core.ExportTypePDF]) } } // ────────────────────────────────────────────────────────────────────── // docx blocks path: multi-item fan-out + attachment sub-items // ────────────────────────────────────────────────────────────────────── // fakeFeishuWithBlocks returns a test server that serves the auth/spaces/nodes // endpoints plus: // - a blocks endpoint for the given docx document (one text block + one file // block whose attachment token is attToken and name is attName) // - a drive file download for attToken returning attContent // // The export endpoint is intentionally absent; any call to it returns 404 so the // test verifies the blocks-API path, not the export-fallback path. func fakeFeishuWithBlocks(nodes []core.WikiNode, docToken, attToken, attName string, attContent []byte) (*httptest.Server, *core.Config) { mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.TokenResponse{ ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.WikiSpaceListResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.WikiSpaceListData{Items: []core.WikiSpace{{SpaceID: "space1", Name: "Test Space"}}}, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.WikiNodeListResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.WikiNodeListData{Items: nodes}, }) }) // blocks API for the given docx document blocksPath := "/open-apis/docx/v1/documents/" + docToken + "/blocks" mux.HandleFunc(blocksPath, func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.DocxBlocksResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.DocxBlocksData{ Items: []core.DocxBlock{ {BlockID: "b1", BlockType: core.BlockTypePage}, {BlockID: "b2", BlockType: core.BlockTypeText, Text: &core.BlockText{ Elements: []core.TextElement{{TextRun: &core.TextRun{Content: "Hello blocks"}}}, }}, {BlockID: "b3", BlockType: core.BlockTypeFile, File: &core.BlockFileRef{Token: attToken, Name: attName}}, }, }, }) }) // media download for embedded attachment tokens (File/Image blocks use /medias/ endpoint) mux.HandleFunc("/open-apis/drive/v1/medias/", func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/download") { w.Header().Set("Content-Type", "application/octet-stream") w.Write(attContent) return } http.NotFound(w, r) }) ts := httptest.NewServer(mux) return ts, &core.Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} } // TestFetchDocxWithBlocks_MultiItem verifies that a docx node returns a main // Markdown item plus an attachment sub-item when the blocks API succeeds. func TestFetchDocxWithBlocks_MultiItem(t *testing.T) { t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const ( nodeToken = "nt-docx" objToken = "obj-docx" attToken = "ft-att-1" attName = "report.pdf" ) // Attachment content must exceed core.MinAttachmentBytes (2 KiB) to not be filtered. attContent := bytes.Repeat([]byte("x"), core.MinAttachmentBytes+1) nodes := []core.WikiNode{{ NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", Title: "My Blocks Doc", NodeEditTime: "1711468800", }} ts, cfg := fakeFeishuWithBlocks(nodes, objToken, attToken, attName, attContent) defer ts.Close() conn := NewConnector(core.RegionFeishu) items, err := conn.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) } if len(items) != 2 { t.Fatalf("want 2 items (main doc + attachment), got %d: %+v", len(items), items) } main := items[0] if main.ExternalID != nodeToken { t.Errorf("items[0].ExternalID = %q, want %q", main.ExternalID, nodeToken) } if main.ContentType != "text/markdown" { t.Errorf("items[0].ContentType = %q, want text/markdown", main.ContentType) } if !main.ReplacesSubtree { t.Errorf("items[0].ReplacesSubtree = false, want true") } if !strings.Contains(string(main.Content), "Hello blocks") { t.Errorf("items[0].Content missing expected text; got %q", string(main.Content)) } att := items[1] wantAttID := nodeToken + "#file#" + attToken if att.ExternalID != wantAttID { t.Errorf("items[1].ExternalID = %q, want %q", att.ExternalID, wantAttID) } if att.Metadata["attachment"] != "true" { t.Errorf("items[1].Metadata[attachment] = %q, want true", att.Metadata["attachment"]) } if att.Title != attName { t.Errorf("items[1].Title = %q, want %q", att.Title, attName) } } // fakeFeishuWithBlocksAndDownloadStatus is like fakeFeishuWithBlocks but the // drive download endpoint returns the given HTTP status code instead of 200. Use // downloadStatus = http.StatusInternalServerError to exercise the // attachment-download-failure path. func fakeFeishuWithBlocksAndDownloadStatus(nodes []core.WikiNode, docToken, attToken, attName string, downloadStatus int) (*httptest.Server, *core.Config) { mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.TokenResponse{ ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.WikiSpaceListResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.WikiSpaceListData{Items: []core.WikiSpace{{SpaceID: "space1", Name: "Test Space"}}}, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.WikiNodeListResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.WikiNodeListData{Items: nodes}, }) }) // blocks API — one text block + one parseable .pdf file block blocksPath := "/open-apis/docx/v1/documents/" + docToken + "/blocks" mux.HandleFunc(blocksPath, func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.DocxBlocksResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.DocxBlocksData{ Items: []core.DocxBlock{ {BlockID: "b1", BlockType: core.BlockTypePage}, {BlockID: "b2", BlockType: core.BlockTypeText, Text: &core.BlockText{ Elements: []core.TextElement{{TextRun: &core.TextRun{Content: "Hello"}}}, }}, {BlockID: "b3", BlockType: core.BlockTypeFile, File: &core.BlockFileRef{Token: attToken, Name: attName}}, }, }, }) }) // media download endpoint returns the requested status (e.g. 500) mux.HandleFunc("/open-apis/drive/v1/medias/", func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/download") { w.WriteHeader(downloadStatus) _, _ = w.Write([]byte(`{"code":99,"msg":"server error"}`)) return } http.NotFound(w, r) }) ts := httptest.NewServer(mux) return ts, &core.Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} } // TestFetchDocxWithBlocks_AttachmentDownloadFailure verifies that when the blocks // API succeeds but a single attachment download fails, the document body is still // ingested (no error, so the node is not pinned into permanent retry), the failure // is surfaced as a visible per-item error item (not just a server log), and the // subtree sweep is suppressed so a transient failure never deletes the good prior // copy of the attachment. One bad attachment must not block the whole document. func TestFetchDocxWithBlocks_AttachmentDownloadFailure(t *testing.T) { t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const ( nodeToken = "nt-docx-fail" objToken = "obj-docx-fail" attToken = "ft-att-bad" attName = "slides.pdf" ) nodes := []core.WikiNode{{ NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", Title: "Doc With Bad Attachment", NodeEditTime: "1711468800", }} ts, cfg := fakeFeishuWithBlocksAndDownloadStatus(nodes, objToken, attToken, attName, http.StatusInternalServerError) defer ts.Close() conn := NewConnector(core.RegionFeishu) ctx := context.Background() client := core.NewClient(cfg) baseMeta := map[string]string{ "obj_token": objToken, "obj_type": "docx", "node_token": nodeToken, "space_id": "space1", "channel": types.ChannelFeishu, } items, err := core.FetchDocxWithBlocks(ctx, client, core.DocxFetchInput{ DocToken: nodeToken, ObjToken: objToken, Title: "Doc With Bad Attachment", URL: conn.region.WikiURL(nodeToken), ResourceID: "space1:nt-docx-fail", EditTime: core.ParseFeishuTimestamp("1711468800"), BaseMeta: baseMeta, MultimodalEnabled: true, }) if err != nil { t.Fatalf("a failed attachment must not fail the whole node, got error: %v", err) } // Main Markdown item + a visible error sub-item for the failed attachment. if len(items) != 2 { t.Fatalf("want 2 items (main doc + attachment error item), got %d: %+v", len(items), items) } var main, errItem *types.FetchedItem for _, it := range items { if it.ExternalID == nodeToken { main = it } else { errItem = it } } if main == nil { t.Fatal("main doc item missing") } if main.ContentType != "text/markdown" { t.Errorf("main ContentType = %q, want text/markdown", main.ContentType) } // The subtree is still reconciled (ReplacesSubtree stays true), but the failed // attachment must be listed in SubtreeKeep so the sweep preserves its good // previously-synced copy instead of deleting it with nothing to replace it. if !main.ReplacesSubtree { t.Error("main.ReplacesSubtree must stay true; the failed child is preserved via SubtreeKeep, not by suppressing the whole sweep") } failedChildID := nodeToken + "#file#" + attToken if !slices.Contains(main.SubtreeKeep, failedChildID) { t.Errorf("SubtreeKeep must contain the failed attachment %q so its prior copy is preserved, got %+v", failedChildID, main.SubtreeKeep) } if errItem == nil { t.Fatal("attachment error item missing") } if len(errItem.Content) != 0 { t.Errorf("attachment error item must carry no content, got %d bytes", len(errItem.Content)) } if errItem.Metadata["error"] == "" { t.Error("attachment error item must carry error metadata for UI visibility") } } // TestFetchDocxWithBlocks_EmbeddedImage verifies that embedded images are emitted // as standalone sub-items (whose bytes flow into the VLM OCR pipeline) only when // the KB has multimodal enabled, but their external_id is ALWAYS kept in // SubtreeKeep so toggling VLM off later does not sweep previously OCR'd images. func TestFetchDocxWithBlocks_EmbeddedImage(t *testing.T) { t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const ( nodeToken = "nt-docx-img" objToken = "obj-docx-img" imgToken = "media-img-1" ) // A valid PNG signature + padding so http.DetectContentType returns image/png // and the bytes exceed core.MinAttachmentBytes. pngBytes := append([]byte("\x89PNG\r\n\x1a\n"), bytes.Repeat([]byte("x"), core.MinAttachmentBytes)...) mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.TokenResponse{ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200}) }) blocksPath := "/open-apis/docx/v1/documents/" + objToken + "/blocks" mux.HandleFunc(blocksPath, func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.DocxBlocksResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.DocxBlocksData{ Items: []core.DocxBlock{ {BlockID: "b1", BlockType: core.BlockTypePage}, {BlockID: "b2", BlockType: core.BlockTypeText, Text: &core.BlockText{ Elements: []core.TextElement{{TextRun: &core.TextRun{Content: "Hello"}}}, }}, {BlockID: "b3", BlockType: core.BlockTypeImage, Image: &core.BlockTokenRef{Token: imgToken}}, }, }, }) }) mux.HandleFunc("/open-apis/drive/v1/medias/", func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/download") { _, _ = w.Write(pngBytes) return } http.NotFound(w, r) }) ts := httptest.NewServer(mux) defer ts.Close() cfg := &core.Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} conn := NewConnector(core.RegionFeishu) ctx := context.Background() client := core.NewClient(cfg) node := core.WikiNode{NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", Title: "Doc With Image", NodeEditTime: "1711468800"} baseMeta := map[string]string{"node_token": nodeToken, "channel": types.ChannelFeishu} imgChildID := nodeToken + "#image#" + imgToken // multimodal ON → image emitted as a sub-item and kept. items, err := core.FetchDocxWithBlocks(ctx, client, core.DocxFetchInput{ DocToken: node.NodeToken, ObjToken: node.ObjToken, Title: node.Title, URL: conn.region.WikiURL(node.NodeToken), ResourceID: "space1:" + nodeToken, EditTime: core.ParseFeishuTimestamp("1711468800"), BaseMeta: baseMeta, MultimodalEnabled: true, }) if err != nil { t.Fatalf("core.FetchDocxWithBlocks (multimodal on): %v", err) } var main, img *types.FetchedItem for _, it := range items { switch it.ExternalID { case nodeToken: main = it case imgChildID: img = it } } if main == nil { t.Fatal("main doc item missing") } if img == nil { t.Fatalf("embedded image sub-item missing; got %+v", items) } if !strings.HasSuffix(img.FileName, ".png") { t.Errorf("image FileName = %q, want a .png extension", img.FileName) } if img.Metadata["embedded_image"] != "true" { t.Errorf("image Metadata[embedded_image] = %q, want true", img.Metadata["embedded_image"]) } if img.Metadata["attachment"] == "true" { t.Error("an embedded image must not be marked as a file attachment") } if len(img.Content) != len(pngBytes) { t.Errorf("image Content = %d bytes, want %d", len(img.Content), len(pngBytes)) } if !slices.Contains(main.SubtreeKeep, imgChildID) { t.Errorf("SubtreeKeep must contain image %q, got %+v", imgChildID, main.SubtreeKeep) } // multimodal OFF → no image sub-item, but the id is still kept (not swept). itemsOff, err := core.FetchDocxWithBlocks(ctx, client, core.DocxFetchInput{ DocToken: node.NodeToken, ObjToken: node.ObjToken, Title: node.Title, URL: conn.region.WikiURL(node.NodeToken), ResourceID: "space1:" + nodeToken, EditTime: core.ParseFeishuTimestamp("1711468800"), BaseMeta: baseMeta, MultimodalEnabled: false, }) if err != nil { t.Fatalf("core.FetchDocxWithBlocks (multimodal off): %v", err) } var mainOff *types.FetchedItem for _, it := range itemsOff { if it.ExternalID == imgChildID { t.Errorf("multimodal off must NOT Emit an image sub-item, got %+v", it) } if it.ExternalID == nodeToken { mainOff = it } } if mainOff == nil { t.Fatal("main doc item missing (multimodal off)") } if !slices.Contains(mainOff.SubtreeKeep, imgChildID) { t.Errorf("SubtreeKeep must contain image %q even when multimodal off (so it isn't swept), got %+v", imgChildID, mainOff.SubtreeKeep) } } // TestFetchDocxWithBlocks_ImageDownloadFailure verifies that a failed image // download (a genuine core.Fetch failure — revoked token, permission gap, transient // error) surfaces a visible error sub-item exactly like a failed attachment // download, rather than being silently dropped to a server log. The image is // still kept in SubtreeKeep so any prior OCR'd copy is preserved. func TestFetchDocxWithBlocks_ImageDownloadFailure(t *testing.T) { t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const ( nodeToken = "nt-docx-img-fail" objToken = "obj-docx-img-fail" imgToken = "media-img-bad" ) mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.TokenResponse{ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200}) }) blocksPath := "/open-apis/docx/v1/documents/" + objToken + "/blocks" mux.HandleFunc(blocksPath, func(w http.ResponseWriter, r *http.Request) { writeJSON(w, core.DocxBlocksResponse{ ApiResponse: core.ApiResponse{Code: 0}, Data: core.DocxBlocksData{ Items: []core.DocxBlock{ {BlockID: "b1", BlockType: core.BlockTypePage}, {BlockID: "b2", BlockType: core.BlockTypeImage, Image: &core.BlockTokenRef{Token: imgToken}}, }, }, }) }) // Media download always fails. mux.HandleFunc("/open-apis/drive/v1/medias/", func(w http.ResponseWriter, r *http.Request) { http.Error(w, "boom", http.StatusInternalServerError) }) ts := httptest.NewServer(mux) defer ts.Close() cfg := &core.Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} conn := NewConnector(core.RegionFeishu) ctx := context.Background() client := core.NewClient(cfg) node := core.WikiNode{NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", Title: "Doc With Bad Image", NodeEditTime: "1711468800"} baseMeta := map[string]string{"node_token": nodeToken, "channel": types.ChannelFeishu} imgChildID := nodeToken + "#image#" + imgToken // multimodal ON → the download is attempted and fails → a visible error item. items, err := core.FetchDocxWithBlocks(ctx, client, core.DocxFetchInput{ DocToken: node.NodeToken, ObjToken: node.ObjToken, Title: node.Title, URL: conn.region.WikiURL(node.NodeToken), ResourceID: "space1:" + nodeToken, EditTime: core.ParseFeishuTimestamp("1711468800"), BaseMeta: baseMeta, MultimodalEnabled: true, }) if err != nil { t.Fatalf("a failed image download must not fail the whole node, got error: %v", err) } var main, errItem *types.FetchedItem for _, it := range items { switch it.ExternalID { case nodeToken: main = it case imgChildID: errItem = it } } if main == nil { t.Fatal("main doc item missing") } if errItem == nil { t.Fatalf("failed image download must Emit a visible error sub-item, got %+v", items) } if len(errItem.Content) != 0 { t.Errorf("image error item must carry no content, got %d bytes", len(errItem.Content)) } if errItem.Metadata["error"] == "" { t.Error("image error item must carry error metadata for UI visibility") } if errItem.Metadata["embedded_image"] != "true" { t.Errorf("image error item must stay tagged embedded_image=true, got %q", errItem.Metadata["embedded_image"]) } if !slices.Contains(main.SubtreeKeep, imgChildID) { t.Errorf("SubtreeKeep must contain the failed image %q so its prior copy is preserved, got %+v", imgChildID, main.SubtreeKeep) } // multimodal OFF → the download is never attempted, so no error item, but the // id is still kept (not swept). itemsOff, err := core.FetchDocxWithBlocks(ctx, client, core.DocxFetchInput{ DocToken: node.NodeToken, ObjToken: node.ObjToken, Title: node.Title, URL: conn.region.WikiURL(node.NodeToken), ResourceID: "space1:" + nodeToken, EditTime: core.ParseFeishuTimestamp("1711468800"), BaseMeta: baseMeta, MultimodalEnabled: false, }) if err != nil { t.Fatalf("core.FetchDocxWithBlocks (multimodal off): %v", err) } for _, it := range itemsOff { if it.ExternalID == imgChildID { t.Errorf("multimodal off must NOT attempt the image download or Emit an item, got %+v", it) } } } // TestFetchDocxWithBlocks_NonWhitelistedExtNotPromoted verifies that a file block // whose extension is not in the parseable-attachment whitelist (e.g. .png) is NOT // promoted to a sub-item, but its inline reference IS present in the main document. func TestFetchDocxWithBlocks_NonWhitelistedExtNotPromoted(t *testing.T) { t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const ( nodeToken = "nt-docx-png" objToken = "obj-docx-png" attToken = "ft-icon" attName = "icon.png" ) // Content size well above core.MinAttachmentBytes — the filter must be extension, not size. attContent := bytes.Repeat([]byte("x"), core.MinAttachmentBytes+100) nodes := []core.WikiNode{{ NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", Title: "Doc With PNG", NodeEditTime: "1711468800", }} ts, cfg := fakeFeishuWithBlocks(nodes, objToken, attToken, attName, attContent) defer ts.Close() conn := NewConnector(core.RegionFeishu) items, err := conn.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) } // Only the main doc — no sub-item for a non-parseable extension. if len(items) != 1 { t.Fatalf("want 1 item (main doc only, .png not promoted), got %d: %+v", len(items), items) } mainContent := string(items[0].Content) if !strings.Contains(mainContent, "📎 附件:icon.png") { t.Errorf("main doc missing inline reference for icon.png; got:\n%s", mainContent) } } // TestFetchDocxWithBlocks_WhitelistedTinyAttachmentNotPromoted verifies that a // whitelisted-extension file block whose download is smaller than core.MinAttachmentBytes // is NOT promoted to a sub-item, but its inline reference IS present in the main doc. func TestFetchDocxWithBlocks_WhitelistedTinyAttachmentNotPromoted(t *testing.T) { t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const ( nodeToken = "nt-docx-tiny" objToken = "obj-docx-tiny" attToken = "ft-tiny-pdf" attName = "tiny.pdf" ) // Content is well below core.MinAttachmentBytes (100 bytes vs 2048 threshold). attContent := bytes.Repeat([]byte("x"), 100) nodes := []core.WikiNode{{ NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", Title: "Doc With Tiny PDF", NodeEditTime: "1711468800", }} ts, cfg := fakeFeishuWithBlocks(nodes, objToken, attToken, attName, attContent) defer ts.Close() conn := NewConnector(core.RegionFeishu) items, err := conn.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) } // Only the main doc — no sub-item for a file below the size threshold. if len(items) != 1 { t.Fatalf("want 1 item (main doc only, tiny .pdf not promoted), got %d: %+v", len(items), items) } mainContent := string(items[0].Content) if !strings.Contains(mainContent, "📎 附件:tiny.pdf") { t.Errorf("main doc missing inline reference for tiny.pdf; got:\n%s", mainContent) } } // ────────────────────────────────────────────────────────────────────── // Feishu cursor serialization // ────────────────────────────────────────────────────────────────────── func TestFeishuCursorRoundTrip(t *testing.T) { original := core.FeishuCursor{ LastSyncTime: time.Date(2026, 3, 26, 12, 0, 0, 0, time.UTC), SpaceNodeTimes: map[string]map[string]string{ "space1": { "nt1": "100", "nt2": "200", }, }, } // Serialize to map[string]interface{} (as stored in SyncCursor.ConnectorCursor) data, err := json.Marshal(original) if err != nil { t.Fatalf("marshal error: %v", err) } var cursorMap map[string]interface{} if err := json.Unmarshal(data, &cursorMap); err != nil { t.Fatalf("unmarshal to map error: %v", err) } // Deserialize back data2, _ := json.Marshal(cursorMap) var restored core.FeishuCursor if err := json.Unmarshal(data2, &restored); err != nil { t.Fatalf("restore error: %v", err) } if restored.SpaceNodeTimes["space1"]["nt1"] != "100" { t.Errorf("restored nt1 = %q, want 100", restored.SpaceNodeTimes["space1"]["nt1"]) } if restored.SpaceNodeTimes["space1"]["nt2"] != "200" { t.Errorf("restored nt2 = %q, want 200", restored.SpaceNodeTimes["space1"]["nt2"]) } } // TestSupportedImageExt covers every arm of the sniff table: the png/jpg/gif // formats WeKnora accepts (aligned with isValidFileType's image set), and the // unsupported case which must return ok=false while still surfacing the detected // content type so the caller can log it without re-sniffing. func TestSupportedImageExt(t *testing.T) { cases := []struct { name string data []byte wantExt string wantCT string wantOK bool }{ {"png", []byte("\x89PNG\r\n\x1a\nrest"), ".png", "image/png", true}, {"jpeg", []byte("\xFF\xD8\xFF\xE0\x00\x10JFIF"), ".jpg", "image/jpeg", true}, {"gif", []byte("GIF89a\x01\x00\x01\x00"), ".gif", "image/gif", true}, {"webp_unsupported", append([]byte("RIFF\x00\x00\x00\x00WEBPVP8 "), make([]byte, 8)...), "", "image/webp", false}, {"text_unsupported", []byte("just some plain text, not an image at all"), "", "text/plain; charset=utf-8", false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { ext, ct, ok := core.SupportedImageExt(tc.data) if ok != tc.wantOK { t.Fatalf("ok = %v, want %v", ok, tc.wantOK) } if ext != tc.wantExt { t.Errorf("ext = %q, want %q", ext, tc.wantExt) } if ct != tc.wantCT { t.Errorf("contentType = %q, want %q (must be returned even when !ok)", ct, tc.wantCT) } }) } }