WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes
Anydoc / anydoc parser engine (push) Canceled after 0s
App / Format, vet, test, and build (push) Canceled after 0s
CLI / test (macos-latest / go 1.26) (push) Canceled after 0s
CLI / test (ubuntu-latest / go 1.26) (push) Canceled after 0s
CLI / test (windows-latest / go 1.26) (push) Canceled after 0s
Build and Push Docker Image / build-ui (push) Canceled after 0s
Build and Push Docker Image / build-docreader (push) Canceled after 0s
Build and Push Docker Image / build-sandbox (push) Canceled after 0s
Build and Push Docker Image / build-app (amd64, linux/amd64, ubuntu-latest) (push) Canceled after 0s
Build and Push Docker Image / build-app (arm64, linux/arm64, ubuntu-24.04-arm) (push) Canceled after 0s
DocReader / Python tests and Go client tests (push) Canceled after 0s
Build & Publish (dsh-weknora) / Typecheck, test, pack (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (pinned) (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (latest, advisory) (push) Canceled after 0s
Frontend / Test, type-check, and build (push) Canceled after 0s
Go Lint Cache / warm lint cache (.) (push) Canceled after 0s
Go Lint Cache / warm lint cache (cli) (push) Canceled after 0s
Go Lint Cache / warm lint cache (client) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.10) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.11) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.12) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.13) (push) Canceled after 0s
Build and Push Docker Image / merge (push) Canceled after 0s
Build & Publish (dsh-weknora) / Publish to npm (push) Canceled after 0s
Build & Publish (mcp-server) / Build sdist + wheel (push) Canceled after 0s
Build & Publish (mcp-server) / Publish to PyPI (push) Canceled after 0s
Anydoc / anydoc parser engine (push) Canceled after 0s
App / Format, vet, test, and build (push) Canceled after 0s
CLI / test (macos-latest / go 1.26) (push) Canceled after 0s
CLI / test (ubuntu-latest / go 1.26) (push) Canceled after 0s
CLI / test (windows-latest / go 1.26) (push) Canceled after 0s
Build and Push Docker Image / build-ui (push) Canceled after 0s
Build and Push Docker Image / build-docreader (push) Canceled after 0s
Build and Push Docker Image / build-sandbox (push) Canceled after 0s
Build and Push Docker Image / build-app (amd64, linux/amd64, ubuntu-latest) (push) Canceled after 0s
Build and Push Docker Image / build-app (arm64, linux/arm64, ubuntu-24.04-arm) (push) Canceled after 0s
DocReader / Python tests and Go client tests (push) Canceled after 0s
Build & Publish (dsh-weknora) / Typecheck, test, pack (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (pinned) (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (latest, advisory) (push) Canceled after 0s
Frontend / Test, type-check, and build (push) Canceled after 0s
Go Lint Cache / warm lint cache (.) (push) Canceled after 0s
Go Lint Cache / warm lint cache (cli) (push) Canceled after 0s
Go Lint Cache / warm lint cache (client) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.10) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.11) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.12) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.13) (push) Canceled after 0s
Build and Push Docker Image / merge (push) Canceled after 0s
Build & Publish (dsh-weknora) / Publish to npm (push) Canceled after 0s
Build & Publish (mcp-server) / Build sdist + wheel (push) Canceled after 0s
Build & Publish (mcp-server) / Publish to PyPI (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1,465 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
// Feishu docx block_type integer enum (subset this connector handles).
|
||||
// https://open.feishu.cn/document/server-docs/docs/docs/docx-v1/docx-structure
|
||||
const (
|
||||
BlockTypePage = 1
|
||||
BlockTypeText = 2
|
||||
BlockTypeHeading1 = 3
|
||||
blockTypeHeading9 = 11
|
||||
BlockTypeBullet = 12
|
||||
BlockTypeOrdered = 13
|
||||
BlockTypeCode = 14
|
||||
BlockTypeQuote = 15
|
||||
BlockTypeTodo = 17
|
||||
BlockTypeBitable = 18
|
||||
BlockTypeCallout = 19
|
||||
BlockTypeDivider = 22
|
||||
BlockTypeFile = 23
|
||||
BlockTypeImage = 27
|
||||
BlockTypeSheet = 30
|
||||
BlockTypeTable = 31
|
||||
BlockTypeTableCell = 32
|
||||
)
|
||||
|
||||
// maxDocumentBlocks caps how many blocks a single document contributes, guarding
|
||||
// against pathological/adversarial documents. Far above any real Feishu doc.
|
||||
const maxDocumentBlocks = 50000
|
||||
|
||||
// TextElement is one inline run inside a text-bearing block.
|
||||
// TextRun is the text_run payload of a TextElement.
|
||||
type TextRun struct {
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
type TextElement struct {
|
||||
TextRun *TextRun `json:"text_run"`
|
||||
}
|
||||
|
||||
// BlockText is the shared shape of text-bearing blocks (text, headingN, bullet…).
|
||||
type BlockText struct {
|
||||
Elements []TextElement `json:"elements"`
|
||||
}
|
||||
|
||||
// BlockTokenRef is the shared shape of sheet/bitable/image block payloads
|
||||
// (the JSON tag differs per field; only the token matters).
|
||||
type BlockTokenRef struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
// BlockFileRef is the file block payload: the attachment token plus its name.
|
||||
type BlockFileRef struct {
|
||||
Token string `json:"token"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// BlockTableProperty carries the table grid shape.
|
||||
type BlockTableProperty struct {
|
||||
ColumnSize int `json:"column_size"`
|
||||
}
|
||||
|
||||
// BlockTable is the table block payload: cell block IDs plus grid property.
|
||||
type BlockTable struct {
|
||||
Cells []string `json:"cells"`
|
||||
Property *BlockTableProperty `json:"property"`
|
||||
}
|
||||
|
||||
// DocxBlock is one node in the flat block array returned by the blocks API.
|
||||
type DocxBlock struct {
|
||||
BlockID string `json:"block_id"`
|
||||
ParentID string `json:"parent_id"`
|
||||
BlockType int `json:"block_type"`
|
||||
Children []string `json:"children"`
|
||||
|
||||
Text *BlockText `json:"text"`
|
||||
Heading1 *BlockText `json:"heading1"`
|
||||
Heading2 *BlockText `json:"heading2"`
|
||||
Heading3 *BlockText `json:"heading3"`
|
||||
Heading4 *BlockText `json:"heading4"`
|
||||
Heading5 *BlockText `json:"heading5"`
|
||||
Heading6 *BlockText `json:"heading6"`
|
||||
Heading7 *BlockText `json:"heading7"`
|
||||
Heading8 *BlockText `json:"heading8"`
|
||||
Heading9 *BlockText `json:"heading9"`
|
||||
Bullet *BlockText `json:"bullet"`
|
||||
Ordered *BlockText `json:"ordered"`
|
||||
Code *BlockText `json:"code"`
|
||||
Quote *BlockText `json:"quote"`
|
||||
Todo *BlockText `json:"todo"`
|
||||
Callout *BlockText `json:"callout"`
|
||||
|
||||
Sheet *BlockTokenRef `json:"sheet"`
|
||||
Bitable *BlockTokenRef `json:"bitable"`
|
||||
File *BlockFileRef `json:"file"`
|
||||
Image *BlockTokenRef `json:"image"`
|
||||
|
||||
Table *BlockTable `json:"table"`
|
||||
}
|
||||
|
||||
// DocxBlocksData is the data payload of DocxBlocksResponse.
|
||||
type DocxBlocksData struct {
|
||||
Items []DocxBlock `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
}
|
||||
|
||||
// DocxBlocksResponse is the response for GET .../documents/:id/blocks.
|
||||
type DocxBlocksResponse struct {
|
||||
ApiResponse
|
||||
Data DocxBlocksData `json:"data"`
|
||||
}
|
||||
|
||||
// listDocumentBlocks returns every block of a docx document as a flat,
|
||||
// pre-order array. Paginates at 500 blocks/page. documentID is the obj_token.
|
||||
func (c *Client) listDocumentBlocks(ctx context.Context, documentID string) ([]DocxBlock, error) {
|
||||
var all []DocxBlock
|
||||
pageToken := ""
|
||||
for {
|
||||
path := fmt.Sprintf("/open-apis/docx/v1/documents/%s/blocks?page_size=500&document_revision_id=-1",
|
||||
url.PathEscape(documentID))
|
||||
if pageToken != "" {
|
||||
path += "&page_token=" + url.QueryEscape(pageToken)
|
||||
}
|
||||
var resp DocxBlocksResponse
|
||||
if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return nil, fmt.Errorf("list document blocks: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return nil, fmt.Errorf("list document blocks error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
all = append(all, resp.Data.Items...)
|
||||
if len(resp.Data.Items) == 0 {
|
||||
// Defensive: a malformed page (empty items but has_more=true and a
|
||||
// non-empty page_token) would otherwise loop until the task deadline,
|
||||
// burning API quota. There is nothing more to collect, so stop.
|
||||
break
|
||||
}
|
||||
if len(all) >= maxDocumentBlocks {
|
||||
logger.Warnf(ctx, "[Feishu] document %s exceeded %d blocks; truncating", documentID, maxDocumentBlocks)
|
||||
break
|
||||
}
|
||||
if !resp.Data.HasMore || resp.Data.PageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.Data.PageToken
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// maxTableRows caps how many rows of an embedded sheet/bitable are rendered,
|
||||
// protecting chunking from pathologically large tables. Beyond this the table
|
||||
// is truncated and the caller annotates the omission.
|
||||
const maxTableRows = 500
|
||||
|
||||
// sheetValueRange is the valueRange payload of sheetValuesData.
|
||||
type sheetValueRange struct {
|
||||
Values [][]any `json:"values"`
|
||||
}
|
||||
|
||||
// sheetValuesData is the data payload of sheetValuesResponse.
|
||||
type sheetValuesData struct {
|
||||
ValueRange sheetValueRange `json:"valueRange"`
|
||||
}
|
||||
|
||||
// sheetValuesResponse is the response for sheets-v2 values read.
|
||||
type sheetValuesResponse struct {
|
||||
ApiResponse
|
||||
Data sheetValuesData `json:"data"`
|
||||
}
|
||||
|
||||
// readSheetRange reads the cell values of an embedded spreadsheet block.
|
||||
// embedToken is the block's sheet.token, formatted "spreadsheetToken_sheetId".
|
||||
// Cells are stringified (display value) for RAG text retrieval. Rows are capped
|
||||
// at maxTableRows; truncated is true when the source had more rows than that.
|
||||
func (c *Client) readSheetRange(ctx context.Context, embedToken string) ([][]string, bool, error) {
|
||||
idx := strings.LastIndex(embedToken, "_")
|
||||
if idx < 0 {
|
||||
return nil, false, fmt.Errorf("invalid sheet embed token: %q", embedToken)
|
||||
}
|
||||
spreadsheetToken, sheetID := embedToken[:idx], embedToken[idx+1:]
|
||||
path := fmt.Sprintf("/open-apis/sheets/v2/spreadsheets/%s/values/%s?valueRenderOption=ToString",
|
||||
url.PathEscape(spreadsheetToken), url.PathEscape(sheetID))
|
||||
var resp sheetValuesResponse
|
||||
if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return nil, false, fmt.Errorf("read sheet range: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return nil, false, fmt.Errorf("read sheet range error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
raw, truncated := capRows(resp.Data.ValueRange.Values)
|
||||
return stringifyMatrix(raw), truncated, nil
|
||||
}
|
||||
|
||||
// capRows limits rows to maxTableRows, reporting whether truncation happened. It
|
||||
// is generic so the sheet ([][]any) and bitable ([][]string) read paths share one
|
||||
// truncation rule instead of each inlining their own.
|
||||
func capRows[T any](rows []T) ([]T, bool) {
|
||||
if len(rows) > maxTableRows {
|
||||
return rows[:maxTableRows], true
|
||||
}
|
||||
return rows, false
|
||||
}
|
||||
|
||||
// stringifyMatrix renders arbitrary cell values to strings; nil → "".
|
||||
func stringifyMatrix(in [][]any) [][]string {
|
||||
out := make([][]string, len(in))
|
||||
for i, row := range in {
|
||||
cells := make([]string, len(row))
|
||||
for j, v := range row {
|
||||
cells[j] = cellToString(v)
|
||||
}
|
||||
out[i] = cells
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// cellToString renders a single JSON cell value to a string; nil → "".
|
||||
func cellToString(v any) string {
|
||||
switch t := v.(type) {
|
||||
case nil:
|
||||
return ""
|
||||
case string:
|
||||
return t
|
||||
case float64:
|
||||
return strconv.FormatFloat(t, 'f', -1, 64)
|
||||
case bool:
|
||||
return strconv.FormatBool(t)
|
||||
default:
|
||||
return fmt.Sprintf("%v", t)
|
||||
}
|
||||
}
|
||||
|
||||
// bitableFieldTypeDateTime is the Feishu bitable field type for a date/datetime
|
||||
// column. Its cell value is a bare Unix-millisecond number, so without type-aware
|
||||
// formatting it would render as a 13-digit integer instead of a readable date.
|
||||
// https://open.feishu.cn/document/server-docs/docs/bitable-v1/app-table-field/guide
|
||||
const bitableFieldTypeDateTime = 5
|
||||
|
||||
// bitableColumn describes one bitable field: its name, integer type, and (for a
|
||||
// date column) its date_formatter, which distinguishes a date-only column
|
||||
// ("yyyy/MM/dd") from a datetime one ("yyyy-MM-dd HH:mm").
|
||||
type bitableColumn struct {
|
||||
name string
|
||||
fieldType int
|
||||
dateFormatter string
|
||||
}
|
||||
|
||||
// bitableFieldCell renders one bitable cell. Date columns carry a Unix-millisecond
|
||||
// UTC instant; the calendar date a user sees is that instant in the table's
|
||||
// timezone, so it is formatted in loc (rendering in UTC would shift the date, e.g.
|
||||
// a GMT+8 "2024-04-01" would show as "2024-03-31 16:00:00"). A date-only formatter
|
||||
// yields "2006-01-02"; a datetime formatter adds the "15:04" time. Every other
|
||||
// type falls through to bitableCellToString.
|
||||
func bitableFieldCell(v any, col bitableColumn, loc *time.Location) string {
|
||||
if col.fieldType == bitableFieldTypeDateTime {
|
||||
// Empty date cells arrive as nil (→ fall through to blank); ms==0 (epoch) is
|
||||
// not a real Feishu date value either — render blank, not "1970-01-01" or "0".
|
||||
if ms, ok := v.(float64); ok {
|
||||
if ms == 0 {
|
||||
return ""
|
||||
}
|
||||
t := time.UnixMilli(int64(ms)).In(loc)
|
||||
if dateFormatterHasTime(col.dateFormatter) {
|
||||
return t.Format("2006-01-02 15:04")
|
||||
}
|
||||
return t.Format("2006-01-02")
|
||||
}
|
||||
}
|
||||
return bitableCellToString(v)
|
||||
}
|
||||
|
||||
// dateFormatterHasTime reports whether a Feishu date_formatter includes a time
|
||||
// component. Feishu formatters are Java-style: 'H'/'h' hour, lowercase 'm' minute,
|
||||
// uppercase 'M' month — so a time part is present iff the string carries an hour or
|
||||
// a lowercase-'m' minute token (e.g. "yyyy-MM-dd HH:mm"). An empty formatter
|
||||
// (Feishu's default "yyyy/MM/dd") is date-only.
|
||||
func dateFormatterHasTime(f string) bool {
|
||||
return strings.ContainsAny(f, "Hh") || strings.Contains(f, "m")
|
||||
}
|
||||
|
||||
// bitableCellToString renders a bitable field value — which may be text segments,
|
||||
// a person/link object, or an attachment/multi-select array — to a readable
|
||||
// string for RAG. Scalars delegate to cellToString.
|
||||
func bitableCellToString(v any) string {
|
||||
switch t := v.(type) {
|
||||
case []any:
|
||||
parts := make([]string, 0, len(t))
|
||||
for _, e := range t {
|
||||
if s := bitableCellToString(e); s != "" {
|
||||
parts = append(parts, s)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
case map[string]any:
|
||||
for _, k := range []string{"text", "name", "en_name", "link"} {
|
||||
if s, ok := t[k].(string); ok && s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
default:
|
||||
return cellToString(v)
|
||||
}
|
||||
}
|
||||
|
||||
// bitableFieldProperty carries a field's type-specific settings.
|
||||
type bitableFieldProperty struct {
|
||||
// DateFormatter distinguishes a date-only column from a datetime one.
|
||||
DateFormatter string `json:"date_formatter"`
|
||||
}
|
||||
|
||||
// bitableField is one entry of bitableFieldsData.Items.
|
||||
type bitableField struct {
|
||||
FieldName string `json:"field_name"`
|
||||
Type int `json:"type"`
|
||||
Property *bitableFieldProperty `json:"property"`
|
||||
}
|
||||
|
||||
// bitableFieldsData is the data payload of bitableFieldsResponse.
|
||||
type bitableFieldsData struct {
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
Items []bitableField `json:"items"`
|
||||
}
|
||||
|
||||
type bitableFieldsResponse struct {
|
||||
ApiResponse
|
||||
Data bitableFieldsData `json:"data"`
|
||||
}
|
||||
|
||||
// maxBitableFieldPageSize is the documented per-page cap for the bitable
|
||||
// list-fields endpoint (100). Unlike list-records (max 500), passing a larger
|
||||
// page_size here is rejected, so fields must be fetched 100 at a time and paged.
|
||||
const maxBitableFieldPageSize = 100
|
||||
|
||||
// bitableRecord is one entry of bitableRecordsData.Items.
|
||||
type bitableRecord struct {
|
||||
Fields map[string]any `json:"fields"`
|
||||
}
|
||||
|
||||
// bitableRecordsData is the data payload of bitableRecordsResponse.
|
||||
type bitableRecordsData struct {
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
Items []bitableRecord `json:"items"`
|
||||
}
|
||||
|
||||
type bitableRecordsResponse struct {
|
||||
ApiResponse
|
||||
Data bitableRecordsData `json:"data"`
|
||||
}
|
||||
|
||||
// readBitableRecords reads an embedded bitable block as a table: a header row of
|
||||
// field names followed by one row per record. embedToken is the block's
|
||||
// bitable.token, formatted "appToken_tableId". Record rows are capped at
|
||||
// maxTableRows; truncated is true when the source had more records than that.
|
||||
func (c *Client) readBitableRecords(ctx context.Context, embedToken string) ([][]string, bool, error) {
|
||||
idx := strings.LastIndex(embedToken, "_")
|
||||
if idx < 0 {
|
||||
return nil, false, fmt.Errorf("invalid bitable embed token: %q", embedToken)
|
||||
}
|
||||
appToken, tableID := embedToken[:idx], embedToken[idx+1:]
|
||||
|
||||
var cols []bitableColumn
|
||||
baseFPath := fmt.Sprintf("/open-apis/bitable/v1/apps/%s/tables/%s/fields?page_size=%d",
|
||||
url.PathEscape(appToken), url.PathEscape(tableID), maxBitableFieldPageSize)
|
||||
fieldPageToken := ""
|
||||
for {
|
||||
fpath := baseFPath
|
||||
if fieldPageToken != "" {
|
||||
fpath += "&page_token=" + url.QueryEscape(fieldPageToken)
|
||||
}
|
||||
var fieldsResp bitableFieldsResponse
|
||||
if err := c.DoRequest(ctx, http.MethodGet, fpath, nil, &fieldsResp); err != nil {
|
||||
return nil, false, fmt.Errorf("read bitable fields: %w", err)
|
||||
}
|
||||
if fieldsResp.Code != 0 {
|
||||
return nil, false, fmt.Errorf("read bitable fields error: code=%d msg=%s", fieldsResp.Code, fieldsResp.Msg)
|
||||
}
|
||||
for _, f := range fieldsResp.Data.Items {
|
||||
formatter := ""
|
||||
if f.Property != nil {
|
||||
formatter = f.Property.DateFormatter
|
||||
}
|
||||
cols = append(cols, bitableColumn{name: f.FieldName, fieldType: f.Type, dateFormatter: formatter})
|
||||
}
|
||||
if len(fieldsResp.Data.Items) == 0 {
|
||||
// Defensive: an empty page with has_more=true would loop forever (this
|
||||
// loop has no size cap of its own). Nothing more to read, so stop.
|
||||
break
|
||||
}
|
||||
if !fieldsResp.Data.HasMore || fieldsResp.Data.PageToken == "" {
|
||||
break
|
||||
}
|
||||
fieldPageToken = fieldsResp.Data.PageToken
|
||||
}
|
||||
|
||||
header := make([]string, len(cols))
|
||||
for i, col := range cols {
|
||||
header[i] = col.name
|
||||
}
|
||||
loc := c.tz()
|
||||
|
||||
var dataRows [][]string
|
||||
truncated := false
|
||||
// Use the Search-records endpoint (POST .../records/search). The legacy
|
||||
// GET .../records is officially deprecated ("已不推荐使用,可使用[查询记录]替代").
|
||||
// An empty body queries the table's default view, paginated via page_token
|
||||
// (we page to the end below), so a default view with a filter would omit the
|
||||
// filtered-out records — acceptable for RAG, but not literally "all records".
|
||||
baseRPath := fmt.Sprintf("/open-apis/bitable/v1/apps/%s/tables/%s/records/search?page_size=500",
|
||||
url.PathEscape(appToken), url.PathEscape(tableID))
|
||||
pageToken := ""
|
||||
for {
|
||||
rpath := baseRPath
|
||||
if pageToken != "" {
|
||||
rpath += "&page_token=" + url.QueryEscape(pageToken)
|
||||
}
|
||||
var rec bitableRecordsResponse
|
||||
if err := c.DoRequest(ctx, http.MethodPost, rpath, map[string]any{}, &rec); err != nil {
|
||||
return nil, false, fmt.Errorf("search bitable records: %w", err)
|
||||
}
|
||||
if rec.Code != 0 {
|
||||
return nil, false, fmt.Errorf("read bitable records error: code=%d msg=%s", rec.Code, rec.Msg)
|
||||
}
|
||||
for _, item := range rec.Data.Items {
|
||||
row := make([]string, len(cols))
|
||||
for i, col := range cols {
|
||||
row[i] = bitableFieldCell(item.Fields[col.name], col, loc)
|
||||
}
|
||||
dataRows = append(dataRows, row)
|
||||
}
|
||||
if len(rec.Data.Items) == 0 {
|
||||
// Defensive: an empty page with has_more=true never advances dataRows,
|
||||
// so the maxTableRows cap below would never trip — stop instead of
|
||||
// looping until the task deadline.
|
||||
break
|
||||
}
|
||||
if len(dataRows) >= maxTableRows {
|
||||
if len(dataRows) > maxTableRows || rec.Data.HasMore {
|
||||
truncated = true
|
||||
}
|
||||
break
|
||||
}
|
||||
if !rec.Data.HasMore || rec.Data.PageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = rec.Data.PageToken
|
||||
}
|
||||
dataRows, capped := capRows(dataRows)
|
||||
truncated = truncated || capped
|
||||
rows := append([][]string{header}, dataRows...)
|
||||
return rows, truncated, nil
|
||||
}
|
||||
@@ -0,0 +1,528 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestListDocumentBlocks_Paginates(t *testing.T) {
|
||||
var gotTokens []string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal" {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200})
|
||||
return
|
||||
}
|
||||
gotTokens = append(gotTokens, r.URL.Query().Get("page_token"))
|
||||
if r.URL.Query().Get("page_size") != "500" {
|
||||
t.Errorf("page_size = %q, want 500", r.URL.Query().Get("page_size"))
|
||||
}
|
||||
if r.URL.Query().Get("page_token") == "" {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"items": []map[string]any{{"block_id": "b1", "block_type": 2}, {"block_id": "b2", "block_type": 2}},
|
||||
"has_more": true,
|
||||
"page_token": "p2",
|
||||
}})
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"items": []map[string]any{{"block_id": "b3", "block_type": 2}},
|
||||
"has_more": false,
|
||||
}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()}
|
||||
blocks, err := c.listDocumentBlocks(context.Background(), "doc123")
|
||||
if err != nil {
|
||||
t.Fatalf("listDocumentBlocks: %v", err)
|
||||
}
|
||||
if len(blocks) != 3 {
|
||||
t.Fatalf("got %d blocks, want 3", len(blocks))
|
||||
}
|
||||
if blocks[0].BlockID != "b1" || blocks[2].BlockID != "b3" {
|
||||
t.Errorf("order not preserved: %+v", blocks)
|
||||
}
|
||||
if len(gotTokens) != 2 || gotTokens[1] != "p2" {
|
||||
t.Errorf("page tokens = %v, want ['', 'p2']", gotTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadSheetRange_SplitsTokenAndReadsValues(t *testing.T) {
|
||||
var gotPath string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal" {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200})
|
||||
return
|
||||
}
|
||||
gotPath = r.URL.Path
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"valueRange": map[string]any{"values": [][]any{{"名称", "数量"}, {"苹果", 3}, {"梨", nil}}},
|
||||
}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()}
|
||||
rows, truncated, err := c.readSheetRange(context.Background(), "sht_abc_0")
|
||||
if err != nil {
|
||||
t.Fatalf("readSheetRange: %v", err)
|
||||
}
|
||||
if gotPath != "/open-apis/sheets/v2/spreadsheets/sht_abc/values/0" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if truncated {
|
||||
t.Errorf("did not expect truncation for 3 rows")
|
||||
}
|
||||
if len(rows) != 3 || rows[0][0] != "名称" || rows[1][1] != "3" || rows[2][1] != "" {
|
||||
t.Errorf("rows = %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadSheetRange_TruncatesLargeTable(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal" {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200})
|
||||
return
|
||||
}
|
||||
values := make([][]any, 600)
|
||||
for i := range values {
|
||||
values[i] = []any{i}
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"valueRange": map[string]any{"values": values},
|
||||
}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()}
|
||||
rows, truncated, err := c.readSheetRange(context.Background(), "sht_x_0")
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if !truncated {
|
||||
t.Errorf("expected truncated=true for 600 rows")
|
||||
}
|
||||
if len(rows) != 500 {
|
||||
t.Errorf("rows capped to %d, want 500", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitableCellToString(t *testing.T) {
|
||||
// text-segment array: [{type:text, text:"你好"}] → "你好"
|
||||
textSeg := []any{map[string]any{"type": "text", "text": "你好"}}
|
||||
if got := bitableCellToString(textSeg); got != "你好" {
|
||||
t.Errorf("text segment: got %q, want %q", got, "你好")
|
||||
}
|
||||
|
||||
// person array: [{id:u1, name:"张三"}] → "张三"
|
||||
person := []any{map[string]any{"id": "u1", "name": "张三"}}
|
||||
if got := bitableCellToString(person); got != "张三" {
|
||||
t.Errorf("person: got %q, want %q", got, "张三")
|
||||
}
|
||||
|
||||
// plain number (float64) → "3"
|
||||
if got := bitableCellToString(float64(3)); got != "3" {
|
||||
t.Errorf("float64: got %q, want %q", got, "3")
|
||||
}
|
||||
|
||||
// plain string → passthrough
|
||||
if got := bitableCellToString("abc"); got != "abc" {
|
||||
t.Errorf("string: got %q, want %q", got, "abc")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitableFieldCell_DateColumn(t *testing.T) {
|
||||
gmt8 := resolveLocation("") // default GMT+8
|
||||
dateOnly := bitableColumn{fieldType: bitableFieldTypeDateTime, dateFormatter: "yyyy/MM/dd"}
|
||||
dateTime := bitableColumn{fieldType: bitableFieldTypeDateTime, dateFormatter: "yyyy-MM-dd HH:mm"}
|
||||
number := bitableColumn{fieldType: 2}
|
||||
|
||||
// Feishu stores a date as a UTC instant at the table-timezone midnight. The
|
||||
// official sample 1711900800000 is 2024-04-01 00:00:00 GMT+8 (== 2024-03-31
|
||||
// 16:00:00 UTC); a date-only column must render the GMT+8 calendar date, NOT
|
||||
// the UTC-shifted "2024-03-31 16:00:00" the old code produced.
|
||||
if got := bitableFieldCell(float64(1711900800000), dateOnly, gmt8); got != "2024-04-01" {
|
||||
t.Errorf("date-only cell: got %q, want %q", got, "2024-04-01")
|
||||
}
|
||||
|
||||
// A datetime column keeps the time, rendered in the same timezone.
|
||||
// 1719802800000 == 2024-07-01 03:00 UTC == 2024-07-01 11:00 GMT+8.
|
||||
if got := bitableFieldCell(float64(1719802800000), dateTime, gmt8); got != "2024-07-01 11:00" {
|
||||
t.Errorf("datetime cell: got %q, want %q", got, "2024-07-01 11:00")
|
||||
}
|
||||
|
||||
// A non-date numeric field is left as its plain value, not a date.
|
||||
if got := bitableFieldCell(float64(42), number, gmt8); got != "42" {
|
||||
t.Errorf("number cell: got %q, want %q", got, "42")
|
||||
}
|
||||
|
||||
// An empty date cell (nil, or epoch 0) renders blank, not "1970-01-01".
|
||||
if got := bitableFieldCell(nil, dateOnly, gmt8); got != "" {
|
||||
t.Errorf("empty date cell: got %q, want empty", got)
|
||||
}
|
||||
if got := bitableFieldCell(float64(0), dateOnly, gmt8); got != "" {
|
||||
t.Errorf("epoch-0 date cell: got %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDateFormatterHasTime(t *testing.T) {
|
||||
withTime := []string{"yyyy-MM-dd HH:mm", "HH:mm", "yyyy/MM/dd hh:mm", "MM-dd HH:mm"}
|
||||
dateOnly := []string{"", "yyyy/MM/dd", "MM-dd", "MM/dd/yyyy", "dd/MM/yyyy"}
|
||||
for _, f := range withTime {
|
||||
if !dateFormatterHasTime(f) {
|
||||
t.Errorf("dateFormatterHasTime(%q) = false, want true", f)
|
||||
}
|
||||
}
|
||||
for _, f := range dateOnly {
|
||||
if dateFormatterHasTime(f) {
|
||||
t.Errorf("dateFormatterHasTime(%q) = true, want false", f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveLocation(t *testing.T) {
|
||||
// Empty and unknown names both fall back to a fixed GMT+8, with no dependency
|
||||
// on system tzdata (minimal container images may omit it).
|
||||
for _, name := range []string{"", "Not/AZone"} {
|
||||
ref := time.Date(2024, 4, 1, 0, 0, 0, 0, resolveLocation(name))
|
||||
if _, off := ref.Zone(); off != defaultTimezoneOffsetSeconds {
|
||||
t.Errorf("resolveLocation(%q) offset = %d, want %d", name, off, defaultTimezoneOffsetSeconds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadBitableRecords_DateColumnUsesFormatterAndTimezone proves the date fix
|
||||
// end-to-end: property.date_formatter is parsed from the fields response, a type-5
|
||||
// column is detected, and the ms instant is rendered in the client timezone
|
||||
// (GMT+8) — date-only vs datetime distinguished by the formatter.
|
||||
func TestReadBitableRecords_DateColumnUsesFormatterAndTimezone(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200})
|
||||
case strings.HasSuffix(r.URL.Path, "/fields"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"items": []map[string]any{
|
||||
{"field_name": "截止日期", "type": 5, "property": map[string]any{"date_formatter": "yyyy/MM/dd"}},
|
||||
{"field_name": "提醒时间", "type": 5, "property": map[string]any{"date_formatter": "yyyy-MM-dd HH:mm"}},
|
||||
},
|
||||
}})
|
||||
case strings.HasSuffix(r.URL.Path, "/records/search"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"has_more": false, "items": []map[string]any{
|
||||
{"fields": map[string]any{"截止日期": float64(1711900800000), "提醒时间": float64(1719802800000)}},
|
||||
},
|
||||
}})
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client(), location: resolveLocation("")}
|
||||
rows, _, err := c.readBitableRecords(context.Background(), "bascabc_tblxyz")
|
||||
if err != nil {
|
||||
t.Fatalf("readBitableRecords: %v", err)
|
||||
}
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("rows = %+v, want header + 1 record", rows)
|
||||
}
|
||||
if rows[1][0] != "2024-04-01" {
|
||||
t.Errorf("date-only column = %q, want 2024-04-01 (GMT+8, not UTC-shifted)", rows[1][0])
|
||||
}
|
||||
if rows[1][1] != "2024-07-01 11:00" {
|
||||
t.Errorf("datetime column = %q, want 2024-07-01 11:00 (GMT+8)", rows[1][1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadBitableRecords_SplitsTokenAndBuildsTable(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200})
|
||||
case strings.HasSuffix(r.URL.Path, "/fields"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"items": []map[string]any{{"field_name": "任务"}, {"field_name": "状态"}},
|
||||
}})
|
||||
case strings.HasSuffix(r.URL.Path, "/records/search"):
|
||||
// Must use the current Search-records endpoint (POST), not the
|
||||
// deprecated GET .../records legacy interface.
|
||||
if r.Method != http.MethodPost {
|
||||
t.Errorf("records read method = %s, want POST (search endpoint)", r.Method)
|
||||
}
|
||||
if !strings.Contains(r.URL.Path, "/apps/bascabc/tables/tblxyz/") {
|
||||
t.Errorf("bad path %q", r.URL.Path)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"has_more": false,
|
||||
"items": []map[string]any{
|
||||
{"fields": map[string]any{"任务": "写文档", "状态": "进行中"}},
|
||||
},
|
||||
}})
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()}
|
||||
rows, truncated, err := c.readBitableRecords(context.Background(), "bascabc_tblxyz")
|
||||
if err != nil {
|
||||
t.Fatalf("readBitableRecords: %v", err)
|
||||
}
|
||||
if truncated {
|
||||
t.Errorf("did not expect truncation")
|
||||
}
|
||||
if len(rows) != 2 || rows[0][0] != "任务" || rows[0][1] != "状态" || rows[1][0] != "写文档" || rows[1][1] != "进行中" {
|
||||
t.Errorf("rows = %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadBitableRecords_PaginatesFieldsAtMax100 guards the real-API constraint
|
||||
// that the list-fields endpoint caps page_size at 100 (unlike list-records, which
|
||||
// allows 500). The connector must request page_size=100 and page through fields,
|
||||
// or a table with >100 columns would silently lose its header tail (or the whole
|
||||
// bitable would degrade if Feishu rejects page_size=500). Verified against the
|
||||
// official docs: bitable-v1/app-table-field/list "最大值:100".
|
||||
func TestReadBitableRecords_PaginatesFieldsAtMax100(t *testing.T) {
|
||||
const totalFields = 150 // spans two pages: 100 + 50
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200})
|
||||
case strings.HasSuffix(r.URL.Path, "/fields"):
|
||||
if ps := r.URL.Query().Get("page_size"); ps != "100" {
|
||||
t.Errorf("fields page_size = %q, want 100 (endpoint max)", ps)
|
||||
}
|
||||
// Page 1 (no token): first 100 fields, has_more. Page 2 (token=f2): last 50.
|
||||
var items []map[string]any
|
||||
hasMore := false
|
||||
nextTok := ""
|
||||
if r.URL.Query().Get("page_token") == "" {
|
||||
for i := 0; i < 100; i++ {
|
||||
items = append(items, map[string]any{"field_name": fieldName(i)})
|
||||
}
|
||||
hasMore, nextTok = true, "f2"
|
||||
} else {
|
||||
for i := 100; i < totalFields; i++ {
|
||||
items = append(items, map[string]any{"field_name": fieldName(i)})
|
||||
}
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"items": items, "has_more": hasMore, "page_token": nextTok,
|
||||
}})
|
||||
case strings.HasSuffix(r.URL.Path, "/records/search"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"has_more": false, "items": []map[string]any{},
|
||||
}})
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()}
|
||||
rows, _, err := c.readBitableRecords(context.Background(), "bascabc_tblxyz")
|
||||
if err != nil {
|
||||
t.Fatalf("readBitableRecords: %v", err)
|
||||
}
|
||||
if len(rows) < 1 {
|
||||
t.Fatal("expected at least a header row")
|
||||
}
|
||||
header := rows[0]
|
||||
if len(header) != totalFields {
|
||||
t.Fatalf("header has %d fields, want %d (fields pagination lost columns)", len(header), totalFields)
|
||||
}
|
||||
if header[0] != fieldName(0) || header[totalFields-1] != fieldName(totalFields-1) {
|
||||
t.Errorf("header endpoints wrong: first=%q last=%q", header[0], header[totalFields-1])
|
||||
}
|
||||
}
|
||||
|
||||
func fieldName(i int) string { return "f" + strconv.Itoa(i) }
|
||||
|
||||
// bitableRows builds n bitable record items each carrying a single "col" field.
|
||||
func bitableRows(n int) []map[string]any {
|
||||
items := make([]map[string]any, n)
|
||||
for i := range items {
|
||||
items[i] = map[string]any{"fields": map[string]any{"col": "v" + strconv.Itoa(i)}}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// TestReadBitableRecords_PaginatesAndTruncatesRecords exercises the record-side
|
||||
// pagination advance and the >maxTableRows truncation — the boundary logic that
|
||||
// no prior test reached (every other bitable test returns a single sub-cap page).
|
||||
func TestReadBitableRecords_PaginatesAndTruncatesRecords(t *testing.T) {
|
||||
var secondPageToken string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200})
|
||||
case strings.HasSuffix(r.URL.Path, "/fields"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"items": []map[string]any{{"field_name": "col"}},
|
||||
}})
|
||||
case strings.HasSuffix(r.URL.Path, "/records/search"):
|
||||
if tok := r.URL.Query().Get("page_token"); tok == "" {
|
||||
// Page 1: 300 records, more to come.
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"has_more": true, "page_token": "r2", "items": bitableRows(300),
|
||||
}})
|
||||
} else {
|
||||
secondPageToken = tok
|
||||
// Page 2: 250 more → 550 total, capped to maxTableRows.
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"has_more": false, "items": bitableRows(250),
|
||||
}})
|
||||
}
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()}
|
||||
rows, truncated, err := c.readBitableRecords(context.Background(), "bascabc_tblxyz")
|
||||
if err != nil {
|
||||
t.Fatalf("readBitableRecords: %v", err)
|
||||
}
|
||||
if secondPageToken != "r2" {
|
||||
t.Errorf("second records page must be fetched with page_token=r2, got %q", secondPageToken)
|
||||
}
|
||||
if len(rows) != maxTableRows+1 { // header + maxTableRows capped data rows
|
||||
t.Errorf("rows = %d, want %d (header + %d capped)", len(rows), maxTableRows+1, maxTableRows)
|
||||
}
|
||||
if !truncated {
|
||||
t.Error("a >maxTableRows bitable must report truncated=true")
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadBitableRecords_Exactly500NotTruncated guards the off-by-one at the cap:
|
||||
// exactly maxTableRows records in a single page must NOT be flagged truncated.
|
||||
func TestReadBitableRecords_Exactly500NotTruncated(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200})
|
||||
case strings.HasSuffix(r.URL.Path, "/fields"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"items": []map[string]any{{"field_name": "col"}},
|
||||
}})
|
||||
case strings.HasSuffix(r.URL.Path, "/records/search"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"has_more": false, "items": bitableRows(maxTableRows),
|
||||
}})
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()}
|
||||
rows, truncated, err := c.readBitableRecords(context.Background(), "bascabc_tblxyz")
|
||||
if err != nil {
|
||||
t.Fatalf("readBitableRecords: %v", err)
|
||||
}
|
||||
if truncated {
|
||||
t.Error("exactly maxTableRows records must not be flagged truncated")
|
||||
}
|
||||
if len(rows) != maxTableRows+1 {
|
||||
t.Errorf("rows = %d, want %d (header + %d)", len(rows), maxTableRows+1, maxTableRows)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadBitableRecords_EmptyRecordPageTerminates guards the pagination
|
||||
// zero-progress break: a malformed page (empty items but has_more=true and a
|
||||
// non-empty page_token) must terminate instead of looping until the task
|
||||
// deadline. A short context deadline is a safety net so a regression fails fast
|
||||
// rather than hanging the suite.
|
||||
func TestReadBitableRecords_EmptyRecordPageTerminates(t *testing.T) {
|
||||
recordCalls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200})
|
||||
case strings.HasSuffix(r.URL.Path, "/fields"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"items": []map[string]any{{"field_name": "col"}},
|
||||
}})
|
||||
case strings.HasSuffix(r.URL.Path, "/records/search"):
|
||||
recordCalls++
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"has_more": true, "page_token": "loop", "items": []map[string]any{},
|
||||
}})
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()}
|
||||
rows, _, err := c.readBitableRecords(ctx, "bascabc_tblxyz")
|
||||
if err != nil {
|
||||
t.Fatalf("empty-page pagination must terminate cleanly, got err: %v", err)
|
||||
}
|
||||
if recordCalls != 1 {
|
||||
t.Errorf("records endpoint called %d times, want 1 (loop must break on the empty page)", recordCalls)
|
||||
}
|
||||
if len(rows) != 1 { // header only
|
||||
t.Errorf("rows = %d, want 1 (header only, no records)", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadBitableRecords_EmptyFieldsPageTerminates guards the fields loop, which
|
||||
// has no size cap of its own — an empty page with has_more=true must still break.
|
||||
func TestReadBitableRecords_EmptyFieldsPageTerminates(t *testing.T) {
|
||||
fieldCalls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200})
|
||||
case strings.HasSuffix(r.URL.Path, "/fields"):
|
||||
fieldCalls++
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"has_more": true, "page_token": "loop", "items": []map[string]any{},
|
||||
}})
|
||||
case strings.HasSuffix(r.URL.Path, "/records/search"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"has_more": false, "items": []map[string]any{},
|
||||
}})
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()}
|
||||
if _, _, err := c.readBitableRecords(ctx, "bascabc_tblxyz"); err != nil {
|
||||
t.Fatalf("empty fields page must terminate cleanly, got err: %v", err)
|
||||
}
|
||||
if fieldCalls != 1 {
|
||||
t.Errorf("fields endpoint called %d times, want 1 (loop must break on the empty page)", fieldCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestListDocumentBlocks_EmptyPageTerminates guards the blocks pagination loop.
|
||||
func TestListDocumentBlocks_EmptyPageTerminates(t *testing.T) {
|
||||
blockCalls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200})
|
||||
case strings.HasSuffix(r.URL.Path, "/blocks"):
|
||||
blockCalls++
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{
|
||||
"has_more": true, "page_token": "loop", "items": []map[string]any{},
|
||||
}})
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()}
|
||||
blocks, err := c.listDocumentBlocks(ctx, "doc-1")
|
||||
if err != nil {
|
||||
t.Fatalf("empty blocks page must terminate cleanly, got err: %v", err)
|
||||
}
|
||||
if blockCalls != 1 {
|
||||
t.Errorf("blocks endpoint called %d times, want 1 (loop must break on the empty page)", blockCalls)
|
||||
}
|
||||
if len(blocks) != 0 {
|
||||
t.Errorf("blocks = %d, want 0", len(blocks))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,916 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
// Client wraps the Feishu Open Platform API for document/wiki operations.
|
||||
type Client struct {
|
||||
baseURL string
|
||||
appID string
|
||||
appSecret string
|
||||
|
||||
// location renders bitable date cells in the table's timezone (default GMT+8).
|
||||
location *time.Location
|
||||
|
||||
httpClient *http.Client
|
||||
|
||||
// Token cache (thread-safe)
|
||||
tokenMu sync.Mutex
|
||||
tokenCache string
|
||||
tokenExpAt time.Time
|
||||
}
|
||||
|
||||
type WikiNodeListFailure struct {
|
||||
Node WikiNode
|
||||
Err error
|
||||
}
|
||||
|
||||
type PartialWikiNodeListError struct {
|
||||
Failures []WikiNodeListFailure
|
||||
}
|
||||
|
||||
func (e *PartialWikiNodeListError) Error() string {
|
||||
if e == nil || len(e.Failures) == 0 {
|
||||
return "partial wiki node listing failed"
|
||||
}
|
||||
parts := make([]string, 0, len(e.Failures))
|
||||
for _, failure := range e.Failures {
|
||||
parts = append(parts, failure.Err.Error())
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
// tz returns the client's date-rendering location, defaulting to GMT+8 when a
|
||||
// Client was constructed without one (e.g. in tests that build Client directly).
|
||||
func (c *Client) tz() *time.Location {
|
||||
if c.location != nil {
|
||||
return c.location
|
||||
}
|
||||
return time.FixedZone("GMT+8", defaultTimezoneOffsetSeconds)
|
||||
}
|
||||
|
||||
// NewClient creates a new Feishu API client.
|
||||
func NewClient(config *Config) *Client {
|
||||
return &Client{
|
||||
baseURL: config.GetBaseURL(),
|
||||
appID: config.AppID,
|
||||
appSecret: config.AppSecret,
|
||||
location: resolveLocation(config.Timezone),
|
||||
httpClient: datasource.NewConnectorHTTPClient(30 * time.Second),
|
||||
}
|
||||
}
|
||||
|
||||
// GetTenantAccessToken retrieves (or returns cached) tenant access token.
|
||||
// Feishu tokens expire in 2 hours; we cache with a 5-minute safety margin.
|
||||
func (c *Client) GetTenantAccessToken(ctx context.Context) (string, error) {
|
||||
c.tokenMu.Lock()
|
||||
defer c.tokenMu.Unlock()
|
||||
|
||||
if c.tokenCache != "" && time.Now().Before(c.tokenExpAt) {
|
||||
return c.tokenCache, nil
|
||||
}
|
||||
|
||||
payload, _ := json.Marshal(map[string]string{
|
||||
"app_id": c.appID,
|
||||
"app_secret": c.appSecret,
|
||||
})
|
||||
|
||||
url := c.baseURL + "/open-apis/auth/v3/tenant_access_token/internal"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create token request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("request token: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var result TokenResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return "", fmt.Errorf("decode token response: %w", err)
|
||||
}
|
||||
if result.Code != 0 {
|
||||
return "", fmt.Errorf("feishu auth error: code=%d msg=%s", result.Code, result.Msg)
|
||||
}
|
||||
|
||||
c.tokenCache = result.TenantAccessToken
|
||||
ttl := time.Duration(result.Expire) * time.Second
|
||||
if ttl > 5*time.Minute {
|
||||
ttl -= 5 * time.Minute
|
||||
}
|
||||
c.tokenExpAt = time.Now().Add(ttl)
|
||||
|
||||
prefixLen := 8
|
||||
if len(result.TenantAccessToken) < prefixLen {
|
||||
prefixLen = len(result.TenantAccessToken)
|
||||
}
|
||||
suffixLen := 4
|
||||
if len(result.TenantAccessToken) < suffixLen {
|
||||
suffixLen = len(result.TenantAccessToken)
|
||||
}
|
||||
logger.Infof(ctx, "[Feishu] got tenant_access_token: %s...%s expire=%ds",
|
||||
result.TenantAccessToken[:prefixLen], result.TenantAccessToken[len(result.TenantAccessToken)-suffixLen:], result.Expire)
|
||||
|
||||
return c.tokenCache, nil
|
||||
}
|
||||
|
||||
// Retry policy shared by DoRequest (JSON API calls) and downloadRawBytes (file
|
||||
// downloads): 429 honours Retry-After, 5xx retries once, transport errors back off.
|
||||
const (
|
||||
feishuMaxRetries = 3
|
||||
feishuMax5xxRetries = 1
|
||||
feishuRetry5xxDelay = 2 * time.Second
|
||||
)
|
||||
|
||||
// maxFeishuDownloadBytes bounds a single file download to protect the sync
|
||||
// worker from adversarial or pathological oversized responses.
|
||||
const maxFeishuDownloadBytes = 512 * 1024 * 1024 // 512 MB
|
||||
|
||||
var feishuRetryBackoff = []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second}
|
||||
|
||||
// DoRequest executes an authenticated API request and decodes the JSON response,
|
||||
// retrying transient failures (transport errors, HTTP 429, 5xx). Feishu's drive
|
||||
// export/wiki APIs are aggressively rate limited, and a thousand-document sync
|
||||
// issues tens of thousands of calls; without backoff a single 429 burst used to
|
||||
// fail whole swathes of documents silently. 429 responses honour Retry-After;
|
||||
// 5xx is retried once; other non-2xx statuses fail fast (no point retrying 4xx).
|
||||
func (c *Client) DoRequest(ctx context.Context, method, path string, body interface{}, result interface{}) error {
|
||||
const (
|
||||
maxRetries = feishuMaxRetries
|
||||
max5xxRetries = feishuMax5xxRetries
|
||||
retry5xxDelay = feishuRetry5xxDelay
|
||||
)
|
||||
backoff := feishuRetryBackoff
|
||||
|
||||
token, err := c.GetTenantAccessToken(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var bodyBytes []byte
|
||||
if body != nil {
|
||||
bodyBytes, err = json.Marshal(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal request body: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
url := c.baseURL + path
|
||||
var lastErr error
|
||||
|
||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||
var bodyReader io.Reader
|
||||
if bodyBytes != nil {
|
||||
bodyReader = bytes.NewReader(bodyBytes)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
if attempt == 0 {
|
||||
logger.Infof(ctx, "[Feishu] %s %s", method, path)
|
||||
} else {
|
||||
logger.Infof(ctx, "[Feishu] %s %s (retry %d/%d)", method, path, attempt, maxRetries)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("execute request: %w", err)
|
||||
if attempt < maxRetries {
|
||||
if sErr := sleepCtx(ctx, backoff[attempt]); sErr != nil {
|
||||
return sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
respBody, readErr := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if readErr != nil {
|
||||
lastErr = fmt.Errorf("read response body: %w", readErr)
|
||||
if attempt < maxRetries {
|
||||
if sErr := sleepCtx(ctx, backoff[attempt]); sErr != nil {
|
||||
return sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] %s %s → status=%d bodyLen=%d body=%s",
|
||||
method, path, resp.StatusCode, len(respBody), truncate(string(respBody), 1000))
|
||||
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
wait := parseRetryAfter(resp.Header.Get("Retry-After"), backoff[min(attempt, len(backoff)-1)])
|
||||
lastErr = fmt.Errorf("feishu rate limited: status=429 body=%s", truncate(string(respBody), 500))
|
||||
if attempt < maxRetries {
|
||||
if sErr := sleepCtx(ctx, wait); sErr != nil {
|
||||
return sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 500 && resp.StatusCode < 600 {
|
||||
lastErr = fmt.Errorf("feishu server error: status=%d body=%s", resp.StatusCode, truncate(string(respBody), 500))
|
||||
if attempt < max5xxRetries {
|
||||
if sErr := sleepCtx(ctx, retry5xxDelay); sErr != nil {
|
||||
return sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("feishu api error: status=%d body=%s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
if result != nil {
|
||||
if err := json.Unmarshal(respBody, result); err != nil {
|
||||
return fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// parseRetryAfter interprets a Retry-After header value (seconds) into a wait
|
||||
// duration, coercing 0/negative to a short delay and falling back when absent
|
||||
// or unparseable.
|
||||
func parseRetryAfter(header string, fallback time.Duration) time.Duration {
|
||||
if header == "" {
|
||||
return fallback
|
||||
}
|
||||
secs, err := strconv.ParseFloat(strings.TrimSpace(header), 64)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
if secs <= 0 {
|
||||
return 100 * time.Millisecond
|
||||
}
|
||||
return time.Duration(secs * float64(time.Second))
|
||||
}
|
||||
|
||||
// sleepCtx waits for d or until ctx is cancelled, returning ctx.Err() if the
|
||||
// context ends first so retries abort promptly on task cancellation/timeout.
|
||||
func sleepCtx(ctx context.Context, d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return ctx.Err()
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// truncate truncates a string to maxLen and appends "..." if truncated.
|
||||
func truncate(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
|
||||
// ListWikiSpaces returns all wiki spaces accessible to the app.
|
||||
func (c *Client) ListWikiSpaces(ctx context.Context) ([]WikiSpace, error) {
|
||||
var allSpaces []WikiSpace
|
||||
pageToken := ""
|
||||
|
||||
for {
|
||||
path := "/open-apis/wiki/v2/spaces?page_size=50"
|
||||
if pageToken != "" {
|
||||
path += "&page_token=" + pageToken
|
||||
}
|
||||
|
||||
var resp WikiSpaceListResponse
|
||||
if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return nil, fmt.Errorf("list wiki spaces: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
logger.Errorf(ctx, "[Feishu] ListWikiSpaces error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
return nil, fmt.Errorf("list wiki spaces error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] ListWikiSpaces: got %d spaces, has_more=%v", len(resp.Data.Items), resp.Data.HasMore)
|
||||
for i, s := range resp.Data.Items {
|
||||
logger.Infof(ctx, "[Feishu] space[%d]: id=%s name=%q visibility=%s", i, s.SpaceID, s.Name, s.Visibility)
|
||||
}
|
||||
|
||||
allSpaces = append(allSpaces, resp.Data.Items...)
|
||||
|
||||
if !resp.Data.HasMore || resp.Data.PageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.Data.PageToken
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] ListWikiSpaces: total %d spaces", len(allSpaces))
|
||||
return allSpaces, nil
|
||||
}
|
||||
|
||||
// ListWikiNodes returns all nodes (documents) under a wiki space.
|
||||
// If parentNodeToken is empty, returns top-level nodes.
|
||||
func (c *Client) ListWikiNodes(ctx context.Context, spaceID string, parentNodeToken string) ([]WikiNode, error) {
|
||||
var allNodes []WikiNode
|
||||
pageToken := ""
|
||||
|
||||
for {
|
||||
path := fmt.Sprintf("/open-apis/wiki/v2/spaces/%s/nodes?page_size=50", spaceID)
|
||||
if parentNodeToken != "" {
|
||||
path += "&parent_node_token=" + parentNodeToken
|
||||
}
|
||||
if pageToken != "" {
|
||||
path += "&page_token=" + pageToken
|
||||
}
|
||||
|
||||
var resp WikiNodeListResponse
|
||||
if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return nil, fmt.Errorf("list wiki nodes: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return nil, fmt.Errorf("list wiki nodes error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
|
||||
for _, node := range resp.Data.Items {
|
||||
if parentNodeToken != "" && node.ParentNodeID == "" {
|
||||
node.ParentNodeID = parentNodeToken
|
||||
}
|
||||
if node.SpaceID == "" {
|
||||
node.SpaceID = spaceID
|
||||
}
|
||||
allNodes = append(allNodes, node)
|
||||
}
|
||||
|
||||
if !resp.Data.HasMore || resp.Data.PageToken == "" {
|
||||
break
|
||||
}
|
||||
pageToken = resp.Data.PageToken
|
||||
}
|
||||
|
||||
return allNodes, nil
|
||||
}
|
||||
|
||||
// GetWikiNode returns metadata for a single wiki node.
|
||||
func (c *Client) GetWikiNode(ctx context.Context, spaceID string, nodeToken string) (WikiNode, error) {
|
||||
path := fmt.Sprintf("/open-apis/wiki/v2/spaces/get_node?token=%s", url.QueryEscape(nodeToken))
|
||||
|
||||
var resp WikiNodeInfoResponse
|
||||
if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return WikiNode{}, fmt.Errorf("get wiki node: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return WikiNode{}, fmt.Errorf("get wiki node error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
|
||||
node := resp.Data.Node
|
||||
if node.SpaceID == "" {
|
||||
node.SpaceID = spaceID
|
||||
}
|
||||
return node, nil
|
||||
}
|
||||
|
||||
// listAllWikiNodesRecursive recursively lists all nodes under a wiki space.
|
||||
// It walks the tree depth-first to discover all nested documents.
|
||||
func (c *Client) listAllWikiNodesRecursive(ctx context.Context, spaceID string) ([]WikiNode, error) {
|
||||
// Start with top-level nodes
|
||||
topNodes, err := c.ListWikiNodes(ctx, spaceID, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var allNodes []WikiNode
|
||||
var failures []WikiNodeListFailure
|
||||
var walk func(nodes []WikiNode)
|
||||
|
||||
walk = func(nodes []WikiNode) {
|
||||
for _, node := range nodes {
|
||||
allNodes = append(allNodes, node)
|
||||
|
||||
// Recurse into child nodes if this node has children
|
||||
if node.HasChild {
|
||||
children, err := c.ListWikiNodes(ctx, spaceID, node.NodeToken)
|
||||
if err != nil {
|
||||
wrappedErr := fmt.Errorf("list children of %s: %w", node.NodeToken, err)
|
||||
failures = append(failures, WikiNodeListFailure{
|
||||
Node: node,
|
||||
Err: wrappedErr,
|
||||
})
|
||||
logger.Warnf(ctx, "[Feishu] partial wiki node listing failure: space=%s node=%s err=%v",
|
||||
spaceID, node.NodeToken, err)
|
||||
continue
|
||||
}
|
||||
walk(children)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
walk(topNodes)
|
||||
if len(failures) > 0 {
|
||||
return allNodes, &PartialWikiNodeListError{Failures: failures}
|
||||
}
|
||||
|
||||
return allNodes, nil
|
||||
}
|
||||
|
||||
// ListWikiNodesRecursiveFrom returns a wiki node and all descendants below it.
|
||||
func (c *Client) ListWikiNodesRecursiveFrom(ctx context.Context, spaceID string, nodeToken string) ([]WikiNode, error) {
|
||||
if nodeToken == "" {
|
||||
return c.listAllWikiNodesRecursive(ctx, spaceID)
|
||||
}
|
||||
|
||||
root, err := c.GetWikiNode(ctx, spaceID, nodeToken)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
nodes, err := c.listWikiNodeDescendants(ctx, spaceID, root)
|
||||
if err != nil {
|
||||
return append([]WikiNode{root}, nodes...), err
|
||||
}
|
||||
return append([]WikiNode{root}, nodes...), nil
|
||||
}
|
||||
|
||||
func (c *Client) listWikiNodeDescendants(ctx context.Context, spaceID string, root WikiNode) ([]WikiNode, error) {
|
||||
if !root.HasChild {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
children, err := c.ListWikiNodes(ctx, spaceID, root.NodeToken)
|
||||
if err != nil {
|
||||
wrappedErr := fmt.Errorf("list children of %s: %w", root.NodeToken, err)
|
||||
logger.Warnf(ctx, "[Feishu] partial wiki node listing failure: space=%s node=%s err=%v",
|
||||
spaceID, root.NodeToken, err)
|
||||
return nil, &PartialWikiNodeListError{
|
||||
Failures: []WikiNodeListFailure{{
|
||||
Node: root,
|
||||
Err: wrappedErr,
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
var allNodes []WikiNode
|
||||
var failures []WikiNodeListFailure
|
||||
var walk func(nodes []WikiNode)
|
||||
|
||||
walk = func(nodes []WikiNode) {
|
||||
for _, node := range nodes {
|
||||
allNodes = append(allNodes, node)
|
||||
if !node.HasChild {
|
||||
continue
|
||||
}
|
||||
|
||||
grandChildren, err := c.ListWikiNodes(ctx, spaceID, node.NodeToken)
|
||||
if err != nil {
|
||||
wrappedErr := fmt.Errorf("list children of %s: %w", node.NodeToken, err)
|
||||
failures = append(failures, WikiNodeListFailure{
|
||||
Node: node,
|
||||
Err: wrappedErr,
|
||||
})
|
||||
logger.Warnf(ctx, "[Feishu] partial wiki node listing failure: space=%s node=%s err=%v",
|
||||
spaceID, node.NodeToken, err)
|
||||
continue
|
||||
}
|
||||
walk(grandChildren)
|
||||
}
|
||||
}
|
||||
|
||||
walk(children)
|
||||
if len(failures) > 0 {
|
||||
return allNodes, &PartialWikiNodeListError{Failures: failures}
|
||||
}
|
||||
return allNodes, nil
|
||||
}
|
||||
|
||||
// getDocumentRawContent retrieves the raw text content of a Feishu docx document.
|
||||
// This returns plain text (not rich text / block structure).
|
||||
// Deprecated: prefer ExportAndDownload which preserves formatting.
|
||||
func (c *Client) getDocumentRawContent(ctx context.Context, documentID string) (string, error) {
|
||||
path := fmt.Sprintf("/open-apis/docx/v1/documents/%s/raw_content", documentID)
|
||||
|
||||
var resp docRawContentResponse
|
||||
if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return "", fmt.Errorf("get document raw content: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return "", fmt.Errorf("get document raw content error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
|
||||
return resp.Data.Content, nil
|
||||
}
|
||||
|
||||
// Ping verifies the credentials by attempting to get a tenant access token.
|
||||
func (c *Client) Ping(ctx context.Context) error {
|
||||
_, err := c.GetTenantAccessToken(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Export task API: export docx/sheet/bitable to downloadable files
|
||||
//
|
||||
// Flow:
|
||||
// 1. POST /drive/v1/export_tasks → create export task, get ticket
|
||||
// 2. GET /drive/v1/export_tasks/:ticket → poll until status=0 (success)
|
||||
// 3. GET /drive/v1/export_tasks/file/:ticket/download → download file bytes
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
// createExportTask creates an async export task for a Feishu document.
|
||||
// - token: the obj_token of the document (e.g. docx token, sheet token)
|
||||
// - objType: the Feishu obj_type ("docx", "doc", "sheet", "bitable")
|
||||
// - fileExtension: desired output format ("docx", "xlsx", "pdf")
|
||||
func (c *Client) createExportTask(ctx context.Context, token, objType, fileExtension string) (string, error) {
|
||||
body := map[string]string{
|
||||
"file_extension": fileExtension,
|
||||
"token": token,
|
||||
"type": objType,
|
||||
}
|
||||
|
||||
var resp ExportTaskCreateResponse
|
||||
if err := c.DoRequest(ctx, http.MethodPost, "/open-apis/drive/v1/export_tasks", body, &resp); err != nil {
|
||||
return "", fmt.Errorf("create export task: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return "", fmt.Errorf("create export task error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
|
||||
return resp.Data.Ticket, nil
|
||||
}
|
||||
|
||||
// getExportTaskStatus polls the status of an export task.
|
||||
// Returns (fileToken, fileName, error). fileToken is non-empty only when the job succeeds.
|
||||
// The token parameter is the obj_token of the document being exported (required by the API).
|
||||
func (c *Client) getExportTaskStatus(ctx context.Context, ticket string, token string) (string, string, error) {
|
||||
path := fmt.Sprintf("/open-apis/drive/v1/export_tasks/%s?token=%s", ticket, token)
|
||||
|
||||
var resp ExportTaskStatusResponse
|
||||
if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return "", "", fmt.Errorf("get export task status: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return "", "", fmt.Errorf("get export task status error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
|
||||
r := resp.Data.Result
|
||||
switch r.JobStatus {
|
||||
case 0: // success
|
||||
return r.FileToken, r.FileName, nil
|
||||
case 1, 2: // initializing, processing
|
||||
return "", "", nil // not ready yet
|
||||
default:
|
||||
return "", "", fmt.Errorf("export task failed: status=%d msg=%s", r.JobStatus, r.JobErrorMsg)
|
||||
}
|
||||
}
|
||||
|
||||
// downloadExportFile downloads the exported file by its file_token.
|
||||
// The file_token is returned by getExportTaskStatus when the export job completes.
|
||||
// The file must be downloaded within 10 minutes of export completion.
|
||||
func (c *Client) downloadExportFile(ctx context.Context, fileToken string) ([]byte, error) {
|
||||
path := fmt.Sprintf("/open-apis/drive/v1/export_tasks/file/%s/download", fileToken)
|
||||
return c.downloadRawBytes(ctx, path)
|
||||
}
|
||||
|
||||
// ExportAndDownload is a high-level helper that creates an export task, polls until
|
||||
// completion, and downloads the resulting file. Returns (fileBytes, fileName, error).
|
||||
//
|
||||
// Timeout: 60 seconds. Poll interval: 2 seconds.
|
||||
func (c *Client) ExportAndDownload(ctx context.Context, objToken, objType string) ([]byte, string, error) {
|
||||
// Determine export format
|
||||
fileExt, ok := ObjTypeToExportFileExtension[objType]
|
||||
if !ok {
|
||||
return nil, "", fmt.Errorf("unsupported obj_type for export: %s", objType)
|
||||
}
|
||||
|
||||
exportType, ok := ObjTypeToExportType[objType]
|
||||
if !ok {
|
||||
return nil, "", fmt.Errorf("unsupported obj_type for export: %s", objType)
|
||||
}
|
||||
|
||||
// Step 1: create export task
|
||||
ticket, err := c.createExportTask(ctx, objToken, exportType, fileExt)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
// Step 2: poll until ready (max 60s, every 2s)
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
var fileToken, fileName string
|
||||
|
||||
for time.Now().Before(deadline) {
|
||||
fileToken, fileName, err = c.getExportTaskStatus(ctx, ticket, objToken)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if fileToken != "" {
|
||||
break // export ready
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, "", ctx.Err()
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
if fileToken == "" {
|
||||
return nil, "", fmt.Errorf("export task timed out after 60s (ticket=%s)", ticket)
|
||||
}
|
||||
|
||||
// Step 3: download file using file_token (NOT ticket)
|
||||
data, err := c.downloadExportFile(ctx, fileToken)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
// Build a sensible file name
|
||||
if fileName == "" {
|
||||
fileName = "export" + ExportFileExtToSuffix[fileExt]
|
||||
}
|
||||
|
||||
return data, fileName, nil
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Drive file download: for "file" type wiki nodes (uploaded PDF/Word/etc.)
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
// DownloadDriveFile downloads a file from Feishu Drive by its file token.
|
||||
// Used for wiki nodes with obj_type="file" (user-uploaded PDF, Word, images, etc.).
|
||||
func (c *Client) DownloadDriveFile(ctx context.Context, fileToken string) ([]byte, error) {
|
||||
path := fmt.Sprintf("/open-apis/drive/v1/files/%s/download", fileToken)
|
||||
return c.downloadRawBytes(ctx, path)
|
||||
}
|
||||
|
||||
// downloadMediaFile downloads embedded media (attachments/images referenced by
|
||||
// document File/Image blocks) by its media token. Embedded block media live in a
|
||||
// different token space than standalone Drive files and must use the /medias/
|
||||
// endpoint rather than /files/.
|
||||
func (c *Client) downloadMediaFile(ctx context.Context, fileToken string) ([]byte, error) {
|
||||
path := fmt.Sprintf("/open-apis/drive/v1/medias/%s/download", url.PathEscape(fileToken))
|
||||
return c.downloadRawBytes(ctx, path)
|
||||
}
|
||||
|
||||
// downloadRawBytes performs an authenticated GET and returns the raw response body.
|
||||
func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, error) {
|
||||
token, err := c.GetTenantAccessToken(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
url := c.baseURL + path
|
||||
var lastErr error
|
||||
|
||||
for attempt := 0; attempt <= feishuMaxRetries; attempt++ {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create download request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
if attempt == 0 {
|
||||
logger.Infof(ctx, "[Feishu] download GET %s", path)
|
||||
} else {
|
||||
logger.Infof(ctx, "[Feishu] download GET %s (retry %d/%d)", path, attempt, feishuMaxRetries)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("download request: %w", err)
|
||||
if attempt < feishuMaxRetries {
|
||||
if sErr := sleepCtx(ctx, feishuRetryBackoff[attempt]); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
wait := parseRetryAfter(resp.Header.Get("Retry-After"), feishuRetryBackoff[min(attempt, len(feishuRetryBackoff)-1)])
|
||||
lastErr = fmt.Errorf("download rate limited: status=429 body=%s", truncate(string(body), 500))
|
||||
if attempt < feishuMaxRetries {
|
||||
if sErr := sleepCtx(ctx, wait); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 500 && resp.StatusCode < 600 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
lastErr = fmt.Errorf("download server error: status=%d body=%s", resp.StatusCode, truncate(string(body), 500))
|
||||
if attempt < feishuMax5xxRetries {
|
||||
if sErr := sleepCtx(ctx, feishuRetry5xxDelay); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
logger.Errorf(ctx, "[Feishu] download GET %s → status=%d body=%s", path, resp.StatusCode, truncate(string(body), 500))
|
||||
return nil, fmt.Errorf("download failed: status=%d body=%s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
data, readErr := io.ReadAll(io.LimitReader(resp.Body, maxFeishuDownloadBytes+1))
|
||||
if readErr == nil && int64(len(data)) > maxFeishuDownloadBytes {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("download exceeds max size (%d bytes): %s", maxFeishuDownloadBytes, path)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if readErr != nil {
|
||||
lastErr = fmt.Errorf("read download body: %w", readErr)
|
||||
if attempt < feishuMaxRetries {
|
||||
if sErr := sleepCtx(ctx, feishuRetryBackoff[attempt]); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] download GET %s → OK, %d bytes", path, len(data))
|
||||
return data, nil
|
||||
}
|
||||
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Drive (云盘) file listing: for feishu_drive / lark_drive connectors.
|
||||
// Mirrors the wiki ListWikiNodes / ListWikiNodesRecursiveFrom shape so the
|
||||
// Drive connector's FetchStream mirrors the wiki connector's. See ADR-0001/0002.
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
// listDriveFiles lists files in a Drive folder (non-recursive), one page at a
|
||||
// time. Pass pageToken="" for the first page; the returned nextPageToken is ""
|
||||
// when there are no more pages.
|
||||
//
|
||||
// folderToken == "" is rejected: the root folder is not paginated and does
|
||||
// not return shortcuts (Feishu API limitation), which would silently drop
|
||||
// content and risk an unbounded single response. See ADR-0004.
|
||||
func (c *Client) listDriveFiles(ctx context.Context, folderToken, pageToken string) ([]DriveFile, string, error) {
|
||||
if folderToken == "" {
|
||||
return nil, "", fmt.Errorf("root folder not supported; specify a concrete folder_token (root folder is not paginated and does not return shortcuts)")
|
||||
}
|
||||
|
||||
path := "/open-apis/drive/v1/files?folder_token=" + url.QueryEscape(folderToken)
|
||||
path += "&page_size=200" // max
|
||||
path += "&order_by=EditedTime&direction=DESC"
|
||||
if pageToken != "" {
|
||||
path += "&page_token=" + url.QueryEscape(pageToken)
|
||||
}
|
||||
|
||||
var resp DriveFileListResponse
|
||||
if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return nil, "", fmt.Errorf("list drive files: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return nil, "", fmt.Errorf("list drive files error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[FeishuDrive] listDriveFiles: folder=%s got %d files, has_more=%v",
|
||||
folderToken, len(resp.Data.Files), resp.Data.HasMore)
|
||||
return resp.Data.Files, resp.Data.NextPageToken, nil
|
||||
}
|
||||
|
||||
// GetDriveFolderMeta returns the metadata (name, owner, etc.) of a single Drive
|
||||
// folder. Used to resolve a root folder's human-readable name - the list API
|
||||
// only returns the folder's children, not the folder itself.
|
||||
//
|
||||
// GET /open-apis/drive/explorer/v2/folder/:folderToken/meta
|
||||
func (c *Client) GetDriveFolderMeta(ctx context.Context, folderToken string) (driveFolderMetaResponse, error) {
|
||||
var resp driveFolderMetaResponse
|
||||
if folderToken == "" {
|
||||
return resp, fmt.Errorf("root folder not supported; specify a concrete folder_token")
|
||||
}
|
||||
path := "/open-apis/drive/explorer/v2/folder/" + url.QueryEscape(folderToken) + "/meta"
|
||||
if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return resp, fmt.Errorf("get drive folder meta: %w", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
return resp, fmt.Errorf("get drive folder meta error: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// ListDriveFilesAllPages lists every direct child of a folder across all pages.
|
||||
func (c *Client) ListDriveFilesAllPages(ctx context.Context, folderToken string) ([]DriveFile, error) {
|
||||
var all []DriveFile
|
||||
pageToken := ""
|
||||
for {
|
||||
files, next, err := c.listDriveFiles(ctx, folderToken, pageToken)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all = append(all, files...)
|
||||
if next == "" {
|
||||
break
|
||||
}
|
||||
pageToken = next
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// ListDriveFilesRecursiveFrom walks a Drive folder subtree depth-first,
|
||||
// returning all non-folder files. Mirrors ListWikiNodesRecursiveFrom.
|
||||
//
|
||||
// - folder -> recurse (visited is a pure-defensive cycle guard; Drive folders
|
||||
// have no shortcut concept so cycles are not expected - see glossary).
|
||||
// - shortcut -> expand to its target (target_type is never "folder", verified)
|
||||
// and include the target as a regular file. No extra API call: shortcut_info
|
||||
// is returned by the list API.
|
||||
// - other -> collect.
|
||||
//
|
||||
// Partial failures (a sub-folder listing returns an error) are collected into a
|
||||
// *PartialDriveFileListError and the walk continues, mirroring the wiki
|
||||
// connector's PartialWikiNodeListError semantics.
|
||||
func (c *Client) ListDriveFilesRecursiveFrom(ctx context.Context, folderToken string) ([]DriveFile, error) {
|
||||
visited := make(map[string]bool)
|
||||
var all []DriveFile
|
||||
var failures []DriveFileListFailure
|
||||
|
||||
var walk func(folderToken string)
|
||||
walk = func(folderToken string) {
|
||||
if visited[folderToken] {
|
||||
return
|
||||
}
|
||||
visited[folderToken] = true
|
||||
|
||||
files, err := c.ListDriveFilesAllPages(ctx, folderToken)
|
||||
if err != nil {
|
||||
wrappedErr := fmt.Errorf("list children of %s: %w", folderToken, err)
|
||||
failures = append(failures, DriveFileListFailure{
|
||||
FolderToken: folderToken,
|
||||
Err: wrappedErr,
|
||||
})
|
||||
logger.Warnf(ctx, "[FeishuDrive] partial drive file listing failure: folder=%s err=%v",
|
||||
folderToken, err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
switch f.Type {
|
||||
case "folder":
|
||||
walk(f.Token)
|
||||
case "shortcut":
|
||||
// Expand to target. target_type is never "folder" (verified), so
|
||||
// no recursion here - the target is a regular file.
|
||||
if f.ShortcutInfo != nil && f.ShortcutInfo.TargetToken != "" {
|
||||
expanded := DriveFile{
|
||||
Token: f.ShortcutInfo.TargetToken,
|
||||
Name: f.Name,
|
||||
Type: f.ShortcutInfo.TargetType,
|
||||
ParentToken: f.ParentToken,
|
||||
URL: f.URL,
|
||||
CreatedTime: f.CreatedTime,
|
||||
ModifiedTime: f.ModifiedTime,
|
||||
OwnerID: f.OwnerID,
|
||||
}
|
||||
all = append(all, expanded)
|
||||
}
|
||||
default:
|
||||
all = append(all, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
walk(folderToken)
|
||||
if len(failures) > 0 {
|
||||
return all, &PartialDriveFileListError{Failures: failures}
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// retryTestServer builds a server that always answers the auth-token call and
|
||||
// routes the given target path to h, so tests can drive DoRequest's retry loop.
|
||||
func retryTestServer(target string, h http.HandlerFunc) (*httptest.Server, *Config) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, TokenResponse{
|
||||
ApiResponse: ApiResponse{Code: 0},
|
||||
TenantAccessToken: "fake-token",
|
||||
Expire: 7200,
|
||||
})
|
||||
})
|
||||
mux.HandleFunc(target, h)
|
||||
ts := httptest.NewServer(mux)
|
||||
return ts, &Config{AppID: "a", AppSecret: "b", BaseURL: ts.URL}
|
||||
}
|
||||
|
||||
func TestDoRequest_RetriesOn429ThenSucceeds(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
// "0" is coerced to a short delay inside the client so the test stays fast.
|
||||
w.Header().Set("Retry-After", "0")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
_, _ = io.WriteString(w, `{"code":99991400,"msg":"rate limited"}`)
|
||||
return
|
||||
}
|
||||
writeJSON(w, ApiResponse{Code: 0})
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
var resp ApiResponse
|
||||
if err := c.DoRequest(context.Background(), http.MethodGet, "/target", nil, &resp); err != nil {
|
||||
t.Fatalf("expected success after retry, got %v", err)
|
||||
}
|
||||
if attempts < 2 {
|
||||
t.Errorf("attempts = %d, want >= 2 (should retry after 429)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoRequest_429ExhaustsRetries(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
w.Header().Set("Retry-After", "0")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
_, _ = io.WriteString(w, `{"code":99991400,"msg":"rate limited"}`)
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
err := c.DoRequest(context.Background(), http.MethodGet, "/target", nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error when 429s exceed the retry budget")
|
||||
}
|
||||
if attempts != 4 { // initial + 3 retries
|
||||
t.Errorf("attempts = %d, want 4 (1 + 3 retries)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoRequest_5xxRetriesOnce(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = io.WriteString(w, `{"code":1,"msg":"internal error"}`)
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
c := NewClient(cfg)
|
||||
if err := c.DoRequest(ctx, http.MethodGet, "/target", nil, nil); err == nil {
|
||||
t.Fatal("expected error after 5xx exhaustion")
|
||||
}
|
||||
if attempts != 2 { // initial + 1 retry
|
||||
t.Errorf("attempts = %d, want 2 (5xx retries exactly once)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoRequest_4xxNotRetried(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = io.WriteString(w, `{"code":1,"msg":"bad request"}`)
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
if err := c.DoRequest(context.Background(), http.MethodGet, "/target", nil, nil); err == nil {
|
||||
t.Fatal("expected error on 400")
|
||||
}
|
||||
if attempts != 1 {
|
||||
t.Errorf("attempts = %d, want 1 (non-429/5xx 4xx must not retry)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadRawBytes_RetriesOn429ThenSucceeds(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/dl", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
w.Header().Set("Retry-After", "0")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, _ = w.Write([]byte("payload-bytes"))
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
data, err := c.downloadRawBytes(context.Background(), "/dl")
|
||||
if err != nil {
|
||||
t.Fatalf("expected success after retry, got %v", err)
|
||||
}
|
||||
if string(data) != "payload-bytes" {
|
||||
t.Errorf("data = %q, want %q", string(data), "payload-bytes")
|
||||
}
|
||||
if attempts < 2 {
|
||||
t.Errorf("attempts = %d, want >= 2 (should retry download after 429)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadRawBytes_4xxNotRetried(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/dl", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
if _, err := c.downloadRawBytes(context.Background(), "/dl"); err == nil {
|
||||
t.Fatal("expected error on 403")
|
||||
}
|
||||
if attempts != 1 {
|
||||
t.Errorf("attempts = %d, want 1 (403 must not retry)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRetryAfter(t *testing.T) {
|
||||
fallback := 5 * time.Second
|
||||
tests := []struct {
|
||||
header string
|
||||
want time.Duration
|
||||
}{
|
||||
{"", fallback},
|
||||
{"0", 100 * time.Millisecond},
|
||||
{"-1", 100 * time.Millisecond}, // negative coerced to a short delay
|
||||
{"3", 3 * time.Second},
|
||||
{"abc", fallback}, // unparseable
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := parseRetryAfter(tt.header, fallback); got != tt.want {
|
||||
t.Errorf("parseRetryAfter(%q) = %v, want %v", tt.header, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// feishuFailure classifies a raw connector/API error into a stable i18n code
|
||||
// (+ optional numeric feishu code) and an English fallback message. The frontend
|
||||
// localises by code; the raw status/body/log_id stays in the server logs and
|
||||
// must never appear in the code, codeValue or fallback.
|
||||
func TestFeishuFailure(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
wantCode string
|
||||
wantCodeValue string
|
||||
fallbackHas []string
|
||||
noLeak []string // must appear in neither code nor fallback
|
||||
}{
|
||||
{
|
||||
name: "rate limited",
|
||||
err: errors.New("feishu rate limited: status=429 body={\"code\":99991400,\"msg\":\"too many request\"}"),
|
||||
wantCode: "feishu_rate_limited",
|
||||
fallbackHas: []string{"retry"},
|
||||
noLeak: []string{"body=", "{", "99991400"},
|
||||
},
|
||||
{
|
||||
name: "server 5xx",
|
||||
err: errors.New("feishu server error: status=500 body={\"code\":1663}"),
|
||||
wantCode: "feishu_server_unavailable",
|
||||
noLeak: []string{"body=", "{"},
|
||||
},
|
||||
{
|
||||
name: "export timeout",
|
||||
err: errors.New("export 季度报告 (docx): export task timed out after 60s (ticket=abc)"),
|
||||
wantCode: "feishu_timeout",
|
||||
noLeak: []string{"ticket=", "abc"},
|
||||
},
|
||||
{
|
||||
name: "api error carries the feishu code as a param",
|
||||
err: errors.New("feishu api error: status=500 body={\"code\":1663,\"msg\":\"internal error\",\"Error\":{\"log_id\":\"20260\"}}"),
|
||||
wantCode: "feishu_api_error",
|
||||
wantCodeValue: "1663",
|
||||
noLeak: []string{"log_id", "body=", "{"},
|
||||
},
|
||||
{
|
||||
name: "api error without a code is generic",
|
||||
err: errors.New("feishu api error: status=502 body=bad gateway"),
|
||||
wantCode: "feishu_api_error_generic",
|
||||
},
|
||||
{
|
||||
name: "auth failure is actionable, not a retry",
|
||||
err: errors.New("feishu auth error: code=99991663 msg=Invalid access token"),
|
||||
wantCode: "feishu_auth_or_permission",
|
||||
noLeak: []string{"Invalid access token", "99991663"},
|
||||
},
|
||||
{
|
||||
name: "unknown error falls back to sync_failed",
|
||||
err: errors.New("some totally unexpected failure"),
|
||||
wantCode: "sync_failed",
|
||||
noLeak: []string{"totally unexpected"},
|
||||
},
|
||||
{
|
||||
// "decode"/"encode"/"unicode" all contain the substring "code" but
|
||||
// are not Feishu API errors — they must not be classified as one.
|
||||
name: "decode error is not a feishu api error",
|
||||
err: errors.New("failed to decode response body"),
|
||||
wantCode: "sync_failed",
|
||||
},
|
||||
{
|
||||
// The failed item is retried on the next sync (the cursor is not
|
||||
// advanced), which for a manual-only source is not automatic. The
|
||||
// message must not promise an unconditional automatic retry.
|
||||
name: "transient message promises retry on next sync, not automatic",
|
||||
err: errors.New("feishu rate limited: status=429"),
|
||||
wantCode: "feishu_rate_limited",
|
||||
fallbackHas: []string{"next sync"},
|
||||
noLeak: []string{"automatically"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
code, codeValue, fallback := feishuFailure(tc.err)
|
||||
if code != tc.wantCode {
|
||||
t.Errorf("code = %q, want %q", code, tc.wantCode)
|
||||
}
|
||||
if codeValue != tc.wantCodeValue {
|
||||
t.Errorf("codeValue = %q, want %q", codeValue, tc.wantCodeValue)
|
||||
}
|
||||
if strings.TrimSpace(fallback) == "" {
|
||||
t.Errorf("fallback message must not be empty")
|
||||
}
|
||||
for _, s := range tc.fallbackHas {
|
||||
if !strings.Contains(strings.ToLower(fallback), s) {
|
||||
t.Errorf("fallback %q missing %q", fallback, s)
|
||||
}
|
||||
}
|
||||
for _, s := range tc.noLeak {
|
||||
if strings.Contains(code, s) || strings.Contains(fallback, s) {
|
||||
t.Errorf("raw detail %q leaked into code/fallback (code=%q fallback=%q)", s, code, fallback)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// engine.go holds the single generic streaming sync engine shared by the wiki
|
||||
// Connector and the Drive DriveConnector. The per-connector differences (node
|
||||
// type, listing API, edit-time field, cursor wire format, fetch dispatch,
|
||||
// log tag) are isolated behind the NodeOps adapter interface. FetchAll /
|
||||
// FetchIncremental are thin wrappers over the same engine that collect Emits
|
||||
// instead of streaming them.
|
||||
//
|
||||
// Behaviour note (deliberate, see ADR-0005 / design §2.4):
|
||||
// - Resume/incremental fast-path: a node recorded at its current edit time
|
||||
// is skipped, keeping the cursor entry.
|
||||
// - A fetch failure does NOT advance the cursor: the prior edit time is
|
||||
// retained so the node is retried next run instead of being permanently
|
||||
// skipped on a transient export failure (Tencent/WeKnora#2136). This now
|
||||
// also holds for the FetchIncremental path (previously it advanced the
|
||||
// cursor before fetching, a latent #2136 bug).
|
||||
// - Logs use the "stream progress/summary" wording uniformly; the
|
||||
// FetchIncremental path additionally gains per-100 progress + tally
|
||||
// summary logs it did not Emit before (log-only change).
|
||||
|
||||
// NodeOps adapts one connector's node type to the shared sync engine. Every
|
||||
// method is a pure accessor or a thin wrapper - no engine logic lives here.
|
||||
type NodeOps[N any] interface {
|
||||
// List returns every syncable node under resourceID. A non-nil partial
|
||||
// (with err == nil) signals a partial listing: nodes is still usable and
|
||||
// the sync continues, but the caller surfaces the failed sub-trees via
|
||||
// ListFailureItems. A non-nil err is fatal and aborts the sync.
|
||||
List(ctx context.Context, client *Client, resourceID string) (nodes []N, partial error, err error)
|
||||
|
||||
Token(n N) string
|
||||
Title(n N) string
|
||||
ObjType(n N) string
|
||||
// EditTime is the change-detection timestamp string stored in the cursor.
|
||||
EditTime(n N) string
|
||||
|
||||
// fetch retrieves one node's content; (nil, nil) means an unsupported
|
||||
// type that yields no item.
|
||||
Fetch(ctx context.Context, client *Client, n N, resourceID string, multimodal bool) ([]*types.FetchedItem, error)
|
||||
|
||||
// ListFailureItems converts a partial-listing error into error FetchedItems.
|
||||
ListFailureItems(resourceID string, partial error) []types.FetchedItem
|
||||
// ResourceNoun is the noun in user-visible error text: "nodes" / "files".
|
||||
ResourceNoun() string
|
||||
// EmptyResourceIDsError is the connector-specific message when no resource
|
||||
// IDs are configured (wiki/drive text differs and is preserved verbatim).
|
||||
EmptyResourceIDsError() string
|
||||
// LogTag is the log prefix: "[Feishu]" / "[FeishuDrive]".
|
||||
LogTag() string
|
||||
|
||||
// DecodeCursorTimes extracts the per-resource edit-time map from a
|
||||
// persisted ConnectorCursor (nil-safe: returns nil when absent).
|
||||
DecodeCursorTimes(m map[string]interface{}) map[string]map[string]string
|
||||
// EncodeCursor wraps the engine's internal times map into the connector's
|
||||
// wire-format SyncCursor. JSON-marshals for snapshot isolation, mirroring
|
||||
// the original FeishuCursor.toSyncCursor / FeishuDriveCursor.toSyncCursor.
|
||||
EncodeCursor(times map[string]map[string]string, lastSync time.Time) *types.SyncCursor
|
||||
}
|
||||
|
||||
// CollectHandler is the StreamHandler used by FetchAll / FetchIncremental to
|
||||
// gather every Emitted item into a slice instead of streaming. Checkpoint is a
|
||||
// no-op: those paths return a single cursor at the end.
|
||||
type CollectHandler struct {
|
||||
items []types.FetchedItem
|
||||
}
|
||||
|
||||
func (h *CollectHandler) Emit(_ context.Context, item types.FetchedItem) error {
|
||||
h.items = append(h.items, item)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *CollectHandler) Checkpoint(_ context.Context, _ *types.SyncCursor) error { return nil }
|
||||
|
||||
// runSync is the single implementation behind FetchStream / FetchAll /
|
||||
// FetchIncremental. With cursor == nil it fetches everything (full sync);
|
||||
// with a cursor it skips nodes whose recorded edit time is unchanged
|
||||
// (incremental + resume). resourceIDs comes from the caller: FetchStream /
|
||||
// FetchIncremental pass config.ResourceIDs (after a non-empty check),
|
||||
// FetchAll passes its own argument without one.
|
||||
func runSync[N any](
|
||||
ctx context.Context, client *Client, config *types.DataSourceConfig,
|
||||
resourceIDs []string, cursor *types.SyncCursor, h datasource.StreamHandler,
|
||||
ops NodeOps[N],
|
||||
) (*types.SyncCursor, error) {
|
||||
var prevTimes map[string]map[string]string
|
||||
if cursor != nil && cursor.ConnectorCursor != nil {
|
||||
prevTimes = ops.DecodeCursorTimes(cursor.ConnectorCursor)
|
||||
}
|
||||
|
||||
newTimes := make(map[string]map[string]string)
|
||||
lastSync := time.Now()
|
||||
|
||||
processed := 0
|
||||
lastCheckpoint := time.Now()
|
||||
for _, resourceID := range resourceIDs {
|
||||
nodes, partial, err := ops.List(ctx, client, resourceID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list %s for resource %s: %w", ops.ResourceNoun(), resourceID, err)
|
||||
}
|
||||
if partial != nil {
|
||||
for _, item := range ops.ListFailureItems(resourceID, partial) {
|
||||
if eerr := h.Emit(ctx, item); eerr != nil {
|
||||
return nil, eerr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
newTimes[resourceID] = make(map[string]string)
|
||||
// On a partial listing, carry prior edit times forward so a later full
|
||||
// listing can still detect changes and deletions.
|
||||
if partial != nil && prevTimes != nil {
|
||||
if prev, ok := prevTimes[resourceID]; ok {
|
||||
for tok, et := range prev {
|
||||
newTimes[resourceID][tok] = et
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
currentNodes := make(map[string]bool)
|
||||
tally := newFetchTally(len(nodes))
|
||||
for i, node := range nodes {
|
||||
tok := ops.Token(node)
|
||||
currentNodes[tok] = true
|
||||
editTimeStr := ops.EditTime(node)
|
||||
|
||||
var prevEdit string
|
||||
var hadPrev bool
|
||||
if prevTimes != nil {
|
||||
if prev, ok := prevTimes[resourceID]; ok {
|
||||
prevEdit, hadPrev = prev[tok]
|
||||
}
|
||||
}
|
||||
|
||||
// Resume/incremental fast-path: a node recorded at its current edit
|
||||
// time is unchanged (or already synced this run) - keep the record
|
||||
// and Skip re-fetching.
|
||||
if hadPrev && prevEdit == editTimeStr {
|
||||
newTimes[resourceID][tok] = editTimeStr
|
||||
continue
|
||||
}
|
||||
|
||||
items, ferr := ops.Fetch(ctx, client, node, resourceID, config.MultimodalEnabled)
|
||||
if ferr != nil {
|
||||
tally.fail()
|
||||
// Do NOT advance the cursor: the content was never fetched.
|
||||
// Retain the prior edit time (if any) so prev != current next
|
||||
// run and the node is retried, instead of being permanently
|
||||
// skipped on a transient export failure (Tencent/WeKnora#2136).
|
||||
if hadPrev {
|
||||
newTimes[resourceID][tok] = prevEdit
|
||||
}
|
||||
if eerr := h.Emit(ctx, types.FetchedItem{
|
||||
ExternalID: tok,
|
||||
Title: ops.Title(node),
|
||||
SourceResourceID: resourceID,
|
||||
Metadata: FeishuErrorItemMeta(ferr, nil),
|
||||
}); eerr != nil {
|
||||
return nil, eerr
|
||||
}
|
||||
} else {
|
||||
// Fetched, or an unsupported type (nothing to fetch): record
|
||||
// the current edit time so the node is not re-processed next run.
|
||||
newTimes[resourceID][tok] = editTimeStr
|
||||
if len(items) > 0 {
|
||||
tally.fetch()
|
||||
for _, it := range items {
|
||||
if eerr := h.Emit(ctx, *it); eerr != nil {
|
||||
return nil, eerr
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Unsupported type (mindnote/slides/…): no item.
|
||||
tally.Skip(ops.ObjType(node))
|
||||
}
|
||||
}
|
||||
|
||||
processed++
|
||||
if processed%FeishuStreamCheckpointInterval == 0 || time.Since(lastCheckpoint) >= FeishuStreamCheckpointMaxInterval {
|
||||
if cerr := h.Checkpoint(ctx, ops.EncodeCursor(newTimes, lastSync)); cerr != nil {
|
||||
logger.Warnf(ctx, "%s stream Checkpoint failed: %v", ops.LogTag(), cerr)
|
||||
}
|
||||
lastCheckpoint = time.Now()
|
||||
}
|
||||
if n := i + 1; n%100 == 0 {
|
||||
logger.Infof(ctx, "%s stream progress resource=%s %d/%d (%s)",
|
||||
ops.LogTag(), resourceID, n, len(nodes), tally.summary())
|
||||
}
|
||||
}
|
||||
|
||||
// Detect deleted nodes (only when the full tree was listed successfully).
|
||||
// A partial listing did not enumerate the whole subtree, so deletion
|
||||
// detection would false-positive.
|
||||
if partial == nil && prevTimes != nil {
|
||||
if prev, ok := prevTimes[resourceID]; ok {
|
||||
for tok := range prev {
|
||||
if !currentNodes[tok] {
|
||||
if eerr := h.Emit(ctx, types.FetchedItem{
|
||||
ExternalID: tok,
|
||||
IsDeleted: true,
|
||||
SourceResourceID: resourceID,
|
||||
}); eerr != nil {
|
||||
return nil, eerr
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.Infof(ctx, "%s stream summary resource=%s %s", ops.LogTag(), resourceID, tally.summary())
|
||||
}
|
||||
|
||||
return ops.EncodeCursor(newTimes, lastSync), nil
|
||||
}
|
||||
|
||||
// FetchStreamEngine runs the streaming sync. FetchStream / FetchIncremental
|
||||
// shells differ only in whether they pass a cursor and how they collect
|
||||
// results; both route here.
|
||||
func FetchStreamEngine[N any](
|
||||
ctx context.Context, client *Client, config *types.DataSourceConfig,
|
||||
cursor *types.SyncCursor, h datasource.StreamHandler, ops NodeOps[N],
|
||||
) (*types.SyncCursor, error) {
|
||||
return runSync(ctx, client, config, config.ResourceIDs, cursor, h, ops)
|
||||
}
|
||||
|
||||
// FetchAllEngine runs a full sync, collecting every item into a slice. It
|
||||
// passes the caller-supplied resourceIDs without a non-empty check (mirroring
|
||||
// the original FetchAll which accepted an empty list and returned no items).
|
||||
// On error any partially collected items are discarded, matching the original
|
||||
// behaviour (a fatal list error dropped everything collected so far).
|
||||
func FetchAllEngine[N any](
|
||||
ctx context.Context, client *Client, config *types.DataSourceConfig,
|
||||
resourceIDs []string, ops NodeOps[N],
|
||||
) ([]types.FetchedItem, error) {
|
||||
ch := &CollectHandler{}
|
||||
if _, err := runSync(ctx, client, config, resourceIDs, nil, ch, ops); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ch.items, nil
|
||||
}
|
||||
|
||||
// FetchIncrementalEngine runs an incremental sync against cursor, collecting
|
||||
// items. resourceIDs come from config.ResourceIDs (non-empty check is the
|
||||
// caller's responsibility).
|
||||
func FetchIncrementalEngine[N any](
|
||||
ctx context.Context, client *Client, config *types.DataSourceConfig,
|
||||
cursor *types.SyncCursor, ops NodeOps[N],
|
||||
) ([]types.FetchedItem, *types.SyncCursor, error) {
|
||||
ch := &CollectHandler{}
|
||||
next, err := runSync(ctx, client, config, config.ResourceIDs, cursor, ch, ops)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return ch.items, next, nil
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost,open.feishu.cn,open.larksuite.com")
|
||||
secutils.ResetSSRFWhitelistForTest()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, v interface{}) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
// blk constructors mirroring wiki/connector_golden_test.go, for core-internal
|
||||
// block-rendering tests (DocxBlock is same-package here, no core. prefix).
|
||||
func sheetBlk(id, token string) DocxBlock {
|
||||
return DocxBlock{BlockID: id, BlockType: BlockTypeSheet, Sheet: &BlockTokenRef{Token: token}}
|
||||
}
|
||||
|
||||
func bitableBlk(id, token string) DocxBlock {
|
||||
return DocxBlock{BlockID: id, BlockType: BlockTypeBitable, Bitable: &BlockTokenRef{Token: token}}
|
||||
}
|
||||
|
||||
func imageBlk(id, token string) DocxBlock {
|
||||
return DocxBlock{BlockID: id, BlockType: BlockTypeImage, Image: &BlockTokenRef{Token: token}}
|
||||
}
|
||||
|
||||
func fileBlk(id, token, name string) DocxBlock {
|
||||
return DocxBlock{BlockID: id, BlockType: BlockTypeFile, File: &BlockFileRef{Token: token, Name: name}}
|
||||
}
|
||||
|
||||
func cellBlk(id string) DocxBlock {
|
||||
return DocxBlock{BlockID: id, BlockType: BlockTypeTableCell, Children: []string{id + "_txt"}}
|
||||
}
|
||||
|
||||
func cellTextBlk(id, text string) DocxBlock {
|
||||
return DocxBlock{BlockID: id + "_txt", BlockType: BlockTypeText, Text: txt(text)}
|
||||
}
|
||||
|
||||
func tableBlk(id string, cols int, cellIDs ...string) DocxBlock {
|
||||
b := DocxBlock{BlockID: id, BlockType: BlockTypeTable}
|
||||
b.Table = &BlockTable{Cells: cellIDs}
|
||||
b.Table.Property = &BlockTableProperty{ColumnSize: cols}
|
||||
return b
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// sheetReader is the subset of *Client that blocksToMarkdown needs, so the
|
||||
// converter can be unit-tested with a fake or nil client.
|
||||
type sheetReader interface {
|
||||
readSheetRange(ctx context.Context, embedToken string) ([][]string, bool, error)
|
||||
readBitableRecords(ctx context.Context, embedToken string) ([][]string, bool, error)
|
||||
}
|
||||
|
||||
// pendingAttachment is an embedded file block awaiting a download decision by
|
||||
// the connector (whitelist/size filtering happens there, not here).
|
||||
type pendingAttachment struct {
|
||||
FileToken string
|
||||
Name string
|
||||
}
|
||||
|
||||
// blocksToMarkdown renders a flat docx block array to Markdown, inlining
|
||||
// embedded spreadsheet/bitable tables (Task 5) and collecting downloadable
|
||||
// attachments. client may be nil when the block set has no downdrill blocks.
|
||||
func blocksToMarkdown(ctx context.Context, client sheetReader, blocks []DocxBlock) ([]byte, []pendingAttachment, error) {
|
||||
byID := make(map[string]DocxBlock, len(blocks))
|
||||
for _, b := range blocks {
|
||||
byID[b.BlockID] = b
|
||||
}
|
||||
// Blocks nested inside a native table (its cells and their content blocks)
|
||||
// are rendered by renderNativeTable. Mark them so the flat loop below does
|
||||
// not also Emit them as stray top-level paragraphs.
|
||||
consumed := tableDescendants(blocks, byID)
|
||||
|
||||
var sb strings.Builder
|
||||
var atts []pendingAttachment
|
||||
for _, b := range blocks {
|
||||
if consumed[b.BlockID] {
|
||||
continue
|
||||
}
|
||||
switch b.BlockType {
|
||||
case BlockTypePage:
|
||||
continue // root container, no text of its own
|
||||
case BlockTypeText:
|
||||
writePara(&sb, plainText(textBearingField(b)))
|
||||
case BlockTypeBullet:
|
||||
writePara(&sb, "- "+plainText(textBearingField(b)))
|
||||
case BlockTypeOrdered:
|
||||
writePara(&sb, "1. "+plainText(textBearingField(b)))
|
||||
case BlockTypeCode:
|
||||
writePara(&sb, "```\n"+plainText(textBearingField(b))+"\n```")
|
||||
case BlockTypeQuote:
|
||||
writePara(&sb, "> "+plainText(textBearingField(b)))
|
||||
case BlockTypeDivider:
|
||||
writePara(&sb, "---")
|
||||
case BlockTypeTable:
|
||||
writePara(&sb, renderNativeTable(b, byID))
|
||||
case BlockTypeTableCell:
|
||||
continue // rendered by its parent table (also covered by `consumed`)
|
||||
case BlockTypeSheet:
|
||||
if b.Sheet != nil {
|
||||
writePara(&sb, inlineTable(ctx, client, b.Sheet.Token, "sheet"))
|
||||
}
|
||||
case BlockTypeBitable:
|
||||
if b.Bitable != nil {
|
||||
writePara(&sb, inlineTable(ctx, client, b.Bitable.Token, "bitable"))
|
||||
}
|
||||
case BlockTypeImage:
|
||||
// Emit a token-free placeholder: images carry no retrievable text, and
|
||||
// leaking the internal media token would pollute embeddings. A neutral
|
||||
// marker preserves surrounding context (e.g. "如下图所示").
|
||||
writePara(&sb, "![图片]()")
|
||||
case BlockTypeTodo:
|
||||
if t := plainText(textBearingField(b)); t != "" {
|
||||
writePara(&sb, "- [ ] "+t)
|
||||
}
|
||||
case BlockTypeCallout:
|
||||
// Callout is a container; its body is usually in child blocks (rendered
|
||||
// separately). Emit its own text only if it carries direct inline text,
|
||||
// so the container case is a safe no-op.
|
||||
if t := plainText(textBearingField(b)); t != "" {
|
||||
writePara(&sb, "> "+t)
|
||||
}
|
||||
case BlockTypeFile:
|
||||
if b.File != nil {
|
||||
name := b.File.Name
|
||||
if name == "" {
|
||||
name = b.File.Token
|
||||
}
|
||||
writePara(&sb, "📎 附件:"+name)
|
||||
atts = append(atts, pendingAttachment{FileToken: b.File.Token, Name: b.File.Name})
|
||||
}
|
||||
default:
|
||||
if b.BlockType >= BlockTypeHeading1 && b.BlockType <= blockTypeHeading9 {
|
||||
level := b.BlockType - BlockTypeHeading1 + 1
|
||||
writePara(&sb, strings.Repeat("#", level)+" "+plainText(textBearingField(b)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out := strings.TrimRight(sb.String(), "\n")
|
||||
if out != "" {
|
||||
out += "\n"
|
||||
}
|
||||
return []byte(out), atts, nil
|
||||
}
|
||||
|
||||
// writePara appends a block of text followed by a blank line; empty text is skipped.
|
||||
func writePara(sb *strings.Builder, s string) {
|
||||
if s == "" {
|
||||
return
|
||||
}
|
||||
sb.WriteString(s)
|
||||
sb.WriteString("\n\n")
|
||||
}
|
||||
|
||||
// plainText concatenates the text runs of a text-bearing block.
|
||||
func plainText(bt *BlockText) string {
|
||||
if bt == nil {
|
||||
return ""
|
||||
}
|
||||
var sb strings.Builder
|
||||
for _, e := range bt.Elements {
|
||||
if e.TextRun != nil {
|
||||
sb.WriteString(e.TextRun.Content)
|
||||
}
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// headingText returns the heading field for the block's level, or Text as fallback.
|
||||
func headingText(b DocxBlock) *BlockText {
|
||||
fields := []*BlockText{
|
||||
b.Heading1, b.Heading2, b.Heading3, b.Heading4, b.Heading5,
|
||||
b.Heading6, b.Heading7, b.Heading8, b.Heading9,
|
||||
}
|
||||
if idx := b.BlockType - BlockTypeHeading1; idx >= 0 && idx < len(fields) && fields[idx] != nil {
|
||||
return fields[idx]
|
||||
}
|
||||
return b.Text
|
||||
}
|
||||
|
||||
// tableDescendants returns the set of block IDs that belong to a native table —
|
||||
// every cell listed in a table block plus everything reachable through those
|
||||
// cells' Children. blocksToMarkdown skips these so table content is emitted only
|
||||
// by the table renderer, never a second time as loose paragraphs.
|
||||
func tableDescendants(blocks []DocxBlock, byID map[string]DocxBlock) map[string]bool {
|
||||
consumed := make(map[string]bool)
|
||||
var mark func(id string)
|
||||
mark = func(id string) {
|
||||
b, ok := byID[id]
|
||||
if !ok || consumed[id] {
|
||||
return
|
||||
}
|
||||
// Attachment/media blocks nested in a cell must still be collected (and
|
||||
// their reference emitted) by the main loop — the table renderer only
|
||||
// extracts text — so do not consume them, only their text structure.
|
||||
if b.BlockType == BlockTypeFile || b.BlockType == BlockTypeImage {
|
||||
return
|
||||
}
|
||||
consumed[id] = true
|
||||
for _, c := range b.Children {
|
||||
mark(c)
|
||||
}
|
||||
}
|
||||
for _, b := range blocks {
|
||||
// Only consume cells of tables we will actually render. A table that
|
||||
// renderNativeTable would bail on (missing property / zero columns) must
|
||||
// NOT have its cells consumed, or their text would be dropped entirely —
|
||||
// leave them for the flat loop to Emit as loose paragraphs instead.
|
||||
if b.BlockType == BlockTypeTable && tableRenderable(b) {
|
||||
for _, cid := range b.Table.Cells {
|
||||
mark(cid)
|
||||
}
|
||||
}
|
||||
}
|
||||
return consumed
|
||||
}
|
||||
|
||||
// tableRenderable reports whether a native table block carries enough structure
|
||||
// (a column count) for renderNativeTable to produce a Markdown table. It is the
|
||||
// single predicate shared by the consume pass and the render pass so the two
|
||||
// never disagree about which tables are handled by the table renderer.
|
||||
func tableRenderable(b DocxBlock) bool {
|
||||
return b.Table != nil && b.Table.Property != nil && b.Table.Property.ColumnSize > 0
|
||||
}
|
||||
|
||||
// textBearingField returns the inline-text payload for whichever type-named
|
||||
// field a block populates (docx stores a block's text in a field named after
|
||||
// its type), so cell content of any text-like type can be extracted uniformly.
|
||||
func textBearingField(b DocxBlock) *BlockText {
|
||||
switch b.BlockType {
|
||||
case BlockTypeText:
|
||||
return b.Text
|
||||
case BlockTypeBullet:
|
||||
return b.Bullet
|
||||
case BlockTypeOrdered:
|
||||
return b.Ordered
|
||||
case BlockTypeCode:
|
||||
return b.Code
|
||||
case BlockTypeQuote:
|
||||
return b.Quote
|
||||
case BlockTypeTodo:
|
||||
return b.Todo
|
||||
case BlockTypeCallout:
|
||||
return b.Callout
|
||||
}
|
||||
if b.BlockType >= BlockTypeHeading1 && b.BlockType <= blockTypeHeading9 {
|
||||
return headingText(b)
|
||||
}
|
||||
return b.Text
|
||||
}
|
||||
|
||||
// cellText renders a native table cell to a single string. A Feishu table_cell
|
||||
// (block_type 32) is a container: its text lives in child blocks, not on the
|
||||
// cell itself, so we concatenate the text of each child block.
|
||||
func cellText(cell DocxBlock, byID map[string]DocxBlock) string {
|
||||
var parts []string
|
||||
for _, childID := range cell.Children {
|
||||
if t := plainText(textBearingField(byID[childID])); t != "" {
|
||||
parts = append(parts, t)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// renderNativeTable renders a native docx table block into a Markdown table.
|
||||
func renderNativeTable(b DocxBlock, byID map[string]DocxBlock) string {
|
||||
if !tableRenderable(b) {
|
||||
return ""
|
||||
}
|
||||
cols := b.Table.Property.ColumnSize
|
||||
var cells []string
|
||||
for _, cid := range b.Table.Cells {
|
||||
cells = append(cells, cellText(byID[cid], byID))
|
||||
}
|
||||
var rows [][]string
|
||||
for i := 0; i < len(cells); i += cols {
|
||||
end := i + cols
|
||||
if end > len(cells) {
|
||||
end = len(cells)
|
||||
}
|
||||
rows = append(rows, cells[i:end])
|
||||
}
|
||||
return markdownTable(rows)
|
||||
}
|
||||
|
||||
// markdownTable renders a [][]string (first row = header) as a GFM table.
|
||||
func markdownTable(rows [][]string) string {
|
||||
// A zero-column header (an embedded sheet/bitable with no columns) would Emit
|
||||
// a header line of "| |" and a separator of just "|" — malformed GFM. Render
|
||||
// nothing instead.
|
||||
if len(rows) == 0 || len(rows[0]) == 0 {
|
||||
return ""
|
||||
}
|
||||
var sb strings.Builder
|
||||
cols := len(rows[0])
|
||||
sb.WriteString("| " + strings.Join(escapePipes(rows[0]), " | ") + " |\n")
|
||||
sb.WriteString("|" + strings.Repeat(" --- |", cols) + "\n")
|
||||
for _, r := range rows[1:] {
|
||||
// Ragged data: a row wider than the header would Emit more cells than the
|
||||
// header/separator declare, producing a malformed GFM table. Clamp to the
|
||||
// header width (truncate overflow, pad shortfall).
|
||||
if len(r) > cols {
|
||||
r = r[:cols]
|
||||
}
|
||||
for len(r) < cols {
|
||||
r = append(r, "")
|
||||
}
|
||||
sb.WriteString("| " + strings.Join(escapePipes(r), " | ") + " |\n")
|
||||
}
|
||||
return strings.TrimRight(sb.String(), "\n")
|
||||
}
|
||||
|
||||
// escapePipes makes cell values safe for a single Markdown table row.
|
||||
func escapePipes(row []string) []string {
|
||||
out := make([]string, len(row))
|
||||
for i, c := range row {
|
||||
out[i] = strings.ReplaceAll(strings.ReplaceAll(c, "\n", " "), "|", "\\|")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// inlineTable reads an embedded sheet/bitable and renders it as a Markdown
|
||||
// table. Read/permission errors degrade to an inline note rather than failing
|
||||
// the whole document (partial content beats no content). A reader-reported
|
||||
// truncation appends a note.
|
||||
func inlineTable(ctx context.Context, client sheetReader, token, kind string) string {
|
||||
if client == nil {
|
||||
return ""
|
||||
}
|
||||
var (
|
||||
rows [][]string
|
||||
truncated bool
|
||||
err error
|
||||
noun string
|
||||
)
|
||||
if kind == "sheet" {
|
||||
rows, truncated, err = client.readSheetRange(ctx, token)
|
||||
noun = "内嵌电子表格"
|
||||
} else {
|
||||
rows, truncated, err = client.readBitableRecords(ctx, token)
|
||||
noun = "内嵌多维表格"
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Sprintf("> [无法读取%s]", noun)
|
||||
}
|
||||
// markdownTable returns "" when there is nothing renderable (no rows, or a
|
||||
// header with no columns). Skip the truncation note too in that case, since it
|
||||
// would otherwise dangle without a table above it.
|
||||
table := markdownTable(rows)
|
||||
if table == "" {
|
||||
return ""
|
||||
}
|
||||
if truncated {
|
||||
table += fmt.Sprintf("\n\n> 表格已截断(仅显示前 %d 行)", maxTableRows)
|
||||
}
|
||||
return table
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// txt builds a text-bearing BlockText from plain content.
|
||||
func txt(s string) *BlockText {
|
||||
return &BlockText{Elements: []TextElement{{TextRun: &TextRun{Content: s}}}}
|
||||
}
|
||||
|
||||
type fakeReader struct {
|
||||
sheet [][]string
|
||||
sheetTruncated bool
|
||||
sheetErr error
|
||||
bitable [][]string
|
||||
bitableTruncated bool
|
||||
bitableErr error
|
||||
}
|
||||
|
||||
func (f fakeReader) readSheetRange(_ context.Context, _ string) ([][]string, bool, error) {
|
||||
return f.sheet, f.sheetTruncated, f.sheetErr
|
||||
}
|
||||
|
||||
func (f fakeReader) readBitableRecords(_ context.Context, _ string) ([][]string, bool, error) {
|
||||
return f.bitable, f.bitableTruncated, f.bitableErr
|
||||
}
|
||||
|
||||
func TestBlocksToMarkdown_EmbeddedSheetAndFile(t *testing.T) {
|
||||
blocks := []DocxBlock{
|
||||
{BlockID: "root", BlockType: BlockTypePage},
|
||||
{BlockID: "s", BlockType: BlockTypeSheet, Sheet: &BlockTokenRef{Token: "sht_a_0"}},
|
||||
{BlockID: "img", BlockType: BlockTypeImage, Image: &BlockTokenRef{Token: "img_t"}},
|
||||
{BlockID: "f", BlockType: BlockTypeFile, File: &BlockFileRef{Token: "file_t", Name: "报表.pdf"}},
|
||||
}
|
||||
fr := fakeReader{sheet: [][]string{{"名称", "数量"}, {"苹果", "3"}}}
|
||||
md, atts, err := blocksToMarkdown(context.Background(), fr, blocks)
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(md), "| 名称 | 数量 |") || !strings.Contains(string(md), "| 苹果 | 3 |") {
|
||||
t.Errorf("sheet not inlined:\n%s", md)
|
||||
}
|
||||
if !strings.Contains(string(md), "![图片]()") {
|
||||
t.Errorf("image placeholder missing:\n%s", md)
|
||||
}
|
||||
if strings.Contains(string(md), "feishu-media") {
|
||||
t.Errorf("internal media token leaked into markdown:\n%s", md)
|
||||
}
|
||||
if len(atts) != 1 || atts[0].FileToken != "file_t" || atts[0].Name != "报表.pdf" {
|
||||
t.Errorf("attachments = %+v", atts)
|
||||
}
|
||||
if !strings.Contains(string(md), "📎 附件:报表.pdf") {
|
||||
t.Errorf("attachment inline reference missing:\n%s", md)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlocksToMarkdown_SheetTruncatedNote(t *testing.T) {
|
||||
blocks := []DocxBlock{
|
||||
{BlockID: "root", BlockType: BlockTypePage},
|
||||
{BlockID: "s", BlockType: BlockTypeSheet, Sheet: &BlockTokenRef{Token: "sht_a_0"}},
|
||||
}
|
||||
fr := fakeReader{sheet: [][]string{{"h"}, {"1"}}, sheetTruncated: true}
|
||||
md, _, err := blocksToMarkdown(context.Background(), fr, blocks)
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(md), "表格已截断") {
|
||||
t.Errorf("want truncation note, got:\n%s", md)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlocksToMarkdown_SheetPermissionDegrades(t *testing.T) {
|
||||
blocks := []DocxBlock{
|
||||
{BlockID: "root", BlockType: BlockTypePage},
|
||||
{BlockID: "s", BlockType: BlockTypeSheet, Sheet: &BlockTokenRef{Token: "sht_a_0"}},
|
||||
}
|
||||
fr := fakeReader{sheetErr: fmt.Errorf("code=99991672 permission denied")}
|
||||
md, _, err := blocksToMarkdown(context.Background(), fr, blocks)
|
||||
if err != nil {
|
||||
t.Fatalf("should not fail on permission error, got %v", err)
|
||||
}
|
||||
if !strings.Contains(string(md), "无法读取内嵌电子表格") {
|
||||
t.Errorf("want degraded placeholder, got:\n%s", md)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlocksToMarkdown_BitableInlinedAndDegrades(t *testing.T) {
|
||||
mk := func(fr fakeReader) string {
|
||||
blocks := []DocxBlock{
|
||||
{BlockID: "root", BlockType: BlockTypePage},
|
||||
{BlockID: "bt", BlockType: BlockTypeBitable, Bitable: &BlockTokenRef{Token: "bascabc_tblxyz"}},
|
||||
}
|
||||
md, _, err := blocksToMarkdown(context.Background(), fr, blocks)
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
return string(md)
|
||||
}
|
||||
|
||||
inlined := mk(fakeReader{bitable: [][]string{{"任务", "状态"}, {"写文档", "进行中"}}})
|
||||
if !strings.Contains(inlined, "| 任务 | 状态 |") || !strings.Contains(inlined, "| 写文档 | 进行中 |") {
|
||||
t.Errorf("bitable not inlined:\n%s", inlined)
|
||||
}
|
||||
|
||||
degraded := mk(fakeReader{bitableErr: fmt.Errorf("code=99991672 permission denied")})
|
||||
if !strings.Contains(degraded, "无法读取内嵌多维表格") {
|
||||
t.Errorf("want bitable degradation note, got:\n%s", degraded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlocksToMarkdown_NativeTableFromCellChildren(t *testing.T) {
|
||||
// Real Feishu shape: a table_cell (block_type 32) is a container whose text
|
||||
// lives in child text blocks, not on the cell. The renderer must read cell
|
||||
// text from those children AND must not also Emit them as loose paragraphs.
|
||||
blocks := []DocxBlock{
|
||||
{BlockID: "root", BlockType: BlockTypePage},
|
||||
tableBlk("t", 2, "c1", "c2", "c3", "c4"),
|
||||
cellBlk("c1"), cellBlk("c2"), cellBlk("c3"), cellBlk("c4"),
|
||||
cellTextBlk("c1", "姓名"), cellTextBlk("c2", "分数"),
|
||||
cellTextBlk("c3", "张三"), cellTextBlk("c4", "95"),
|
||||
}
|
||||
md, _, err := blocksToMarkdown(context.Background(), nil, blocks)
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
s := string(md)
|
||||
if !strings.Contains(s, "| 姓名 | 分数 |") || !strings.Contains(s, "| 张三 | 95 |") {
|
||||
t.Errorf("native table not rendered from cell children:\n%s", s)
|
||||
}
|
||||
// Cell text must appear ONLY inside the table, never as a stray paragraph.
|
||||
if n := strings.Count(s, "姓名"); n != 1 {
|
||||
t.Errorf("cell text leaked outside the table (count=%d, want 1):\n%s", n, s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlocksToMarkdown_AttachmentInsideTableCellStillCollected(t *testing.T) {
|
||||
// A file block nested inside a table cell must still be collected as an
|
||||
// attachment — the table "consumed" set marks a cell's text structure but
|
||||
// must NOT swallow attachment/media blocks, or embedded files silently vanish.
|
||||
blocks := []DocxBlock{
|
||||
{BlockID: "root", BlockType: BlockTypePage},
|
||||
tableBlk("t", 1, "c1"),
|
||||
{BlockID: "c1", BlockType: BlockTypeTableCell, Children: []string{"f1"}},
|
||||
{BlockID: "f1", BlockType: BlockTypeFile, File: &BlockFileRef{Token: "tok-in-cell", Name: "内嵌.pdf"}},
|
||||
}
|
||||
_, atts, err := blocksToMarkdown(context.Background(), nil, blocks)
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if len(atts) != 1 || atts[0].FileToken != "tok-in-cell" {
|
||||
t.Fatalf("attachment nested in a table cell was not collected: %+v", atts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkdownTable_RaggedRowClampedToHeader(t *testing.T) {
|
||||
// Header declares 2 columns; a data row with 4 cells (trailing populated
|
||||
// cells beyond the header range) must be clamped to 2, or the emitted table
|
||||
// has more body cells than header/separator columns → malformed GFM.
|
||||
rows := [][]string{
|
||||
{"名称", "数量"},
|
||||
{"苹果", "3", "多余1", "多余2"},
|
||||
}
|
||||
out := markdownTable(rows)
|
||||
lines := strings.Split(out, "\n")
|
||||
if len(lines) != 3 {
|
||||
t.Fatalf("want 3 lines (header, separator, 1 row), got %d:\n%s", len(lines), out)
|
||||
}
|
||||
// Every row must have the same pipe count as the 2-column header.
|
||||
want := strings.Count(lines[0], "|")
|
||||
for i, ln := range lines {
|
||||
if got := strings.Count(ln, "|"); got != want {
|
||||
t.Errorf("line %d %q has %d pipes, want %d (ragged row not clamped)", i, ln, got, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(out, "多余") {
|
||||
t.Errorf("overflow cells must be dropped, got:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkdownTable_ZeroColumnRendersNothing(t *testing.T) {
|
||||
// An embedded sheet/bitable with a header row but no columns (e.g. a bitable
|
||||
// whose fields were all deleted) yields rows == [][]string{{}}. len(rows)==1
|
||||
// passes a naive empty guard, but cols==0 would Emit a malformed GFM table
|
||||
// ("| |" header + a bare "|" separator). It must render nothing instead.
|
||||
for _, rows := range [][][]string{
|
||||
{{}}, // one empty header row, no data
|
||||
{{}, {}, {}}, // empty header + empty data rows
|
||||
} {
|
||||
out := markdownTable(rows)
|
||||
if out != "" {
|
||||
t.Errorf("zero-column table must render nothing, got %q for rows=%+v", out, rows)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlocksToMarkdown_UnrenderableTablePreservesCellText(t *testing.T) {
|
||||
// A native table block with no column property (Table!=nil, Property==nil)
|
||||
// cannot be rendered as a Markdown table. Its cells must NOT be consumed, so
|
||||
// their text still reaches the output as loose paragraphs rather than
|
||||
// vanishing entirely.
|
||||
blocks := []DocxBlock{
|
||||
{BlockID: "root", BlockType: BlockTypePage},
|
||||
// tableBlk sets Property (renderable); here we want an UNrenderable one.
|
||||
{BlockID: "t", BlockType: BlockTypeTable, Table: &BlockTable{Cells: []string{"c1"}}},
|
||||
{BlockID: "c1", BlockType: BlockTypeTableCell, Children: []string{"c1_txt"}},
|
||||
cellTextBlk("c1", "重要内容"),
|
||||
}
|
||||
md, _, err := blocksToMarkdown(context.Background(), nil, blocks)
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(md), "重要内容") {
|
||||
t.Errorf("cell text of an unrenderable table was dropped:\n%s", md)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlocksToMarkdown_TextConstructs(t *testing.T) {
|
||||
blocks := []DocxBlock{
|
||||
{BlockID: "root", BlockType: BlockTypePage, Children: []string{"h", "p", "b"}},
|
||||
{BlockID: "h", BlockType: BlockTypeHeading1, Heading1: txt("标题")},
|
||||
{BlockID: "p", BlockType: BlockTypeText, Text: txt("一段正文")},
|
||||
{BlockID: "b", BlockType: BlockTypeBullet, Bullet: txt("要点")},
|
||||
}
|
||||
md, atts, err := blocksToMarkdown(context.Background(), nil, blocks)
|
||||
if err != nil {
|
||||
t.Fatalf("blocksToMarkdown: %v", err)
|
||||
}
|
||||
if len(atts) != 0 {
|
||||
t.Errorf("want 0 attachments, got %d", len(atts))
|
||||
}
|
||||
want := "# 标题\n\n一段正文\n\n- 要点\n"
|
||||
if string(md) != want {
|
||||
t.Errorf("md =\n%q\nwant\n%q", md, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlocksToMarkdown_BlankDocRendersEmpty(t *testing.T) {
|
||||
// A page with no renderable children (or only empty-text/container blocks)
|
||||
// must render to empty Markdown. FetchDocxWithBlocks relies on this: an empty
|
||||
// render triggers the export fallback instead of emitting a content-less main
|
||||
// item that would wrongly ingest the login-gated wiki URL. Any File/Image
|
||||
// block writes a placeholder, so a truly empty render also implies no
|
||||
// attachment/image downdrill is lost by that fallback.
|
||||
blocks := []DocxBlock{
|
||||
{BlockID: "root", BlockType: BlockTypePage, Children: []string{"p"}},
|
||||
{BlockID: "p", BlockType: BlockTypeText, Text: txt("")},
|
||||
}
|
||||
md, atts, err := blocksToMarkdown(context.Background(), nil, blocks)
|
||||
if err != nil {
|
||||
t.Fatalf("blocksToMarkdown: %v", err)
|
||||
}
|
||||
if strings.TrimSpace(string(md)) != "" {
|
||||
t.Errorf("blank doc should render empty Markdown, got:\n%q", md)
|
||||
}
|
||||
if len(atts) != 0 {
|
||||
t.Errorf("blank doc should collect no attachments, got %d", len(atts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlocksToMarkdown_TodoAndCallout(t *testing.T) {
|
||||
blocks := []DocxBlock{
|
||||
{BlockID: "root", BlockType: BlockTypePage},
|
||||
{BlockID: "t", BlockType: BlockTypeTodo, Todo: txt("买牛奶")},
|
||||
{BlockID: "c", BlockType: BlockTypeCallout, Callout: txt("注意事项")},
|
||||
}
|
||||
md, _, err := blocksToMarkdown(context.Background(), nil, blocks)
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(md), "- [ ] 买牛奶") {
|
||||
t.Errorf("todo not rendered:\n%s", md)
|
||||
}
|
||||
if !strings.Contains(string(md), "> 注意事项") {
|
||||
t.Errorf("callout direct text not rendered:\n%s", md)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlocksToMarkdown_CalloutContainerNoOp(t *testing.T) {
|
||||
// A callout with no direct text (container form) must Emit nothing itself.
|
||||
blocks := []DocxBlock{
|
||||
{BlockID: "root", BlockType: BlockTypePage},
|
||||
{BlockID: "c", BlockType: BlockTypeCallout},
|
||||
}
|
||||
md, _, err := blocksToMarkdown(context.Background(), nil, blocks)
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if strings.TrimSpace(string(md)) != "" {
|
||||
t.Errorf("empty callout should Emit nothing, got:\n%q", md)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package core
|
||||
|
||||
import "github.com/Tencent/WeKnora/internal/types"
|
||||
|
||||
// Open Platform API origins. Feishu and Lark are the same product deployed on
|
||||
// two isolated clouds; the wiki/docx/drive APIs this connector uses are
|
||||
// identical, only the host differs.
|
||||
const (
|
||||
feishuOpenBaseURL = "https://open.feishu.cn"
|
||||
larkOpenBaseURL = "https://open.larksuite.com"
|
||||
)
|
||||
|
||||
// Web origins used to build human-facing links to wiki spaces and nodes. These
|
||||
// are the end-user app hosts, not the API hosts.
|
||||
const (
|
||||
feishuWebBaseURL = "https://feishu.cn"
|
||||
larkWebBaseURL = "https://larksuite.com"
|
||||
)
|
||||
|
||||
// Region selects which cloud the connector syncs from. Apps, tenants, tokens
|
||||
// and document tokens are scoped to a single cloud and are never valid on the
|
||||
// other, so a Region is fixed per connector instance at registration.
|
||||
type Region struct {
|
||||
// ConnectorType is the types.ConnectorType* identifier this instance serves.
|
||||
ConnectorType string
|
||||
// OpenBaseURL is the Open Platform API origin, without a trailing slash.
|
||||
// It is only the default: a data source may override it via base_url.
|
||||
OpenBaseURL string
|
||||
// WebBaseURL is the app origin used to build wiki links shown to users.
|
||||
WebBaseURL string
|
||||
// Label prefixes log lines so operators can tell the clouds apart.
|
||||
Label string
|
||||
}
|
||||
|
||||
var (
|
||||
// RegionFeishu is the Chinese mainland cloud (飞书).
|
||||
RegionFeishu = Region{
|
||||
ConnectorType: types.ConnectorTypeFeishu,
|
||||
OpenBaseURL: feishuOpenBaseURL,
|
||||
WebBaseURL: feishuWebBaseURL,
|
||||
Label: "Feishu",
|
||||
}
|
||||
|
||||
// RegionLark is the international cloud (Lark).
|
||||
RegionLark = Region{
|
||||
ConnectorType: types.ConnectorTypeLark,
|
||||
OpenBaseURL: larkOpenBaseURL,
|
||||
WebBaseURL: larkWebBaseURL,
|
||||
Label: "Lark",
|
||||
}
|
||||
|
||||
// RegionFeishuDrive is the Chinese mainland cloud, Drive (云盘) mode.
|
||||
// Shares the feishu connector package with RegionFeishu; only the connector
|
||||
// type differs so the registry dispatches to the Drive connector.
|
||||
RegionFeishuDrive = Region{
|
||||
ConnectorType: types.ConnectorTypeFeishuDrive,
|
||||
OpenBaseURL: feishuOpenBaseURL,
|
||||
WebBaseURL: feishuWebBaseURL,
|
||||
Label: "FeishuDrive",
|
||||
}
|
||||
|
||||
// RegionLarkDrive is the international cloud, Drive mode.
|
||||
RegionLarkDrive = Region{
|
||||
ConnectorType: types.ConnectorTypeLarkDrive,
|
||||
OpenBaseURL: larkOpenBaseURL,
|
||||
WebBaseURL: larkWebBaseURL,
|
||||
Label: "LarkDrive",
|
||||
}
|
||||
)
|
||||
|
||||
// WikiURL builds the user-facing link to a wiki space or node on this cloud.
|
||||
// The token is either a space_id or a node_token; both live under /wiki/.
|
||||
func (r Region) WikiURL(token string) string {
|
||||
return r.WebBaseURL + "/wiki/" + token
|
||||
}
|
||||
|
||||
// DriveFolderURL builds the user-facing link to a Drive folder on this cloud.
|
||||
// A folder_token lives under /drive/folder/.
|
||||
func (r Region) DriveFolderURL(folderToken string) string {
|
||||
return r.WebBaseURL + "/drive/folder/" + folderToken
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestRegions_AreDistinct(t *testing.T) {
|
||||
if RegionFeishu.ConnectorType == RegionLark.ConnectorType {
|
||||
t.Errorf("regions share ConnectorType %q", RegionFeishu.ConnectorType)
|
||||
}
|
||||
if RegionFeishu.OpenBaseURL == RegionLark.OpenBaseURL {
|
||||
t.Errorf("regions share OpenBaseURL %q", RegionFeishu.OpenBaseURL)
|
||||
}
|
||||
if RegionFeishu.WebBaseURL == RegionLark.WebBaseURL {
|
||||
t.Errorf("regions share WebBaseURL %q", RegionFeishu.WebBaseURL)
|
||||
}
|
||||
if RegionFeishu.ConnectorType != types.ConnectorTypeFeishu {
|
||||
t.Errorf("RegionFeishu.ConnectorType = %q, want %q", RegionFeishu.ConnectorType, types.ConnectorTypeFeishu)
|
||||
}
|
||||
if RegionLark.ConnectorType != types.ConnectorTypeLark {
|
||||
t.Errorf("RegionLark.ConnectorType = %q, want %q", RegionLark.ConnectorType, types.ConnectorTypeLark)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConnectorType_FollowsRegion lives in package wiki (it exercises
|
||||
// wiki.NewConnector); see wiki/connector_test.go TestConnectorType.
|
||||
|
||||
// The wiki link shown in the resource picker must point at the cloud the data
|
||||
// actually lives on. Feishu's value is the pre-existing one and must not drift.
|
||||
func TestRegion_WikiURL(t *testing.T) {
|
||||
if got, want := RegionFeishu.WikiURL("spc123"), "https://feishu.cn/wiki/spc123"; got != want {
|
||||
t.Errorf("Feishu WikiURL = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := RegionLark.WikiURL("spc123"), "https://larksuite.com/wiki/spc123"; got != want {
|
||||
t.Errorf("Lark WikiURL = %q, want %q", got, want)
|
||||
}
|
||||
// A Lark link must never point at the Feishu host — that was the bug.
|
||||
if strings.Contains(RegionLark.WikiURL("x"), "feishu") {
|
||||
t.Errorf("Lark WikiURL leaks the Feishu host: %q", RegionLark.WikiURL("x"))
|
||||
}
|
||||
}
|
||||
|
||||
// base_url stays an override so data sources created before the lark connector
|
||||
// existed — a "feishu" connector pointed at open.larksuite.com — keep working.
|
||||
func TestParseFeishuConfig_BaseURLDefaultsToRegion(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
region Region
|
||||
baseURL string
|
||||
want string
|
||||
}{
|
||||
{"feishu default", RegionFeishu, "", "https://open.feishu.cn"},
|
||||
{"lark default", RegionLark, "", "https://open.larksuite.com"},
|
||||
{"explicit override wins on feishu", RegionFeishu, "https://open.larksuite.com", "https://open.larksuite.com"},
|
||||
{"explicit override wins on lark", RegionLark, "https://open.feishu.cn", "https://open.feishu.cn"},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
creds := map[string]interface{}{"app_id": "cli_x", "app_secret": "s"}
|
||||
if c.baseURL != "" {
|
||||
creds["base_url"] = c.baseURL
|
||||
}
|
||||
cfg, err := ParseFeishuConfig(&types.DataSourceConfig{Credentials: creds}, c.region)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFeishuConfig: %v", err)
|
||||
}
|
||||
if got := cfg.GetBaseURL(); got != c.want {
|
||||
t.Errorf("GetBaseURL() = %q, want %q", got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,502 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
const FeishuWikiNodeResourceSeparator = ":"
|
||||
|
||||
// shared.go holds the helpers used by BOTH the wiki Connector (connector.go)
|
||||
// and the Drive DriveConnector (connector.go): error classification,
|
||||
// config parsing, stream-Checkpoint tuning, the fetch tally, filename/time
|
||||
// utilities, attachment rules, and the docx blocks fetch path. Anything that
|
||||
// is specific to one connector stays in that connector's own file.
|
||||
|
||||
// FeishuStreamCheckpointInterval is how many processed nodes pass between
|
||||
// cursor checkpoints during a streaming fetch. Small enough that a timed-out
|
||||
// sync loses little work on resume, large enough that Checkpoint persistence
|
||||
// (a DB write) does not dominate. Overridable in tests. See FetchStream.
|
||||
var FeishuStreamCheckpointInterval = 50
|
||||
|
||||
// FeishuStreamCheckpointMaxInterval bounds checkpointing by wall-clock time as
|
||||
// well as node count. Without it, a sync of fewer than
|
||||
// FeishuStreamCheckpointInterval very slow (rate-limited) exports could reach
|
||||
// the 2h task timeout having never checkpointed, and resume from scratch every
|
||||
// retry — the #2136 "never fully syncs" case. Overridable in tests.
|
||||
var FeishuStreamCheckpointMaxInterval = 30 * time.Second
|
||||
|
||||
// fetchTally accumulates the outcome of fetching a wiki node subtree so the
|
||||
// connector can Emit a single actionable summary. Without it, unsupported nodes
|
||||
// (mindnote/slides/etc.) vanish with no item, no error and no log, leaving users
|
||||
// unable to explain why "13 documents synced only 3" (Tencent/WeKnora#2136).
|
||||
type fetchTally struct {
|
||||
discovered int
|
||||
fetched int
|
||||
failed int
|
||||
skippedByType map[string]int
|
||||
}
|
||||
|
||||
func newFetchTally(discovered int) *fetchTally {
|
||||
return &fetchTally{discovered: discovered, skippedByType: map[string]int{}}
|
||||
}
|
||||
|
||||
func (t *fetchTally) fetch() { t.fetched++ }
|
||||
func (t *fetchTally) fail() { t.failed++ }
|
||||
func (t *fetchTally) Skip(objType string) { t.skippedByType[objType]++ }
|
||||
|
||||
func (t *fetchTally) skipped() int {
|
||||
n := 0
|
||||
for _, c := range t.skippedByType {
|
||||
n += c
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (t *fetchTally) summary() string {
|
||||
return fmt.Sprintf("discovered=%d fetched=%d failed=%d skipped_unsupported=%d by_type=%v",
|
||||
t.discovered, t.fetched, t.failed, t.skipped(), t.skippedByType)
|
||||
}
|
||||
|
||||
var reFeishuErrorCode = regexp.MustCompile(`code["\s]*[:=]\s*(\d+)`)
|
||||
|
||||
// feishuErrorCode extracts the numeric Feishu error code from a raw error string
|
||||
// (e.g. `body={"code":1663,...}` or `code=1663`), best-effort.
|
||||
func feishuErrorCode(raw string) string {
|
||||
if m := reFeishuErrorCode.FindStringSubmatch(raw); len(m) == 2 {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// feishuFailure classifies a raw connector/API error into a stable i18n code
|
||||
// (mapped to a localized string on the frontend), an optional numeric Feishu
|
||||
// error code for interpolation, and an English fallback message for clients
|
||||
// without the i18n key. The raw status/JSON body/log_id is never returned here —
|
||||
// it stays in the server logs. Dumping it in the UI is the anti-pattern
|
||||
// Airbyte/Fivetran/Onyx warn against. Transient errors are retried next sync
|
||||
// (the cursor is retained); auth/permission errors point at the fix instead.
|
||||
func feishuFailure(err error) (code, codeValue, fallback string) {
|
||||
if err == nil {
|
||||
return "sync_failed", "", "Sync failed; will retry on the next sync"
|
||||
}
|
||||
s := strings.ToLower(err.Error())
|
||||
|
||||
switch {
|
||||
case strings.Contains(s, "auth error"),
|
||||
strings.Contains(s, "invalid access token"),
|
||||
strings.Contains(s, "permission"),
|
||||
strings.Contains(s, "forbidden"),
|
||||
strings.Contains(s, "status=403"):
|
||||
return "feishu_auth_or_permission", "", "Authentication or permission error; check credentials and app scopes"
|
||||
case strings.Contains(s, "rate limited"), strings.Contains(s, "status=429"):
|
||||
return "feishu_rate_limited", "", "Feishu API rate limited; will retry on the next sync"
|
||||
case strings.Contains(s, "timed out"),
|
||||
strings.Contains(s, "timeout"),
|
||||
strings.Contains(s, "deadline exceeded"):
|
||||
return "feishu_timeout", "", "Export or request timed out; will retry on the next sync"
|
||||
case strings.Contains(s, "server error"):
|
||||
return "feishu_server_unavailable", "", "Feishu service temporarily unavailable; will retry on the next sync"
|
||||
case strings.Contains(s, "api error"),
|
||||
strings.Contains(s, "export task failed"),
|
||||
strings.Contains(s, "download failed"):
|
||||
if v := feishuErrorCode(err.Error()); v != "" {
|
||||
return "feishu_api_error", v, fmt.Sprintf("Feishu API error (code=%s); will retry on the next sync", v)
|
||||
}
|
||||
return "feishu_api_error_generic", "", "Feishu API error; will retry on the next sync"
|
||||
default:
|
||||
return "sync_failed", "", "Sync failed; will retry on the next sync"
|
||||
}
|
||||
}
|
||||
|
||||
// FeishuErrorItemMeta builds the metadata for a failed item: the raw error (for
|
||||
// server logs) plus the classified i18n code / param / fallback (for a
|
||||
// localisable SyncItemError in the UI), merged with any caller-supplied extras.
|
||||
func FeishuErrorItemMeta(err error, extra map[string]string) map[string]string {
|
||||
code, codeValue, fallback := feishuFailure(err)
|
||||
m := map[string]string{
|
||||
"error": err.Error(),
|
||||
"error_reason_code": code,
|
||||
"error_reason": fallback,
|
||||
}
|
||||
if codeValue != "" {
|
||||
m["error_reason_code_value"] = codeValue
|
||||
}
|
||||
maps.Copy(m, extra)
|
||||
return m
|
||||
}
|
||||
|
||||
// parseableAttachmentExts are attachment extensions worth ingesting as their
|
||||
// own knowledge entries; other files (icons, tiny decor) are skipped.
|
||||
var parseableAttachmentExts = map[string]bool{
|
||||
".pdf": true, ".doc": true, ".docx": true, ".xls": true, ".xlsx": true,
|
||||
".ppt": true, ".pptx": true, ".txt": true, ".md": true, ".csv": true,
|
||||
}
|
||||
|
||||
// MinAttachmentBytes filters out decorative micro-files.
|
||||
const MinAttachmentBytes = 2 * 1024
|
||||
|
||||
// SupportedImageExt sniffs image bytes and returns the filename extension and
|
||||
// content type WeKnora accepts for a standalone image knowledge item (png/jpg/
|
||||
// gif — the image set isValidFileType admits). ok is false for non-image or
|
||||
// unsupported formats (e.g. webp/bmp), which the caller skips rather than
|
||||
// mislabel — a wrong extension would fail parsing. The detected content type is
|
||||
// returned even when ok is false so the caller can log it without re-sniffing.
|
||||
func SupportedImageExt(data []byte) (ext, contentType string, ok bool) {
|
||||
switch ct := http.DetectContentType(data); ct {
|
||||
case "image/png":
|
||||
return ".png", ct, true
|
||||
case "image/jpeg":
|
||||
return ".jpg", ct, true
|
||||
case "image/gif":
|
||||
return ".gif", ct, true
|
||||
default:
|
||||
return "", ct, false
|
||||
}
|
||||
}
|
||||
|
||||
// ParseFeishuConfig extracts and validates Feishu/Lark-specific configuration.
|
||||
//
|
||||
// base_url stays an explicit override so existing data sources that pointed a
|
||||
// "feishu" connector at open.larksuite.com keep working; when it is unset the
|
||||
// region's own host is filled in, making the resolved Config.BaseURL concrete
|
||||
// for everything downstream.
|
||||
func ParseFeishuConfig(config *types.DataSourceConfig, region Region) (*Config, error) {
|
||||
if config == nil {
|
||||
return nil, fmt.Errorf("config is nil")
|
||||
}
|
||||
|
||||
credBytes, err := json.Marshal(config.Credentials)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal credentials: %w", err)
|
||||
}
|
||||
|
||||
var feishuConfig Config
|
||||
if err := json.Unmarshal(credBytes, &feishuConfig); err != nil {
|
||||
return nil, fmt.Errorf("parse %s credentials: %w", region.ConnectorType, err)
|
||||
}
|
||||
|
||||
if feishuConfig.AppID == "" || feishuConfig.AppSecret == "" {
|
||||
return nil, fmt.Errorf("%s app_id and app_secret are required", region.ConnectorType)
|
||||
}
|
||||
|
||||
if feishuConfig.BaseURL == "" {
|
||||
feishuConfig.BaseURL = region.OpenBaseURL
|
||||
}
|
||||
|
||||
// Timezone is a display setting (bitable date rendering), not a credential, so
|
||||
// it lives in Settings. Empty falls back to GMT+8 in resolveLocation.
|
||||
if feishuConfig.Timezone == "" && config.Settings != nil {
|
||||
if tz, ok := config.Settings["timezone"].(string); ok {
|
||||
feishuConfig.Timezone = strings.TrimSpace(tz)
|
||||
}
|
||||
}
|
||||
|
||||
if err := datasource.ValidateConnectorBaseURL(feishuConfig.GetBaseURL()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &feishuConfig, nil
|
||||
}
|
||||
|
||||
// IsSupportedDocType checks if a Feishu document type can be synced.
|
||||
// mindnote and slides have no content read API and are skipped.
|
||||
func IsSupportedDocType(objType string) bool {
|
||||
switch objType {
|
||||
case "docx", "doc", "sheet", "bitable", "file":
|
||||
return true
|
||||
default:
|
||||
// mindnote, slides — no content retrieval API available
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// ParseFeishuTimestamp parses a Feishu unix timestamp string (seconds) into time.Time.
|
||||
func ParseFeishuTimestamp(ts string) time.Time {
|
||||
if ts == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
sec, err := strconv.ParseInt(ts, 10, 64)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Unix(sec, 0)
|
||||
}
|
||||
|
||||
// SanitizeFileName removes characters that are invalid in filenames and
|
||||
// truncates at a UTF-8 rune boundary. Raw byte truncation would split a
|
||||
// multi-byte codepoint (Chinese chars are 3 bytes) and produce invalid UTF-8
|
||||
// that downstream validation (utf8.ValidString) rejects.
|
||||
//
|
||||
// The extension is preserved across truncation: only the base name is trimmed,
|
||||
// so a long attachment name like "很长的名字….pdf" keeps its ".pdf" suffix that
|
||||
// downstream file-type classification depends on.
|
||||
func SanitizeFileName(name string) string {
|
||||
if name == "" {
|
||||
return "untitled"
|
||||
}
|
||||
replacer := strings.NewReplacer(
|
||||
"/", "_", "\\", "_", ":", "_", "*", "_",
|
||||
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
|
||||
)
|
||||
result := replacer.Replace(name)
|
||||
const maxBytes = 200
|
||||
if len(result) <= maxBytes {
|
||||
return result
|
||||
}
|
||||
ext := filepath.Ext(result)
|
||||
if len(ext) >= maxBytes {
|
||||
// pathological: extension alone overflows the budget → drop it
|
||||
ext = ""
|
||||
}
|
||||
base := truncateUTF8(result[:len(result)-len(ext)], maxBytes-len(ext))
|
||||
return base + ext
|
||||
}
|
||||
|
||||
// truncateUTF8 shortens s to at most maxBytes bytes without splitting a
|
||||
// multi-byte rune: after a hard byte cut it trims any trailing partial codepoint.
|
||||
func truncateUTF8(s string, maxBytes int) string {
|
||||
if len(s) <= maxBytes {
|
||||
return s
|
||||
}
|
||||
s = s[:maxBytes]
|
||||
for len(s) > 0 {
|
||||
r, size := utf8.DecodeLastRuneInString(s)
|
||||
if r != utf8.RuneError || size != 1 {
|
||||
break
|
||||
}
|
||||
s = s[:len(s)-1]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// DocxFetchInput is the unified description of one docx document from either
|
||||
// source (wiki node or Drive file) that FetchDocxWithBlocks needs.
|
||||
type DocxFetchInput struct {
|
||||
// WeKnora external_id: wiki=node.NodeToken, drive=file.Token
|
||||
DocToken string
|
||||
// Feishu docx document token
|
||||
ObjToken string
|
||||
Title string
|
||||
URL string
|
||||
ResourceID string
|
||||
EditTime time.Time
|
||||
BaseMeta map[string]string
|
||||
MultimodalEnabled bool
|
||||
}
|
||||
|
||||
// FetchDocxWithBlocks retrieves a docx document via the blocks API, converts it
|
||||
// to Markdown, and returns a main item plus any parseable attachment/image
|
||||
// sub-items. Falls back to the export API if the blocks API errors or renders
|
||||
// empty. Shared by the wiki Connector and the Drive DriveConnector.
|
||||
func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput) ([]*types.FetchedItem, error) {
|
||||
// FEISHU_DOCX_PARSE_MODE selects the docx parsing path. The blocks path
|
||||
// renders image blocks as empty `![图片]()` placeholders and fans images out
|
||||
// into separate knowledge items, which breaks image↔document association in
|
||||
// retrieval/wiki/agent. The export path yields a .docx that docreader parses
|
||||
// inline, so images are bound to the parent document via parent_chunk_id
|
||||
// (same as a regular docx upload). Default (unset / "export") uses export so
|
||||
// images associate with the document; set "blocks" for the blocks-first
|
||||
// behaviour (faster, keeps docx attachments, but images are detached).
|
||||
|
||||
// This is a temporary solution. If a better parsing solution is available later, this environment variable will be removed and replaced with a better one.
|
||||
parsingMode := strings.TrimSpace(os.Getenv("FEISHU_DOCX_PARSE_MODE"))
|
||||
if parsingMode == "" {
|
||||
parsingMode = "export"
|
||||
}
|
||||
|
||||
if strings.EqualFold(parsingMode, "export") {
|
||||
item, err := exportDocxFallback(ctx, client, in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []*types.FetchedItem{item}, nil
|
||||
}
|
||||
|
||||
blocks, err := client.listDocumentBlocks(ctx, in.ObjToken)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[Feishu] blocks API failed for %s (%s), falling back to export: %v",
|
||||
in.Title, in.ObjToken, err)
|
||||
item, ferr := exportDocxFallback(ctx, client, in)
|
||||
if ferr != nil {
|
||||
return nil, ferr
|
||||
}
|
||||
// Do NOT set ReplacesSubtree here (see the wiki history: a transient
|
||||
// blocks failure must not sweep good attachment children from the prior
|
||||
// blocks-path sync with nothing to replace them).
|
||||
return []*types.FetchedItem{item}, nil
|
||||
}
|
||||
|
||||
md, atts, err := blocksToMarkdown(ctx, client, blocks)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("convert blocks %s: %w", in.Title, err)
|
||||
}
|
||||
|
||||
if len(strings.TrimSpace(string(md))) == 0 {
|
||||
logger.Infof(ctx, "[Feishu] doc %s (%s): blocks rendered empty Markdown, falling back to export",
|
||||
in.Title, in.ObjToken)
|
||||
item, ferr := exportDocxFallback(ctx, client, in)
|
||||
if ferr != nil {
|
||||
return nil, ferr
|
||||
}
|
||||
return []*types.FetchedItem{item}, nil
|
||||
}
|
||||
|
||||
main := &types.FetchedItem{
|
||||
ExternalID: in.DocToken,
|
||||
Title: in.Title,
|
||||
Content: md,
|
||||
ContentType: "text/markdown",
|
||||
FileName: SanitizeFileName(in.Title) + ".md",
|
||||
URL: in.URL,
|
||||
UpdatedAt: in.EditTime,
|
||||
SourceResourceID: in.ResourceID,
|
||||
Metadata: in.BaseMeta,
|
||||
ReplacesSubtree: true, // sweep stale attachment sub-items on re-sync
|
||||
}
|
||||
items := []*types.FetchedItem{main}
|
||||
|
||||
keep := make([]string, 0, len(atts))
|
||||
childMeta := func() map[string]string {
|
||||
m := maps.Clone(in.BaseMeta)
|
||||
m["parent_node_token"] = in.DocToken
|
||||
m["attachment"] = "true"
|
||||
return m
|
||||
}
|
||||
for _, a := range atts {
|
||||
childID := types.SubtreeChildID(in.DocToken, "file", a.FileToken)
|
||||
keep = append(keep, childID) // present in the doc → never sweep as stale
|
||||
ext := strings.ToLower(filepath.Ext(a.Name))
|
||||
if ext == "" {
|
||||
logger.Warnf(ctx, "[Feishu] doc %s: skipping attachment with no usable filename (token=%s name=%q)",
|
||||
in.ObjToken, a.FileToken, a.Name)
|
||||
continue
|
||||
}
|
||||
if !parseableAttachmentExts[ext] {
|
||||
continue
|
||||
}
|
||||
data, derr := client.downloadMediaFile(ctx, a.FileToken)
|
||||
if derr != nil {
|
||||
logger.Warnf(ctx, "[Feishu] doc %s: attachment %q (token=%s) download failed: %v",
|
||||
in.ObjToken, a.Name, a.FileToken, derr)
|
||||
items = append(items, &types.FetchedItem{
|
||||
ExternalID: childID,
|
||||
Title: a.Name,
|
||||
SourceResourceID: in.ResourceID,
|
||||
Metadata: FeishuErrorItemMeta(derr, childMeta()),
|
||||
})
|
||||
continue
|
||||
}
|
||||
if len(data) < MinAttachmentBytes {
|
||||
logger.Infof(ctx, "[Feishu] doc %s: skipping tiny attachment %q (token=%s, %d bytes < %d)",
|
||||
in.ObjToken, a.Name, a.FileToken, len(data), MinAttachmentBytes)
|
||||
continue
|
||||
}
|
||||
items = append(items, &types.FetchedItem{
|
||||
ExternalID: childID,
|
||||
Title: a.Name,
|
||||
Content: data,
|
||||
ContentType: "application/octet-stream",
|
||||
FileName: SanitizeFileName(a.Name),
|
||||
URL: in.URL,
|
||||
UpdatedAt: in.EditTime,
|
||||
SourceResourceID: in.ResourceID,
|
||||
Metadata: childMeta(),
|
||||
})
|
||||
}
|
||||
|
||||
imgMeta := func() map[string]string {
|
||||
m := maps.Clone(in.BaseMeta)
|
||||
m["parent_node_token"] = in.DocToken
|
||||
m["embedded_image"] = "true"
|
||||
return m
|
||||
}
|
||||
for _, b := range blocks {
|
||||
if b.BlockType != BlockTypeImage || b.Image == nil || b.Image.Token == "" {
|
||||
continue
|
||||
}
|
||||
childID := types.SubtreeChildID(in.DocToken, "image", b.Image.Token)
|
||||
keep = append(keep, childID) // present in the doc → never sweep as stale
|
||||
if !in.MultimodalEnabled {
|
||||
continue // KB can't OCR images; the inline placeholder is all we keep
|
||||
}
|
||||
data, derr := client.downloadMediaFile(ctx, b.Image.Token)
|
||||
if derr != nil {
|
||||
logger.Warnf(ctx, "[Feishu] doc %s: image (token=%s) download failed: %v",
|
||||
in.ObjToken, b.Image.Token, derr)
|
||||
items = append(items, &types.FetchedItem{
|
||||
ExternalID: childID,
|
||||
Title: fmt.Sprintf("%s(内嵌图片)", in.Title),
|
||||
SourceResourceID: in.ResourceID,
|
||||
Metadata: FeishuErrorItemMeta(derr, imgMeta()),
|
||||
})
|
||||
continue
|
||||
}
|
||||
if len(data) < MinAttachmentBytes {
|
||||
continue // decorative micro-image (icon/spacer)
|
||||
}
|
||||
ext, contentType, ok := SupportedImageExt(data)
|
||||
if !ok {
|
||||
logger.Warnf(ctx, "[Feishu] doc %s: skipping image (token=%s) of unsupported type %q",
|
||||
in.ObjToken, b.Image.Token, contentType)
|
||||
continue
|
||||
}
|
||||
items = append(items, &types.FetchedItem{
|
||||
ExternalID: childID,
|
||||
Title: fmt.Sprintf("%s(内嵌图片)", in.Title),
|
||||
Content: data,
|
||||
ContentType: contentType,
|
||||
FileName: "image-" + b.Image.Token + ext,
|
||||
URL: in.URL,
|
||||
UpdatedAt: in.EditTime,
|
||||
SourceResourceID: in.ResourceID,
|
||||
Metadata: imgMeta(),
|
||||
})
|
||||
}
|
||||
|
||||
main.SubtreeKeep = keep
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// exportDocxFallback exports a docx document via the async export API and
|
||||
// returns a single FetchedItem containing the exported .docx binary. Used by
|
||||
// FetchDocxWithBlocks when the blocks API is unavailable or renders empty.
|
||||
func exportDocxFallback(ctx context.Context, client *Client, in DocxFetchInput) (*types.FetchedItem, error) {
|
||||
data, fileName, err := client.ExportAndDownload(ctx, in.ObjToken, "docx")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("export %s (docx): %w", in.Title, err)
|
||||
}
|
||||
|
||||
ext := ExportFileExtToSuffix[ObjTypeToExportFileExtension["docx"]]
|
||||
if fileName == "" {
|
||||
fileName = SanitizeFileName(in.Title) + ext
|
||||
} else if !strings.HasSuffix(strings.ToLower(fileName), ext) {
|
||||
fileName = SanitizeFileName(fileName) + ext
|
||||
}
|
||||
|
||||
return &types.FetchedItem{
|
||||
ExternalID: in.DocToken,
|
||||
Title: in.Title,
|
||||
Content: data,
|
||||
ContentType: "application/octet-stream",
|
||||
FileName: fileName,
|
||||
URL: in.URL,
|
||||
UpdatedAt: in.EditTime,
|
||||
SourceResourceID: in.ResourceID,
|
||||
Metadata: in.BaseMeta,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFetchTally_CountsAndSummary(t *testing.T) {
|
||||
tally := newFetchTally(13)
|
||||
tally.fetch()
|
||||
tally.fetch()
|
||||
tally.fetch()
|
||||
tally.Skip("mindnote")
|
||||
tally.Skip("mindnote")
|
||||
tally.Skip("slides")
|
||||
tally.fail()
|
||||
|
||||
if got := tally.skipped(); got != 3 {
|
||||
t.Errorf("skipped() = %d, want 3", got)
|
||||
}
|
||||
|
||||
summary := tally.summary()
|
||||
for _, want := range []string{
|
||||
"discovered=13",
|
||||
"fetched=3",
|
||||
"failed=1",
|
||||
"skipped_unsupported=3",
|
||||
"mindnote:2",
|
||||
"slides:1",
|
||||
} {
|
||||
if !strings.Contains(summary, want) {
|
||||
t.Errorf("summary() = %q, missing %q", summary, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchTally_EmptyHasNoSkips(t *testing.T) {
|
||||
tally := newFetchTally(0)
|
||||
if got := tally.skipped(); got != 0 {
|
||||
t.Errorf("skipped() = %d, want 0", got)
|
||||
}
|
||||
if !strings.Contains(tally.summary(), "skipped_unsupported=0") {
|
||||
t.Errorf("summary() = %q, want skipped_unsupported=0", tally.summary())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
// Package feishu implements the Feishu (飞书/Lark) data source connector for WeKnora.
|
||||
//
|
||||
// It syncs documents from Feishu Wiki spaces and cloud documents into WeKnora knowledge bases.
|
||||
//
|
||||
// Feishu API docs:
|
||||
// - Wiki spaces: https://open.feishu.cn/document/server-docs/docs/wiki-v2/space/list
|
||||
// - Wiki nodes: https://open.feishu.cn/document/server-docs/docs/wiki-v2/space-node/list
|
||||
// - Export tasks: https://open.feishu.cn/document/server-docs/docs/drive-v1/export_task/export-user-guide
|
||||
// - File download: https://open.feishu.cn/document/server-docs/docs/drive-v1/file/download
|
||||
// - Auth: https://open.feishu.cn/document/server-docs/authentication-management/access-token/tenant_access_token_internal
|
||||
package core
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Config holds Feishu-specific configuration for the data source connector.
|
||||
// Uses the self-built app (企业自建应用) authentication model.
|
||||
type Config struct {
|
||||
// App ID from Feishu developer console
|
||||
AppID string `json:"app_id"`
|
||||
|
||||
// App Secret from Feishu developer console
|
||||
AppSecret string `json:"app_secret"`
|
||||
|
||||
// Base URL for Feishu API (default: https://open.feishu.cn)
|
||||
// Use https://open.larksuite.com for Lark (international) deployments
|
||||
BaseURL string `json:"base_url,omitempty"`
|
||||
|
||||
// Timezone is the IANA name (e.g. "Asia/Shanghai", "America/New_York") used to
|
||||
// render bitable date cells. Feishu stores a date as a UTC instant, but the
|
||||
// calendar date a user sees is that instant in the table's timezone, so
|
||||
// rendering in UTC shifts the date. Empty defaults to GMT+8, which matches
|
||||
// Feishu (mainland) tenants; set it for Lark tenants in other zones.
|
||||
Timezone string `json:"timezone,omitempty"`
|
||||
}
|
||||
|
||||
// defaultTimezoneOffsetSeconds is GMT+8 (the Feishu mainland default), used when
|
||||
// no timezone is configured. A fixed zone avoids any dependency on system tzdata,
|
||||
// which minimal container images may omit.
|
||||
const defaultTimezoneOffsetSeconds = 8 * 3600
|
||||
|
||||
// resolveLocation returns the *time.Location used to render bitable date cells.
|
||||
// An empty name yields a fixed GMT+8 zone (no tzdata dependency); a named zone is
|
||||
// loaded from the system tz database, falling back to GMT+8 if it is unavailable.
|
||||
func resolveLocation(name string) *time.Location {
|
||||
if name != "" {
|
||||
if loc, err := time.LoadLocation(name); err == nil {
|
||||
return loc
|
||||
}
|
||||
}
|
||||
return time.FixedZone("GMT+8", defaultTimezoneOffsetSeconds)
|
||||
}
|
||||
|
||||
// defaultBaseURL is the default Feishu Open Platform API base URL.
|
||||
const defaultBaseURL = "https://open.feishu.cn"
|
||||
|
||||
// larkBaseURL is the Lark (international) API base URL.
|
||||
const larkBaseURL = "https://open.larksuite.com"
|
||||
|
||||
// GetBaseURL returns the effective base URL, defaulting to Feishu if not set.
|
||||
func (c *Config) GetBaseURL() string {
|
||||
if c.BaseURL != "" {
|
||||
return c.BaseURL
|
||||
}
|
||||
return defaultBaseURL
|
||||
}
|
||||
|
||||
// --- Export format constants ---
|
||||
// Used by the export task API: POST /drive/v1/export_tasks
|
||||
|
||||
const (
|
||||
// ExportTypeDocx exports Feishu documents to .docx format.
|
||||
ExportTypeDocx = "docx"
|
||||
// ExportTypeXlsx exports spreadsheets / bitable to .xlsx format.
|
||||
ExportTypeXlsx = "xlsx"
|
||||
// ExportTypePDF exports documents to .pdf format (fallback).
|
||||
ExportTypePDF = "pdf"
|
||||
)
|
||||
|
||||
// ObjTypeToExportFileExtension maps Feishu obj_type to the best export file_extension.
|
||||
var ObjTypeToExportFileExtension = map[string]string{
|
||||
"docx": ExportTypeDocx,
|
||||
"doc": ExportTypeDocx,
|
||||
"sheet": ExportTypeXlsx,
|
||||
"bitable": ExportTypeXlsx,
|
||||
}
|
||||
|
||||
// ObjTypeToExportType maps Feishu obj_type to the export API "type" parameter.
|
||||
// See: https://open.feishu.cn/document/server-docs/docs/drive-v1/export_task/create
|
||||
var ObjTypeToExportType = map[string]string{
|
||||
"docx": "docx",
|
||||
"doc": "doc",
|
||||
"sheet": "sheet",
|
||||
"bitable": "bitable",
|
||||
}
|
||||
|
||||
// ExportFileExtToSuffix maps export file_extension to the file suffix for FileName.
|
||||
var ExportFileExtToSuffix = map[string]string{
|
||||
ExportTypeDocx: ".docx",
|
||||
ExportTypeXlsx: ".xlsx",
|
||||
ExportTypePDF: ".pdf",
|
||||
}
|
||||
|
||||
// --- Feishu API response structures ---
|
||||
|
||||
// ApiResponse is the common Feishu API response wrapper.
|
||||
type ApiResponse struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
}
|
||||
|
||||
// TokenResponse is the response for tenant_access_token API.
|
||||
type TokenResponse struct {
|
||||
ApiResponse
|
||||
TenantAccessToken string `json:"tenant_access_token"`
|
||||
Expire int `json:"expire"` // seconds
|
||||
}
|
||||
|
||||
// WikiSpaceListData is the data payload of WikiSpaceListResponse.
|
||||
type WikiSpaceListData struct {
|
||||
Items []WikiSpace `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
}
|
||||
|
||||
// WikiSpaceListResponse is the response for GET /open-apis/wiki/v2/spaces.
|
||||
type WikiSpaceListResponse struct {
|
||||
ApiResponse
|
||||
Data WikiSpaceListData `json:"data"`
|
||||
}
|
||||
|
||||
// WikiSpace represents a Feishu Wiki space.
|
||||
type WikiSpace struct {
|
||||
SpaceID string `json:"space_id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Visibility string `json:"visibility"` // "public" or "private"
|
||||
}
|
||||
|
||||
// WikiNodeListData is the data payload of WikiNodeListResponse.
|
||||
type WikiNodeListData struct {
|
||||
Items []WikiNode `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
}
|
||||
|
||||
// WikiNodeListResponse is the response for GET /open-apis/wiki/v2/spaces/:space_id/nodes.
|
||||
type WikiNodeListResponse struct {
|
||||
ApiResponse
|
||||
Data WikiNodeListData `json:"data"`
|
||||
}
|
||||
|
||||
// WikiNode represents a node (document or folder) in a Feishu Wiki space.
|
||||
type WikiNode struct {
|
||||
SpaceID string `json:"space_id"`
|
||||
NodeToken string `json:"node_token"`
|
||||
ObjToken string `json:"obj_token"` // document token
|
||||
ObjType string `json:"obj_type"` // "doc", "sheet", "mindnote", "bitable", "file", "docx", "slides"
|
||||
ParentNodeID string `json:"parent_node_token"`
|
||||
NodeType string `json:"node_type"` // "origin" or "shortcut"
|
||||
OriginNodeID string `json:"origin_node_id"`
|
||||
OriginSpaceID string `json:"origin_space_id"`
|
||||
HasChild bool `json:"has_child"`
|
||||
Title string `json:"title"`
|
||||
Creator string `json:"creator"`
|
||||
Owner string `json:"owner"`
|
||||
ObjCreateTime string `json:"obj_create_time"` // document creation time (unix timestamp string)
|
||||
ObjEditTime string `json:"obj_edit_time"` // document last edit time (unix timestamp string) — tracks content changes
|
||||
NodeCreateTime string `json:"node_create_time"` // node creation time (unix timestamp string)
|
||||
NodeEditTime string `json:"node_edit_time"` // node edit time (unix timestamp string) — only tracks node attribute changes
|
||||
}
|
||||
|
||||
// WikiNodeInfoData is the data payload of WikiNodeInfoResponse.
|
||||
type WikiNodeInfoData struct {
|
||||
Node WikiNode `json:"node"`
|
||||
}
|
||||
|
||||
// WikiNodeInfoResponse is the response for GET /open-apis/wiki/v2/spaces/get_node.
|
||||
type WikiNodeInfoResponse struct {
|
||||
ApiResponse
|
||||
Data WikiNodeInfoData `json:"data"`
|
||||
}
|
||||
|
||||
// --- Export task API responses ---
|
||||
|
||||
// docRawContentData is the data payload of docRawContentResponse.
|
||||
type docRawContentData struct {
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
// docRawContentResponse is the response for GET /open-apis/docx/v1/documents/:document_id/raw_content.
|
||||
// Deprecated: prefer export API for full-fidelity document export.
|
||||
type docRawContentResponse struct {
|
||||
ApiResponse
|
||||
Data docRawContentData `json:"data"`
|
||||
}
|
||||
|
||||
// ExportTaskCreateData is the data payload of ExportTaskCreateResponse.
|
||||
type ExportTaskCreateData struct {
|
||||
Ticket string `json:"ticket"`
|
||||
}
|
||||
|
||||
// ExportTaskCreateResponse is the response for POST /drive/v1/export_tasks.
|
||||
type ExportTaskCreateResponse struct {
|
||||
ApiResponse
|
||||
Data ExportTaskCreateData `json:"data"`
|
||||
}
|
||||
|
||||
// ExportTaskResult is the per-task result inside ExportTaskStatusData.
|
||||
type ExportTaskResult struct {
|
||||
FileToken string `json:"file_token"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
// JobStatus: 0=success, 1=initializing, 2=processing
|
||||
JobStatus int `json:"job_status"`
|
||||
JobErrorMsg string `json:"job_error_msg"`
|
||||
FileName string `json:"file_name"`
|
||||
}
|
||||
|
||||
// ExportTaskStatusData is the data payload of ExportTaskStatusResponse.
|
||||
type ExportTaskStatusData struct {
|
||||
Result ExportTaskResult `json:"result"`
|
||||
}
|
||||
|
||||
// ExportTaskStatusResponse is the response for GET /drive/v1/export_tasks/{ticket}.
|
||||
type ExportTaskStatusResponse struct {
|
||||
ApiResponse
|
||||
Data ExportTaskStatusData `json:"data"`
|
||||
}
|
||||
|
||||
// --- File download response ---
|
||||
|
||||
// driveFileMeta is one entry of driveFileMetaData.Metas.
|
||||
type driveFileMeta struct {
|
||||
DocToken string `json:"doc_token"`
|
||||
DocType string `json:"doc_type"`
|
||||
Title string `json:"title"`
|
||||
}
|
||||
|
||||
// driveFileMetaData is the data payload of driveFileMetaResponse.
|
||||
type driveFileMetaData struct {
|
||||
Metas []driveFileMeta `json:"metas"`
|
||||
}
|
||||
|
||||
// driveFileMetaResponse is the response for GET /drive/v1/metas for file type nodes.
|
||||
type driveFileMetaResponse struct {
|
||||
ApiResponse
|
||||
Data driveFileMetaData `json:"data"`
|
||||
}
|
||||
|
||||
// FeishuCursor stores incremental sync state for Feishu.
|
||||
type FeishuCursor struct {
|
||||
// LastSyncTime is the timestamp of the last successful sync.
|
||||
LastSyncTime time.Time `json:"last_sync_time"`
|
||||
|
||||
// SpaceNodeTimes maps space_id -> node_token -> last known edit time.
|
||||
// Used to detect which nodes have changed since last sync.
|
||||
SpaceNodeTimes map[string]map[string]string `json:"space_node_times,omitempty"`
|
||||
}
|
||||
|
||||
// --- Drive (云盘) file listing types (feishu_drive / lark_drive connectors) ---
|
||||
// Added by feat/datasource-feishu-drive. These are independent of the wiki
|
||||
// types above and do not affect the wiki connector.
|
||||
|
||||
// DriveFile represents a file/folder in Feishu Drive (云空间). Returned by
|
||||
// GET /open-apis/drive/v1/files?folder_token=xxx. The list API returns
|
||||
// modified_time directly (verified), so no batch_query/metas call is needed for
|
||||
// incremental detection - see ADR-0002.
|
||||
type DriveFile struct {
|
||||
Token string `json:"token"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"` // doc/docx/sheet/bitable/file/folder/shortcut/mindnote/slides/board
|
||||
ParentToken string `json:"parent_token"`
|
||||
URL string `json:"url"`
|
||||
CreatedTime string `json:"created_time"` // unix seconds string
|
||||
ModifiedTime string `json:"modified_time"` // unix seconds string - 等价知识库 obj_edit_time
|
||||
OwnerID string `json:"owner_id"`
|
||||
// ShortcutInfo is populated only for type=="shortcut". target_type can only
|
||||
// be doc/sheet/mindnote/bitable/file/docx (Feishu does not allow shortcuts to
|
||||
// folders, verified) - see ADR-0002 / glossary shortcut entry.
|
||||
ShortcutInfo *driveShortcutInfo `json:"shortcut_info,omitempty"`
|
||||
}
|
||||
|
||||
// driveShortcutInfo is the target metadata of a Drive shortcut.
|
||||
type driveShortcutInfo struct {
|
||||
TargetToken string `json:"target_token"`
|
||||
TargetType string `json:"target_type"`
|
||||
}
|
||||
|
||||
// DriveFileListData is the data payload of DriveFileListResponse.
|
||||
type DriveFileListData struct {
|
||||
Files []DriveFile `json:"files"`
|
||||
HasMore bool `json:"has_more"`
|
||||
NextPageToken string `json:"next_page_token"`
|
||||
}
|
||||
|
||||
// DriveFileListResponse is the response for GET /open-apis/drive/v1/files.
|
||||
type DriveFileListResponse struct {
|
||||
ApiResponse
|
||||
Data DriveFileListData `json:"data"`
|
||||
}
|
||||
|
||||
// driveFolderMetaData is the data payload of driveFolderMetaResponse.
|
||||
type driveFolderMetaData struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Token string `json:"token"`
|
||||
CreateUid string `json:"createUid"`
|
||||
EditUid string `json:"editUid"`
|
||||
ParentID string `json:"parentId"`
|
||||
OwnUid string `json:"ownUid"`
|
||||
}
|
||||
|
||||
// driveFolderMetaResponse is the response for GET /open-apis/drive/explorer/v2/folder/:folderToken/meta.
|
||||
// Used to resolve a root folder's human-readable name (the list API only returns
|
||||
// the folder's children, not the folder itself).
|
||||
type driveFolderMetaResponse struct {
|
||||
ApiResponse
|
||||
Data driveFolderMetaData `json:"data"`
|
||||
}
|
||||
|
||||
// DriveFileListFailure records a single sub-folder listing that failed during a
|
||||
// recursive walk. Mirrors WikiNodeListFailure.
|
||||
type DriveFileListFailure struct {
|
||||
FolderToken string
|
||||
Err error
|
||||
}
|
||||
|
||||
// PartialDriveFileListError aggregates per-folder listing failures so the walk
|
||||
// can continue and the caller can still surface the partial result. Mirrors
|
||||
// PartialWikiNodeListError.
|
||||
type PartialDriveFileListError struct {
|
||||
Failures []DriveFileListFailure
|
||||
}
|
||||
|
||||
func (e *PartialDriveFileListError) Error() string {
|
||||
if e == nil || len(e.Failures) == 0 {
|
||||
return "partial drive file listing failed"
|
||||
}
|
||||
parts := make([]string, 0, len(e.Failures))
|
||||
for _, failure := range e.Failures {
|
||||
parts = append(parts, failure.Err.Error())
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
// FeishuDriveCursor stores incremental sync state for Feishu Drive (云盘).
|
||||
// Structurally symmetric with FeishuCursor: outer key = resourceID
|
||||
// ("folderToken" or "folderToken:fileToken"), inner key = file_token,
|
||||
// value = modified_time. See ADR-0001.
|
||||
type FeishuDriveCursor struct {
|
||||
// LastSyncTime is the timestamp of the last successful sync.
|
||||
LastSyncTime time.Time `json:"last_sync_time"`
|
||||
|
||||
// FileTimes maps resourceID -> file_token -> last known modified_time.
|
||||
// Used to detect which files have changed since last sync.
|
||||
FileTimes map[string]map[string]string `json:"file_times,omitempty"`
|
||||
}
|
||||
Reference in New Issue
Block a user