WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/datasource/connector/ima/client.go

@@ -0,0 +1,403 @@
+// Package ima implements the WeKnora data source connector for Tencent IMA
+// (ima.qq.com), syncing documents and files out of IMA knowledge bases via the
+// /openapi/wiki/v1 OpenAPI.
+package ima
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/datasource"
+	"github.com/Tencent/WeKnora/internal/logger"
+	"github.com/Tencent/WeKnora/internal/utils"
+)
+
+const (
+	defaultTimeout  = 60 * time.Second
+	defaultPageSize = 50
+	userAgent       = "WeKnora-IMA-Connector/1.0"
+
+	// IMA hard limit: get_knowledge_list max limit=50; search_knowledge_base max=20.
+	searchPageSize = 20
+
+	// Default per-item download timeout — IMA COS URLs can be slow for large PDFs.
+	downloadTimeout = 120 * time.Second
+
+	// Max body we accept from get_media_info-URLs. 200MB matches IMA's largest
+	maxDownloadBytes = 200 * 1024 * 1024
+)
+
+// client wraps the IMA OpenAPI. It is safe for concurrent use.
+type client struct {
+	baseURL    string
+	clientID   string
+	apiKey     string
+	httpClient *http.Client
+	// downloadClient is a separate client with a longer timeout for pulling
+	// media bodies from COS/CDN URLs returned by get_media_info.
+	downloadClient    *http.Client
+	logCredentialOnce sync.Once
+}
+
+// newClient constructs a client with a normalized base URL.
+func newClient(cfg *Config) *client {
+	return &client{
+		baseURL:        cfg.GetBaseURL(),
+		clientID:       cfg.ClientID,
+		apiKey:         cfg.APIKey,
+		httpClient:     datasource.NewConnectorHTTPClient(defaultTimeout),
+		downloadClient: datasource.NewConnectorHTTPClient(downloadTimeout),
+	}
+}
+
+// callAPI executes an authenticated POST to /openapi/wiki/v1/<action>.
+func (c *client) callAPI(ctx context.Context, action string, req interface{}, result interface{}) error {
+	return c.callAPIAt(ctx, apiBasePath, action, req, result)
+}
+
+// callAPIAt executes an authenticated POST to <basePath>/<action>. It parses
+// the standard {code, msg, data} envelope and unmarshals `data` into `result`.
+// A non-zero business code is returned as a plain error so the caller can
+// surface `msg` to the user.
+//
+// basePath is a parameter because notes live under a second namespace
+// (/openapi/note/v1) that shares this authentication and envelope.
+func (c *client) callAPIAt(
+	ctx context.Context, basePath, action string, req interface{}, result interface{},
+) error {
+	const (
+		maxRetries    = 3
+		max5xxRetries = 1
+		retry5xxDelay = 2 * time.Second
+	)
+	backoff := []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second}
+
+	c.logCredentialOnce.Do(func() {
+		logger.Infof(ctx, "[IMA] client configured client_id=%s api_key=%s base=%s",
+			redact(c.clientID), redact(c.apiKey), c.baseURL)
+	})
+
+	var body []byte
+	if req != nil {
+		b, err := json.Marshal(req)
+		if err != nil {
+			return fmt.Errorf("marshal request: %w", err)
+		}
+		body = b
+	} else {
+		body = []byte("{}")
+	}
+
+	url := c.baseURL + basePath + "/" + action
+	// Both namespaces expose distinct actions, but log the full path so the two
+	// are never ambiguous.
+	label := basePath + "/" + action
+
+	var lastErr error
+	for attempt := 0; attempt <= maxRetries; attempt++ {
+		httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
+		if err != nil {
+			return fmt.Errorf("create request: %w", err)
+		}
+		httpReq.Header.Set("ima-openapi-clientid", c.clientID)
+		httpReq.Header.Set("ima-openapi-apikey", c.apiKey)
+		httpReq.Header.Set("Content-Type", "application/json; charset=utf-8")
+		httpReq.Header.Set("User-Agent", userAgent)
+
+		if attempt == 0 {
+			logger.Infof(ctx, "[IMA] POST %s", label)
+		} else {
+			logger.Infof(ctx, "[IMA] POST %s (retry %d/%d)", label, attempt, maxRetries)
+		}
+
+		resp, err := c.httpClient.Do(httpReq)
+		if err != nil {
+			lastErr = fmt.Errorf("execute request: %w", err)
+			if attempt < maxRetries {
+				if sErr := sleepCtx(ctx, backoff[attempt]); sErr != nil {
+					return sErr
+				}
+				continue
+			}
+			return lastErr
+		}
+
+		respBody, readErr := io.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		if readErr != nil {
+			lastErr = fmt.Errorf("read response: %w", readErr)
+			if attempt < maxRetries {
+				if sErr := sleepCtx(ctx, backoff[attempt]); sErr != nil {
+					return sErr
+				}
+				continue
+			}
+			return lastErr
+		}
+
+		// The body is only ever logged on an error path: successful responses
+		// (get_media_info in particular) carry pre-signed storage URLs and
+		// per-request auth headers that must not land in the server log.
+		bodyPreview := truncate(string(respBody), 500)
+		logger.Infof(ctx, "[IMA] POST %s → status=%d bodyLen=%d", label, resp.StatusCode, len(respBody))
+
+		// HTTP-level auth failures: surface as ErrInvalidCredentials.
+		if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
+			return fmt.Errorf("%w: status=%d body=%s",
+				datasource.ErrInvalidCredentials, resp.StatusCode, bodyPreview)
+		}
+
+		if resp.StatusCode == http.StatusTooManyRequests {
+			wait := backoff[minInt(attempt, len(backoff)-1)]
+			lastErr = fmt.Errorf("ima rate limited: status=429 body=%s", bodyPreview)
+			if attempt < maxRetries {
+				if sErr := sleepCtx(ctx, wait); sErr != nil {
+					return sErr
+				}
+				continue
+			}
+			return lastErr
+		}
+
+		if resp.StatusCode >= 500 && resp.StatusCode < 600 {
+			lastErr = fmt.Errorf("ima server error: status=%d body=%s", resp.StatusCode, bodyPreview)
+			if attempt < max5xxRetries {
+				if sErr := sleepCtx(ctx, retry5xxDelay); sErr != nil {
+					return sErr
+				}
+				continue
+			}
+			return lastErr
+		}
+
+		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+			return fmt.Errorf("ima api http error: status=%d body=%s", resp.StatusCode, bodyPreview)
+		}
+
+		var env apiEnvelope
+		if err := json.Unmarshal(respBody, &env); err != nil {
+			return fmt.Errorf("decode envelope: %w", err)
+		}
+
+		// Business-level errors.
+		// 110030 (无权限) is mapped to ErrInvalidCredentials so the service marks
+		// the source in `error` state and stops scheduling until re-authenticated.
+		// 110021 (限频) is retried. Everything else — including 110001 (参数非法),
+		// which is a bug on our side rather than a credential problem — surfaces
+		// verbatim so the user sees IMA's own message.
+		if code := env.statusCode(); code != 0 {
+			if code == 110030 {
+				return fmt.Errorf("%w: ima code=%d msg=%s",
+					datasource.ErrInvalidCredentials, code, env.message())
+			}
+			if code == 110021 && attempt < maxRetries {
+				lastErr = fmt.Errorf("ima rate limited: code=%d msg=%s", code, env.message())
+				if sErr := sleepCtx(ctx, backoff[minInt(attempt, len(backoff)-1)]); sErr != nil {
+					return sErr
+				}
+				continue
+			}
+			return fmt.Errorf("ima api error: code=%d msg=%s", code, env.message())
+		}
+
+		if result != nil && len(env.Data) > 0 && string(env.Data) != "null" {
+			if err := json.Unmarshal(env.Data, result); err != nil {
+				return fmt.Errorf("decode data: %w", err)
+			}
+		}
+		return nil
+	}
+	return lastErr
+}
+
+// SearchKnowledgeBase — POST /search_knowledge_base. Empty query returns the
+// list of every knowledge base visible to the current token (see api.md §7).
+func (c *client) SearchKnowledgeBase(
+	ctx context.Context, query, cursor string, limit int,
+) (*searchKnowledgeBaseResp, error) {
+	if limit <= 0 || limit > searchPageSize {
+		limit = searchPageSize
+	}
+	req := map[string]interface{}{
+		"query":  query,
+		"cursor": cursor,
+		"limit":  uint64(limit),
+	}
+	var resp searchKnowledgeBaseResp
+	if err := c.callAPI(ctx, "search_knowledge_base", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+// GetKnowledgeBase — POST /get_knowledge_base for a batch of ids (1-20).
+func (c *client) GetKnowledgeBase(ctx context.Context, ids []string) (map[string]knowledgeBaseInfo, error) {
+	if len(ids) == 0 {
+		return map[string]knowledgeBaseInfo{}, nil
+	}
+	req := map[string]interface{}{"ids": ids}
+	var resp getKnowledgeBaseResp
+	if err := c.callAPI(ctx, "get_knowledge_base", req, &resp); err != nil {
+		return nil, err
+	}
+	if resp.Infos == nil {
+		resp.Infos = map[string]knowledgeBaseInfo{}
+	}
+	return resp.Infos, nil
+}
+
+// GetAddableKnowledgeBaseList — POST /get_addable_knowledge_base_list.
+// Lists knowledge bases the current token has permission to add content to.
+// This is the authoritative "what KBs can this credential see?" endpoint
+// (see api.md §7). Used as the primary enumeration source in ListResources —
+// search_knowledge_base with empty query has been observed to return an empty
+// list even when the token owns KBs, so we prefer this endpoint.
+func (c *client) GetAddableKnowledgeBaseList(
+	ctx context.Context, cursor string, limit int,
+) (*getAddableKnowledgeBaseListResp, error) {
+	if limit <= 0 || limit > defaultPageSize {
+		limit = defaultPageSize
+	}
+	req := map[string]interface{}{
+		"cursor": cursor,
+		"limit":  uint64(limit),
+	}
+	var resp getAddableKnowledgeBaseListResp
+	if err := c.callAPI(ctx, "get_addable_knowledge_base_list", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+// GetKnowledgeList — POST /get_knowledge_list. folderID may be empty for root.
+func (c *client) GetKnowledgeList(
+	ctx context.Context, kbID, folderID, cursor string, limit int,
+) (*getKnowledgeListResp, error) {
+	if limit <= 0 || limit > defaultPageSize {
+		limit = defaultPageSize
+	}
+	req := map[string]interface{}{
+		"cursor":            cursor,
+		"limit":             uint64(limit),
+		"knowledge_base_id": kbID,
+	}
+	if folderID != "" {
+		req["folder_id"] = folderID
+	}
+	var resp getKnowledgeListResp
+	if err := c.callAPI(ctx, "get_knowledge_list", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+// GetMediaInfo — POST /get_media_info. Returns URL access info and (for notes)
+// the notebook_id under notebook_ext_info.
+func (c *client) GetMediaInfo(ctx context.Context, mediaID string) (*getMediaInfoResp, error) {
+	req := map[string]interface{}{"media_id": mediaID}
+	var resp getMediaInfoResp
+	if err := c.callAPI(ctx, "get_media_info", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+// GetNoteContent — POST /openapi/note/v1/get_doc_content. Returns an IMA note's
+// body as plain text (target_content_format=0).
+//
+// noteID is the notebook_id that get_media_info reports for a note: the wiki
+// namespace never exposes note bodies, it only points at them.
+func (c *client) GetNoteContent(ctx context.Context, noteID string) (string, error) {
+	req := map[string]interface{}{
+		"note_id":               noteID,
+		"target_content_format": 0,
+	}
+	var resp getDocContentResp
+	if err := c.callAPIAt(ctx, noteBasePath, "get_doc_content", req, &resp); err != nil {
+		return "", err
+	}
+	return resp.Content, nil
+}
+
+// DownloadURL fetches a URL returned by GetMediaInfo, respecting the auth
+// headers IMA may return alongside it. Enforces maxDownloadBytes to avoid a
+// runaway body blowing up sync memory.
+func (c *client) DownloadURL(ctx context.Context, u urlInfo) ([]byte, string, error) {
+	if u.URL == "" {
+		return nil, "", fmt.Errorf("empty url")
+	}
+	// The URL comes from an API response, so it is attacker-influenced as far
+	// as this process is concerned. downloadClient already guards the dial and
+	// every redirect hop; this rejects an unsafe target before the first
+	// connection is even attempted.
+	if err := utils.ValidateURLForSSRF(u.URL); err != nil {
+		return nil, "", fmt.Errorf("media URL rejected: %w", err)
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.URL, nil)
+	if err != nil {
+		return nil, "", fmt.Errorf("create download request: %w", err)
+	}
+	for k, v := range u.Headers {
+		req.Header.Set(k, v)
+	}
+	req.Header.Set("User-Agent", userAgent)
+
+	resp, err := c.downloadClient.Do(req)
+	if err != nil {
+		return nil, "", fmt.Errorf("download: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, "", fmt.Errorf("download http error: status=%d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, maxDownloadBytes+1))
+	if err != nil {
+		return nil, "", fmt.Errorf("read download body: %w", err)
+	}
+	if int64(len(body)) > maxDownloadBytes {
+		return nil, "", fmt.Errorf("download body exceeds %d bytes", maxDownloadBytes)
+	}
+	return body, resp.Header.Get("Content-Type"), nil
+}
+
+// sleepCtx pauses for d, returning early if ctx is cancelled.
+func sleepCtx(ctx context.Context, d time.Duration) error {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-t.C:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func truncate(s string, maxLen int) string {
+	if len(s) <= maxLen {
+		return s
+	}
+	return s[:maxLen] + "..."
+}
+
+// redact returns a masked form of a credential for logging (never log full).
+func redact(t string) string {
+	if len(t) < 12 {
+		return "***"
+	}
+	return t[:6] + "..." + t[len(t)-4:]
+}
+
+func minInt(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}

internal/datasource/connector/ima/connector.go

@@ -0,0 +1,612 @@
+package ima
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/Tencent/WeKnora/internal/datasource"
+	"github.com/Tencent/WeKnora/internal/logger"
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// Compile-time proof that *Connector satisfies the datasource.Connector interface.
+var _ datasource.Connector = (*Connector)(nil)
+
+// Connector implements datasource.Connector for Tencent IMA (ima.qq.com).
+type Connector struct{}
+
+// NewConnector creates a new IMA connector.
+func NewConnector() *Connector { return &Connector{} }
+
+// Type returns the connector type identifier.
+func (c *Connector) Type() string { return types.ConnectorTypeIMA }
+
+// Validate verifies the credentials by calling get_addable_knowledge_base_list
+// — the endpoint most likely to succeed even when the token has zero KBs, and
+// the same one ListResources uses. It returns 110030 (无权限) when the token
+// itself is invalid, which client.callAPI already maps to ErrInvalidCredentials.
+func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
+	cfg, err := parseIMAConfig(config)
+	if err != nil {
+		return err
+	}
+	cli := newClient(cfg)
+	if _, err := cli.GetAddableKnowledgeBaseList(ctx, "", defaultPageSize); err != nil {
+		return fmt.Errorf("ima connection failed: %w", err)
+	}
+	return nil
+}
+
+// ResolveResourceAncestors returns nothing: IMA knowledge bases are exposed as
+// a flat list of top-level resources, so a lazy-loaded picker has no ancestors
+// to reveal.
+func (c *Connector) ResolveResourceAncestors(
+	_ context.Context, _ *types.DataSourceConfig, _ []string,
+) ([]string, error) {
+	return []string{}, nil
+}
+
+// ListResources returns the flat list of knowledge bases the token can read.
+// parentID is honoured only for the "no children" contract — see the note on
+// Connector.ListResources in internal/datasource/connector.go.
+//
+// Primary source is get_addable_knowledge_base_list, which returns the KBs the
+// current OpenAPI credential has permission to operate on. When that endpoint
+// returns an empty list (e.g. a legacy tenant only exposes read scopes) we
+// fall back to search_knowledge_base with an empty query so users still see
+// something to pick.
+func (c *Connector) ListResources(
+	ctx context.Context, config *types.DataSourceConfig, parentID string,
+) ([]types.Resource, error) {
+	if parentID != "" {
+		return []types.Resource{}, nil
+	}
+
+	cfg, err := parseIMAConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	cli := newClient(cfg)
+
+	type kbLite struct {
+		ID       string
+		Name     string
+		CoverURL string
+	}
+	var bases []kbLite
+
+	cursor := ""
+	for {
+		resp, err := cli.GetAddableKnowledgeBaseList(ctx, cursor, defaultPageSize)
+		if err != nil {
+			return nil, fmt.Errorf("get_addable_knowledge_base_list: %w", err)
+		}
+		for _, b := range resp.AddableKnowledgeBaseList {
+			bases = append(bases, kbLite{ID: b.ID, Name: b.Name})
+		}
+		if resp.IsEnd || resp.NextCursor == "" {
+			break
+		}
+		cursor = resp.NextCursor
+	}
+	logger.Infof(ctx, "[IMA] get_addable_knowledge_base_list returned %d knowledge bases", len(bases))
+
+	if len(bases) == 0 {
+		cursor = ""
+		for {
+			resp, err := cli.SearchKnowledgeBase(ctx, "", cursor, searchPageSize)
+			if err != nil {
+				return nil, fmt.Errorf("search_knowledge_base fallback: %w", err)
+			}
+			for _, b := range resp.InfoList {
+				bases = append(bases, kbLite(b))
+			}
+			if resp.IsEnd || resp.NextCursor == "" {
+				break
+			}
+			cursor = resp.NextCursor
+		}
+		logger.Infof(ctx, "[IMA] search_knowledge_base fallback returned %d knowledge bases", len(bases))
+	}
+
+	ids := make([]string, 0, len(bases))
+	for _, b := range bases {
+		ids = append(ids, b.ID)
+	}
+	details := map[string]knowledgeBaseInfo{}
+	for i := 0; i < len(ids); i += 20 {
+		end := i + 20
+		if end > len(ids) {
+			end = len(ids)
+		}
+		batch, err := cli.GetKnowledgeBase(ctx, ids[i:end])
+		if err != nil {
+			logger.Warnf(ctx, "[IMA] get_knowledge_base batch failed (skipping enrichment): %v", err)
+			continue
+		}
+		for k, v := range batch {
+			details[k] = v
+		}
+	}
+
+	out := make([]types.Resource, 0, len(bases))
+	for _, b := range bases {
+		desc := ""
+		coverURL := b.CoverURL
+		if d, ok := details[b.ID]; ok {
+			desc = d.Description
+			if coverURL == "" {
+				coverURL = d.CoverURL
+			}
+		}
+		out = append(out, types.Resource{
+			ExternalID:  b.ID,
+			Name:        b.Name,
+			Type:        "knowledge_base",
+			Description: desc,
+			URL:         cfg.GetBaseURL(),
+			Metadata: map[string]interface{}{
+				"cover_url": coverURL,
+			},
+		})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].ExternalID < out[j].ExternalID })
+	logger.Infof(ctx, "[IMA] ListResources returning %d knowledge bases to UI", len(out))
+	return out, nil
+}
+
+// FetchAll performs a full sync of every knowledge base in resourceIDs.
+func (c *Connector) FetchAll(
+	ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
+) ([]types.FetchedItem, error) {
+	items, _, err := c.walk(ctx, config, resourceIDs, nil, false)
+	return items, err
+}
+
+// FetchIncremental syncs new / replaced / removed items against the cursor.
+//
+// Each IMA item is identified by a stable logical_key (see logicalKey), NOT by
+// media_id — because IMA reassigns media_id whenever a same-named file is
+// replaced in place, and we want that replacement to surface as an UPDATE to
+// the same knowledge item rather than a delete-plus-insert pair.
+//
+// Emitted this cycle:
+//
+//   - logical_key new since last sync                → add    (new content)
+//   - logical_key present, media_id changed          → update (re-fetched content;
+//     ingest layer's existing
+//     external_id match triggers
+//     delete-and-recreate)
+//   - logical_key unchanged, media_id unchanged      → skip
+//   - logical_key present last sync, absent this sync → IsDeleted tombstone
+//
+// LIMITATION: IMA still exposes no per-item updated_at, so an in-place edit
+// that keeps the SAME media_id (same file bytes replaced under the hood by
+// IMA, if that ever happens) is invisible to us. Users needing a full content
+// refresh should periodically run a Full sync.
+func (c *Connector) FetchIncremental(
+	ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor,
+) ([]types.FetchedItem, *types.SyncCursor, error) {
+	resourceIDs := config.ResourceIDs
+	if len(resourceIDs) == 0 {
+		return nil, nil, fmt.Errorf("no resource IDs (knowledge base IDs) configured")
+	}
+
+	var prev *imaCursor
+	if cursor != nil && cursor.ConnectorCursor != nil {
+		var p imaCursor
+		b, _ := json.Marshal(cursor.ConnectorCursor)
+		_ = json.Unmarshal(b, &p)
+		prev = &p
+	}
+
+	items, newCursor, err := c.walk(ctx, config, resourceIDs, prev, true)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cursorMap := make(map[string]interface{})
+	b, _ := json.Marshal(newCursor)
+	_ = json.Unmarshal(b, &cursorMap)
+
+	return items, &types.SyncCursor{
+		LastSyncTime:    newCursor.LastSyncTime,
+		ConnectorCursor: cursorMap,
+	}, nil
+}
+
+// walk is the shared implementation for FetchAll / FetchIncremental.
+// When incremental is false, prev is ignored and returned cursor is nil.
+func (c *Connector) walk(
+	ctx context.Context,
+	config *types.DataSourceConfig,
+	resourceIDs []string,
+	prev *imaCursor,
+	incremental bool,
+) ([]types.FetchedItem, *imaCursor, error) {
+	cfg, err := parseIMAConfig(config)
+	if err != nil {
+		return nil, nil, err
+	}
+	cli := newClient(cfg)
+
+	newCursor := &imaCursor{
+		LastSyncTime: time.Now(),
+		KBLogical:    make(map[string]map[string]string),
+	}
+	var out []types.FetchedItem
+
+	for _, kbID := range resourceIDs {
+		files, folderPath, err := listAllKBFiles(ctx, cli, kbID)
+		if err != nil {
+			return nil, nil, fmt.Errorf("list KB %s: %w", kbID, err)
+		}
+
+		// present holds every logical_key IMA listed this cycle; it drives
+		// delete detection. syncedLogical holds only the keys we actually
+		// resolved this cycle and is what gets persisted as the cursor, so a
+		// transient failure is retried next run rather than being remembered
+		// as "unchanged" forever.
+		present := make(map[string]struct{}, len(files))
+		syncedLogical := make(map[string]string, len(files))
+		keys := make([]string, len(files))
+		for i, f := range files {
+			key := logicalKey(kbID, f.ParentFolderID, f.Title)
+			keys[i] = key
+			present[key] = struct{}{}
+		}
+		newCursor.KBLogical[kbID] = syncedLogical
+
+		var replaced, failed int
+		for i, f := range files {
+			key := keys[i]
+
+			// Incremental skip / replacement detection.
+			if incremental && prev != nil && prev.KBLogical != nil {
+				if prevSet, ok := prev.KBLogical[kbID]; ok {
+					if prevMediaID, seen := prevSet[key]; seen {
+						if prevMediaID == f.MediaID {
+							// Unchanged: carry the entry forward so it is not
+							// re-fetched next cycle either.
+							syncedLogical[key] = f.MediaID
+							continue
+						}
+						replaced++ // media_id changed → replacement, fall through to re-fetch
+					}
+				}
+			}
+
+			item, outcome := fetchOneMedia(ctx, cli, kbID, key, f, folderPath)
+			switch outcome {
+			case fetchOK:
+				syncedLogical[key] = f.MediaID
+				out = append(out, item)
+			case fetchSkipped:
+				// Deterministic skip (unsupported media type, no URL): record it
+				// so we do not pay for get_media_info on every future sync.
+				syncedLogical[key] = f.MediaID
+			case fetchFailed:
+				// Transient: leave the key out of the cursor so the next run
+				// retries. `present` still contains it, so it is not mistaken
+				// for a deletion.
+				failed++
+			}
+		}
+
+		// Deletion detection (incremental only) — based on logical_key vanishing
+		// from the listing, so neither a same-name replacement (media_id churns
+		// but logical_key stays) nor a failed download fires a spurious delete.
+		deleted := 0
+		if incremental && prev != nil && prev.KBLogical != nil {
+			if prevSet, ok := prev.KBLogical[kbID]; ok {
+				for prevKey := range prevSet {
+					if _, still := present[prevKey]; !still {
+						out = append(out, types.FetchedItem{
+							ExternalID:       prevKey,
+							IsDeleted:        true,
+							SourceResourceID: kbID,
+						})
+						deleted++
+					}
+				}
+			}
+		}
+
+		logger.Infof(ctx, "[IMA] KB %s: total=%d replaced=%d failed=%d deleted=%d",
+			kbID, len(files), replaced, failed, deleted)
+	}
+
+	if !incremental {
+		return out, nil, nil
+	}
+	return out, newCursor, nil
+}
+
+// walkedFile is an internal representation of a knowledge item after we've
+// resolved its folder location within the KB tree.
+type walkedFile struct {
+	knowledgeInfo
+	// FolderPath is the "/A/B" style path (folder names, no leading slash)
+	// of the containing folder — empty for root-level items.
+	FolderPath string
+}
+
+// listAllKBFiles recursively enumerates a knowledge base into a flat list of
+// files, resolving each item's folder path from the on-the-fly folder tree.
+// folderPath maps folder_id → readable path (for metadata).
+func listAllKBFiles(
+	ctx context.Context, cli *client, kbID string,
+) ([]walkedFile, map[string]string, error) {
+	folderPath := map[string]string{}
+	var out []walkedFile
+
+	// BFS/DFS over folders. Start from root (empty folder_id).
+	type todo struct {
+		folderID string
+		path     string
+	}
+	stack := []todo{{folderID: "", path: ""}}
+
+	for len(stack) > 0 {
+		cur := stack[len(stack)-1]
+		stack = stack[:len(stack)-1]
+
+		cursor := ""
+		for {
+			resp, err := cli.GetKnowledgeList(ctx, kbID, cur.folderID, cursor, defaultPageSize)
+			if err != nil {
+				return nil, nil, err
+			}
+			for _, raw := range resp.KnowledgeList {
+				// Probe each entry: an entry with a non-empty folder_id is a
+				// folder; otherwise it's a knowledge item (file / note / etc.).
+				var probe struct {
+					FolderID string `json:"folder_id"`
+					MediaID  string `json:"media_id"`
+				}
+				_ = json.Unmarshal(raw, &probe)
+
+				if probe.FolderID != "" && probe.MediaID == "" {
+					var fi folderInfo
+					if err := json.Unmarshal(raw, &fi); err != nil {
+						continue
+					}
+					child := cur.path
+					if child == "" {
+						child = fi.Name
+					} else {
+						child = cur.path + "/" + fi.Name
+					}
+					folderPath[fi.FolderID] = child
+					stack = append(stack, todo{folderID: fi.FolderID, path: child})
+					continue
+				}
+				if probe.MediaID == "" {
+					continue // unrecognized shape, skip defensively
+				}
+				var ki knowledgeInfo
+				if err := json.Unmarshal(raw, &ki); err != nil {
+					continue
+				}
+				out = append(out, walkedFile{
+					knowledgeInfo: ki,
+					FolderPath:    cur.path,
+				})
+			}
+			if resp.IsEnd || resp.NextCursor == "" {
+				break
+			}
+			cursor = resp.NextCursor
+		}
+	}
+	return out, folderPath, nil
+}
+
+// fetchNote resolves an IMA note (media_type=11). Notes carry no downloadable
+// body: get_media_info only reports the notebook_id, and the text has to be
+// read from the separate note namespace.
+//
+// The body arrives as plain text but is ingested as Markdown, because IMA notes
+// are authored as rich text and the export keeps its heading and list markers —
+// parsing it as Markdown preserves that structure for chunking, and text with
+// no markers is unaffected.
+func fetchNote(
+	ctx context.Context, cli *client,
+	kbID string, externalID string, f walkedFile, folderPath map[string]string,
+	info *getMediaInfoResp,
+) (types.FetchedItem, fetchOutcome) {
+	noteID := info.NotebookExtInfo.NotebookID
+	if noteID == "" {
+		logger.Warnf(ctx, "[IMA] note %s (title=%q) has no notebook_id, skipping", f.MediaID, f.Title)
+		return types.FetchedItem{}, fetchSkipped
+	}
+
+	content, err := cli.GetNoteContent(ctx, noteID)
+	if err != nil {
+		logger.Warnf(ctx, "[IMA] get_doc_content(note %s, title=%q) failed, will retry next sync: %v",
+			noteID, f.Title, err)
+		return types.FetchedItem{}, fetchFailed
+	}
+	if strings.TrimSpace(content) == "" {
+		logger.Infof(ctx, "[IMA] note %s (title=%q) is empty, skipping", noteID, f.Title)
+		return types.FetchedItem{}, fetchSkipped
+	}
+
+	fileName := sanitizeFileName(f.Title)
+	if !strings.HasSuffix(strings.ToLower(fileName), ".md") {
+		fileName += ".md"
+	}
+
+	return types.FetchedItem{
+		ExternalID:       externalID,
+		Title:            f.Title,
+		Content:          []byte(content),
+		ContentType:      "text/markdown",
+		FileName:         fileName,
+		SourceResourceID: kbID,
+		UpdatedAt:        time.Now(),
+		Metadata:         baseMetadata(externalID, f, folderPath, info, kbID),
+	}, fetchOK
+}
+
+// fetchOutcome distinguishes the three ways fetchOneMedia can end, because the
+// caller has to treat them differently when building the sync cursor: only a
+// transient failure must stay out of the cursor so it is retried next run.
+type fetchOutcome int
+
+const (
+	// fetchOK: the item was resolved and should be ingested.
+	fetchOK fetchOutcome = iota
+	// fetchSkipped: IMA cannot give us this item's content and never will
+	// (unsupported media type, no URL). Deterministic, so it is safe to
+	// remember in the cursor and stop re-probing it.
+	fetchSkipped
+	// fetchFailed: a transient error (API call or download). Must be retried.
+	fetchFailed
+)
+
+// fetchOneMedia calls get_media_info for a single item and, when possible,
+// downloads its body.
+//
+// externalID is the stable logical_key computed by the caller — same value
+// across syncs even after an IMA same-name replacement mints a new media_id.
+// The raw media_id is still preserved in metadata for observability.
+func fetchOneMedia(
+	ctx context.Context, cli *client,
+	kbID string, externalID string, f walkedFile, folderPath map[string]string,
+) (types.FetchedItem, fetchOutcome) {
+	info, err := cli.GetMediaInfo(ctx, f.MediaID)
+	if err != nil {
+		logger.Warnf(ctx, "[IMA] get_media_info(%s) failed, will retry next sync: %v", f.MediaID, err)
+		return types.FetchedItem{}, fetchFailed
+	}
+
+	if info.MediaType == mediaTypeNote {
+		return fetchNote(ctx, cli, kbID, externalID, f, folderPath, info)
+	}
+
+	if isSkippableMediaType(info.MediaType) {
+		logger.Infof(ctx, "[IMA] skip media %s (title=%q media_type=%d): unsupported by the IMA OpenAPI",
+			f.MediaID, f.Title, info.MediaType)
+		return types.FetchedItem{}, fetchSkipped
+	}
+
+	if info.URLInfo.URL == "" {
+		logger.Warnf(ctx, "[IMA] media %s (title=%q) has no url_info.url, skipping", f.MediaID, f.Title)
+		return types.FetchedItem{}, fetchSkipped
+	}
+
+	ext := extensionForMediaType(info.MediaType)
+
+	// A media type with no fixed extension (web page, WeChat article, ...) is
+	// normally handed to the ingest layer as a bare URL so WeKnora fetches it
+	// itself. That only works for publicly reachable links: when IMA attaches
+	// auth headers the URL points at IMA-hosted storage and WeKnora's own
+	// fetch — which cannot carry those headers — would be rejected. In that
+	// case download here, where the headers are available, and infer the
+	// extension from the response instead.
+	if ext == "" {
+		if len(info.URLInfo.Headers) == 0 {
+			return types.FetchedItem{
+				ExternalID:       externalID,
+				Title:            f.Title,
+				URL:              info.URLInfo.URL,
+				SourceResourceID: kbID,
+				UpdatedAt:        time.Now(),
+				Metadata:         baseMetadata(externalID, f, folderPath, info, kbID),
+			}, fetchOK
+		}
+		ext = "html"
+	}
+
+	body, ct, err := cli.DownloadURL(ctx, info.URLInfo)
+	if err != nil {
+		logger.Warnf(ctx, "[IMA] download media %s (title=%q) failed, will retry next sync: %v",
+			f.MediaID, f.Title, err)
+		return types.FetchedItem{}, fetchFailed
+	}
+
+	// IMA labels every image media_type=9, so trust the response's own
+	// Content-Type over the media_type default when it names a real format.
+	if detected := extensionForContentType(ct); detected != "" {
+		ext = detected
+	}
+	if ct == "" || strings.HasPrefix(ct, "application/octet-stream") {
+		ct = mimeForExtension(ext)
+	}
+
+	fileName := sanitizeFileName(f.Title)
+	if !strings.HasSuffix(strings.ToLower(fileName), "."+ext) {
+		fileName = fileName + "." + ext
+	}
+
+	return types.FetchedItem{
+		ExternalID:       externalID,
+		Title:            f.Title,
+		Content:          body,
+		ContentType:      ct,
+		FileName:         fileName,
+		URL:              info.URLInfo.URL,
+		UpdatedAt:        time.Now(),
+		SourceResourceID: kbID,
+		Metadata:         baseMetadata(externalID, f, folderPath, info, kbID),
+	}, fetchOK
+}
+
+// baseMetadata builds the metadata map preserved on every ingested item.
+// externalID (the caller's logical_key) is stored so operators can join the
+// stable identity back to the raw media_id — useful when debugging why a
+// same-name replacement showed up as an update instead of an add.
+func baseMetadata(
+	externalID string, f walkedFile, folderPath map[string]string,
+	info *getMediaInfoResp, kbID string,
+) map[string]string {
+	m := map[string]string{
+		"channel":           types.ChannelIMA,
+		"media_id":          f.MediaID,
+		"ima_logical_key":   externalID,
+		"knowledge_base_id": kbID,
+		"folder_path":       f.FolderPath,
+		"media_type":        fmt.Sprintf("%d", info.MediaType),
+	}
+	if f.ParentFolderID != "" {
+		m["parent_folder_id"] = f.ParentFolderID
+	}
+	if fp, ok := folderPath[f.ParentFolderID]; ok && fp != "" {
+		m["folder_path"] = fp
+	}
+	if info.NotebookExtInfo.NotebookID != "" {
+		m["notebook_id"] = info.NotebookExtInfo.NotebookID
+	}
+	return m
+}
+
+// sanitizeFileName removes filesystem-hostile characters and truncates to a
+// safe UTF-8 boundary.
+func sanitizeFileName(name string) string {
+	if name == "" {
+		return "untitled"
+	}
+	replacer := strings.NewReplacer(
+		"/", "_", "\\", "_", ":", "_", "*", "_",
+		"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
+	)
+	result := replacer.Replace(name)
+	const maxBytes = 200
+	if len(result) > maxBytes {
+		result = result[:maxBytes]
+		for len(result) > 0 {
+			r, size := utf8.DecodeLastRuneInString(result)
+			if r != utf8.RuneError || size != 1 {
+				break
+			}
+			result = result[:len(result)-1]
+		}
+	}
+	return result
+}

internal/datasource/connector/ima/connector_test.go

@@ -0,0 +1,511 @@
+package ima
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/datasource"
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestConnector_Type(t *testing.T) {
+	if got := NewConnector().Type(); got != types.ConnectorTypeIMA {
+		t.Errorf("Type() = %q, want %q", got, types.ConnectorTypeIMA)
+	}
+}
+
+// TestFetchIncremental_TransientFailureIsRetried is the regression test for the
+// cursor bug: the cursor used to be built from the raw listing before any
+// content was fetched, so an item whose download failed was recorded as
+// "seen with this media_id" and every later incremental sync skipped it as
+// unchanged. The document was then silently missing from the KB forever.
+func TestFetchIncremental_TransientFailureIsRetried(t *testing.T) {
+	f := newFakeIMA(t)
+	good := fakeFile{MediaID: "m-good", Title: "Good", MediaType: mediaTypeMarkdown, Body: "# ok"}
+	flaky := fakeFile{
+		MediaID: "m-flaky", Title: "Flaky", MediaType: mediaTypeMarkdown,
+		Body: "# later", DownloadFails: true,
+	}
+	f.setKB("kb1", []fakeFile{good, flaky})
+
+	c := NewConnector()
+	cfg := f.config("kb1")
+	goodKey := logicalKey("kb1", "", "Good")
+	flakyKey := logicalKey("kb1", "", "Flaky")
+
+	items, cursor, err := c.FetchIncremental(context.Background(), cfg, nil)
+	if err != nil {
+		t.Fatalf("first FetchIncremental: %v", err)
+	}
+	if _, ok := findItem(items, flakyKey); ok {
+		t.Fatalf("failed download must not be emitted, got %s", describeItems(items))
+	}
+	mustFindItem(t, items, goodKey)
+
+	state := decodeCursor(t, cursor)
+	if _, recorded := state.KBLogical["kb1"][flakyKey]; recorded {
+		t.Fatal("a failed item must not be recorded in the cursor, otherwise it is never retried")
+	}
+	if state.KBLogical["kb1"][goodKey] != "m-good" {
+		t.Fatalf("successful item missing from cursor: %#v", state.KBLogical["kb1"])
+	}
+
+	// Second pass: the download now succeeds and the item must be re-attempted.
+	flaky.DownloadFails = false
+	f.setKB("kb1", []fakeFile{good, flaky})
+
+	items2, cursor2, err := c.FetchIncremental(context.Background(), cfg, cursor)
+	if err != nil {
+		t.Fatalf("second FetchIncremental: %v", err)
+	}
+	retried := mustFindItem(t, items2, flakyKey)
+	if string(retried.Content) != "# later" {
+		t.Errorf("retried content = %q, want %q", retried.Content, "# later")
+	}
+	if _, ok := findItem(items2, goodKey); ok {
+		t.Errorf("unchanged item must be skipped on the second pass, got %s", describeItems(items2))
+	}
+	if decodeCursor(t, cursor2).KBLogical["kb1"][flakyKey] != "m-flaky" {
+		t.Error("the retried item should now be recorded in the cursor")
+	}
+}
+
+// TestFetchIncremental_FailureIsNotADeletion guards the other half of the fix:
+// keeping a failed item out of the cursor must not make the next sync mistake
+// it for a document that vanished from IMA.
+func TestFetchIncremental_FailureIsNotADeletion(t *testing.T) {
+	f := newFakeIMA(t)
+	good := fakeFile{MediaID: "m-good", Title: "Good", MediaType: mediaTypeMarkdown, Body: "# ok"}
+	flaky := fakeFile{MediaID: "m-flaky", Title: "Flaky", MediaType: mediaTypeMarkdown, DownloadFails: true}
+	f.setKB("kb1", []fakeFile{good, flaky})
+
+	c := NewConnector()
+	cfg := f.config("kb1")
+	flakyKey := logicalKey("kb1", "", "Flaky")
+
+	_, cursor, err := c.FetchIncremental(context.Background(), cfg, nil)
+	if err != nil {
+		t.Fatalf("first FetchIncremental: %v", err)
+	}
+
+	// Still failing, still listed by IMA: no tombstone may be emitted.
+	items, _, err := c.FetchIncremental(context.Background(), cfg, cursor)
+	if err != nil {
+		t.Fatalf("second FetchIncremental: %v", err)
+	}
+	for _, it := range items {
+		if it.IsDeleted && it.ExternalID == flakyKey {
+			t.Fatalf("a still-listed item that merely failed to download must not be tombstoned: %s",
+				describeItems(items))
+		}
+	}
+}
+
+func TestFetchIncremental_RemovedItemIsTombstoned(t *testing.T) {
+	f := newFakeIMA(t)
+	keep := fakeFile{MediaID: "m-keep", Title: "Keep", MediaType: mediaTypeMarkdown, Body: "# keep"}
+	drop := fakeFile{MediaID: "m-drop", Title: "Drop", MediaType: mediaTypeMarkdown, Body: "# drop"}
+	f.setKB("kb1", []fakeFile{keep, drop})
+
+	c := NewConnector()
+	cfg := f.config("kb1")
+	dropKey := logicalKey("kb1", "", "Drop")
+
+	_, cursor, err := c.FetchIncremental(context.Background(), cfg, nil)
+	if err != nil {
+		t.Fatalf("first FetchIncremental: %v", err)
+	}
+
+	f.setKB("kb1", []fakeFile{keep})
+	items, _, err := c.FetchIncremental(context.Background(), cfg, cursor)
+	if err != nil {
+		t.Fatalf("second FetchIncremental: %v", err)
+	}
+
+	tombstone, ok := findItem(items, dropKey)
+	if !ok || !tombstone.IsDeleted {
+		t.Fatalf("removed item must be tombstoned, got %s", describeItems(items))
+	}
+	if tombstone.SourceResourceID != "kb1" {
+		t.Errorf("tombstone SourceResourceID = %q, want kb1", tombstone.SourceResourceID)
+	}
+}
+
+// TestFetchIncremental_SameNameReplacementKeepsExternalID covers IMA minting a
+// new media_id when a file is replaced in place: the item must come back as an
+// update under the same external id rather than a delete plus an insert.
+func TestFetchIncremental_SameNameReplacementKeepsExternalID(t *testing.T) {
+	f := newFakeIMA(t)
+	f.setKB("kb1", []fakeFile{{MediaID: "m-v1", Title: "Doc", MediaType: mediaTypeMarkdown, Body: "v1"}})
+
+	c := NewConnector()
+	cfg := f.config("kb1")
+	key := logicalKey("kb1", "", "Doc")
+
+	_, cursor, err := c.FetchIncremental(context.Background(), cfg, nil)
+	if err != nil {
+		t.Fatalf("first FetchIncremental: %v", err)
+	}
+
+	f.setKB("kb1", []fakeFile{{MediaID: "m-v2", Title: "Doc", MediaType: mediaTypeMarkdown, Body: "v2"}})
+	items, cursor2, err := c.FetchIncremental(context.Background(), cfg, cursor)
+	if err != nil {
+		t.Fatalf("second FetchIncremental: %v", err)
+	}
+
+	updated := mustFindItem(t, items, key)
+	if updated.IsDeleted {
+		t.Fatal("a same-name replacement must be an update, not a delete")
+	}
+	if string(updated.Content) != "v2" {
+		t.Errorf("content = %q, want v2", updated.Content)
+	}
+	if updated.Metadata["media_id"] != "m-v2" {
+		t.Errorf("metadata media_id = %q, want m-v2", updated.Metadata["media_id"])
+	}
+	for _, it := range items {
+		if it.IsDeleted {
+			t.Errorf("no tombstone expected, got %s", describeItems(items))
+		}
+	}
+	if decodeCursor(t, cursor2).KBLogical["kb1"][key] != "m-v2" {
+		t.Error("cursor should track the new media_id")
+	}
+}
+
+// TestFetchIncremental_UnsupportedTypeIsProbedOnce checks that a deterministic
+// skip is remembered, so an IMA knowledge base full of notes does not re-pay
+// for get_media_info on every scheduled sync.
+func TestFetchIncremental_UnsupportedTypeIsProbedOnce(t *testing.T) {
+	f := newFakeIMA(t)
+	f.setKB("kb1", []fakeFile{{MediaID: "m-note", Title: "Note", MediaType: mediaTypeNote}})
+
+	c := NewConnector()
+	cfg := f.config("kb1")
+
+	items, cursor, err := c.FetchIncremental(context.Background(), cfg, nil)
+	if err != nil {
+		t.Fatalf("first FetchIncremental: %v", err)
+	}
+	if len(items) != 0 {
+		t.Fatalf("note must be skipped, got %s", describeItems(items))
+	}
+	if f.callCount("get_media_info:m-note") != 1 {
+		t.Fatalf("get_media_info calls = %d, want 1", f.callCount("get_media_info:m-note"))
+	}
+
+	if _, _, err := c.FetchIncremental(context.Background(), cfg, cursor); err != nil {
+		t.Fatalf("second FetchIncremental: %v", err)
+	}
+	if got := f.callCount("get_media_info:m-note"); got != 1 {
+		t.Errorf("get_media_info calls after second sync = %d, want 1 (skip should be cached)", got)
+	}
+}
+
+// TestFetchAll_NoteBodyIsReadFromNoteNamespace covers IMA notes. They expose no
+// url_info at all: get_media_info only reports a notebook_id, and the body has
+// to be read from /openapi/note/v1/get_doc_content.
+func TestFetchAll_NoteBodyIsReadFromNoteNamespace(t *testing.T) {
+	f := newFakeIMA(t)
+	f.setKB("kb1", []fakeFile{{
+		MediaID:    "note_1",
+		Title:      "Meeting notes",
+		MediaType:  mediaTypeNote,
+		NotebookID: "987654321",
+		NoteBody:   "# Standup\n- shipped the connector",
+	}})
+
+	items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})
+	if err != nil {
+		t.Fatalf("FetchAll: %v", err)
+	}
+
+	note := mustFindItem(t, items, logicalKey("kb1", "", "Meeting notes"))
+	if string(note.Content) != "# Standup\n- shipped the connector" {
+		t.Errorf("content = %q, want the note body", note.Content)
+	}
+	if !strings.HasSuffix(note.FileName, ".md") {
+		t.Errorf("FileName = %q, want an .md suffix", note.FileName)
+	}
+	if note.ContentType != "text/markdown" {
+		t.Errorf("ContentType = %q, want text/markdown", note.ContentType)
+	}
+	if note.Metadata["notebook_id"] != "987654321" {
+		t.Errorf("metadata notebook_id = %q, want 987654321", note.Metadata["notebook_id"])
+	}
+	if f.callCount("get_doc_content:987654321") != 1 {
+		t.Errorf("get_doc_content calls = %d, want 1", f.callCount("get_doc_content:987654321"))
+	}
+}
+
+// TestFetchIncremental_NoteReadFailureIsRetried checks that a note whose body
+// could not be read follows the same retry path as a failed download rather
+// than being remembered as synced.
+func TestFetchIncremental_NoteReadFailureIsRetried(t *testing.T) {
+	f := newFakeIMA(t)
+	note := fakeFile{
+		MediaID: "note_1", Title: "Broken", MediaType: mediaTypeNote,
+		NotebookID: "111", NoteBody: "recovered", NoteFails: true,
+	}
+	f.setKB("kb1", []fakeFile{note})
+
+	c := NewConnector()
+	cfg := f.config("kb1")
+	key := logicalKey("kb1", "", "Broken")
+
+	items, cursor, err := c.FetchIncremental(context.Background(), cfg, nil)
+	if err != nil {
+		t.Fatalf("first FetchIncremental: %v", err)
+	}
+	if len(items) != 0 {
+		t.Fatalf("a note that could not be read must not be emitted, got %s", describeItems(items))
+	}
+	if _, recorded := decodeCursor(t, cursor).KBLogical["kb1"][key]; recorded {
+		t.Fatal("a failed note read must not be recorded in the cursor")
+	}
+
+	note.NoteFails = false
+	f.setKB("kb1", []fakeFile{note})
+	items2, _, err := c.FetchIncremental(context.Background(), cfg, cursor)
+	if err != nil {
+		t.Fatalf("second FetchIncremental: %v", err)
+	}
+	if got := mustFindItem(t, items2, key); string(got.Content) != "recovered" {
+		t.Errorf("content = %q, want recovered", got.Content)
+	}
+}
+
+func TestFetchAll_EmptyNoteIsSkipped(t *testing.T) {
+	f := newFakeIMA(t)
+	f.setKB("kb1", []fakeFile{{
+		MediaID: "note_1", Title: "Blank", MediaType: mediaTypeNote,
+		NotebookID: "222", NoteBody: "   \n  ",
+	}})
+
+	items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})
+	if err != nil {
+		t.Fatalf("FetchAll: %v", err)
+	}
+	if len(items) != 0 {
+		t.Fatalf("an empty note must be skipped, got %s", describeItems(items))
+	}
+}
+
+// TestFetchAll_AISessionIsStillSkipped pins the types IMA genuinely cannot
+// export, so widening note support does not quietly widen these too.
+func TestFetchAll_AISessionIsStillSkipped(t *testing.T) {
+	f := newFakeIMA(t)
+	f.setKB("kb1", []fakeFile{
+		{MediaID: "m-ai", Title: "Chat", MediaType: mediaTypeAISession},
+		{MediaID: "m-video", Title: "Clip", MediaType: mediaTypeVideo},
+	})
+
+	items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})
+	if err != nil {
+		t.Fatalf("FetchAll: %v", err)
+	}
+	if len(items) != 0 {
+		t.Fatalf("AI sessions and video parses must be skipped, got %s", describeItems(items))
+	}
+}
+
+// TestFetchAll_AuthenticatedURLIsDownloaded covers media types with no fixed
+// extension. When IMA attaches auth headers the URL is IMA-hosted and WeKnora's
+// own fetch could not authenticate, so the connector must download it here and
+// forward the headers.
+func TestFetchAll_AuthenticatedURLIsDownloaded(t *testing.T) {
+	f := newFakeIMA(t)
+	f.setKB("kb1", []fakeFile{{
+		MediaID:     "m-web",
+		Title:       "Article",
+		MediaType:   mediaTypeWeb,
+		Body:        "<html>body</html>",
+		ContentType: "text/html",
+		URLHeaders:  map[string]string{"X-Ima-Token": "secret"},
+	}})
+
+	items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})
+	if err != nil {
+		t.Fatalf("FetchAll: %v", err)
+	}
+	item := mustFindItem(t, items, logicalKey("kb1", "", "Article"))
+	if string(item.Content) != "<html>body</html>" {
+		t.Errorf("content = %q, want the downloaded body", item.Content)
+	}
+	if !strings.HasSuffix(item.FileName, ".html") {
+		t.Errorf("FileName = %q, want an .html suffix", item.FileName)
+	}
+	if got := f.downloadHeaders["m-web"].Get("X-Ima-Token"); got != "secret" {
+		t.Errorf("auth header forwarded = %q, want %q", got, "secret")
+	}
+}
+
+// TestFetchAll_PublicURLStaysURLOnly is the counterpart: without auth headers
+// the link is publicly reachable, so it is handed to the ingest layer as a URL
+// and WeKnora renders the live page instead of a snapshot.
+func TestFetchAll_PublicURLStaysURLOnly(t *testing.T) {
+	f := newFakeIMA(t)
+	f.setKB("kb1", []fakeFile{{MediaID: "m-web", Title: "Article", MediaType: mediaTypeWeb, Body: "ignored"}})
+
+	items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})
+	if err != nil {
+		t.Fatalf("FetchAll: %v", err)
+	}
+	item := mustFindItem(t, items, logicalKey("kb1", "", "Article"))
+	if len(item.Content) != 0 {
+		t.Errorf("public URL must not be downloaded, got %d bytes", len(item.Content))
+	}
+	if item.URL == "" {
+		t.Error("URL-only item must carry a URL")
+	}
+	if f.callCount("download:m-web") != 0 {
+		t.Error("no download should have been attempted")
+	}
+}
+
+// TestFetchAll_ImageExtensionFollowsContentType guards against naming a JPEG
+// ".png": IMA reports every image as media_type=9, so only the download's
+// Content-Type reveals the real format, and the extension drives the ingest
+// pipeline's file-type checks.
+func TestFetchAll_ImageExtensionFollowsContentType(t *testing.T) {
+	f := newFakeIMA(t)
+	f.setKB("kb1", []fakeFile{{
+		MediaID:     "m-img",
+		Title:       "Photo",
+		MediaType:   mediaTypeImage,
+		Body:        "\xff\xd8\xff\xe0jpeg",
+		ContentType: "image/jpeg",
+	}})
+
+	items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})
+	if err != nil {
+		t.Fatalf("FetchAll: %v", err)
+	}
+	item := mustFindItem(t, items, logicalKey("kb1", "", "Photo"))
+	if !strings.HasSuffix(item.FileName, ".jpg") {
+		t.Errorf("FileName = %q, want a .jpg suffix for an image/jpeg body", item.FileName)
+	}
+}
+
+func TestFetchAll_SkipsItemWithoutURL(t *testing.T) {
+	f := newFakeIMA(t)
+	f.setKB("kb1", []fakeFile{{MediaID: "m-empty", Title: "Empty", MediaType: mediaTypePDF, NoURL: true}})
+
+	items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})
+	if err != nil {
+		t.Fatalf("FetchAll: %v", err)
+	}
+	if len(items) != 0 {
+		t.Fatalf("item without a URL must be skipped, got %s", describeItems(items))
+	}
+}
+
+// TestFetchAll_ResolvesFolderPath checks the mixed file/folder listing walk.
+func TestFetchAll_ResolvesFolderPath(t *testing.T) {
+	f := newFakeIMA(t)
+	f.setKB("kb1",
+		[]fakeFile{
+			{MediaID: "m-root", Title: "Root", MediaType: mediaTypeMarkdown, Body: "root"},
+			{MediaID: "m-deep", Title: "Deep", ParentFolderID: "f2", MediaType: mediaTypeMarkdown, Body: "deep"},
+		},
+		fakeFolder{FolderID: "f1", Name: "Outer"},
+		fakeFolder{FolderID: "f2", Name: "Inner", ParentFolderID: "f1"},
+	)
+
+	items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})
+	if err != nil {
+		t.Fatalf("FetchAll: %v", err)
+	}
+
+	deep := mustFindItem(t, items, logicalKey("kb1", "f2", "Deep"))
+	if deep.Metadata["folder_path"] != "Outer/Inner" {
+		t.Errorf("folder_path = %q, want Outer/Inner", deep.Metadata["folder_path"])
+	}
+	root := mustFindItem(t, items, logicalKey("kb1", "", "Root"))
+	if root.Metadata["folder_path"] != "" {
+		t.Errorf("root folder_path = %q, want empty", root.Metadata["folder_path"])
+	}
+}
+
+func TestListResources_ReturnsAddableBases(t *testing.T) {
+	f := newFakeIMA(t)
+	f.setKB("kb-b", nil)
+	f.setKB("kb-a", nil)
+
+	resources, err := NewConnector().ListResources(context.Background(), f.config(), "")
+	if err != nil {
+		t.Fatalf("ListResources: %v", err)
+	}
+	if len(resources) != 2 {
+		t.Fatalf("got %d resources, want 2", len(resources))
+	}
+	if resources[0].ExternalID != "kb-a" || resources[1].ExternalID != "kb-b" {
+		t.Errorf("resources are not sorted by id: %+v", resources)
+	}
+	if resources[0].Description != "desc kb-a" {
+		t.Errorf("description not enriched from get_knowledge_base: %q", resources[0].Description)
+	}
+}
+
+// TestListResources_FallsBackToSearch covers tenants whose credential exposes
+// read-only scopes, where get_addable_knowledge_base_list comes back empty.
+func TestListResources_FallsBackToSearch(t *testing.T) {
+	f := newFakeIMA(t)
+	f.searchBases = []searchedKnowledgeBaseInfo{{ID: "kb-x", Name: "X", CoverURL: "https://cover"}}
+
+	resources, err := NewConnector().ListResources(context.Background(), f.config(), "")
+	if err != nil {
+		t.Fatalf("ListResources: %v", err)
+	}
+	if len(resources) != 1 || resources[0].ExternalID != "kb-x" {
+		t.Fatalf("expected the search fallback to surface kb-x, got %+v", resources)
+	}
+	if resources[0].Metadata["cover_url"] != "https://cover" {
+		t.Errorf("cover_url = %v, want https://cover", resources[0].Metadata["cover_url"])
+	}
+}
+
+func TestListResources_ChildrenAreEmpty(t *testing.T) {
+	f := newFakeIMA(t)
+	f.setKB("kb1", nil)
+
+	resources, err := NewConnector().ListResources(context.Background(), f.config(), "kb1")
+	if err != nil {
+		t.Fatalf("ListResources: %v", err)
+	}
+	if len(resources) != 0 {
+		t.Errorf("knowledge bases have no children, got %+v", resources)
+	}
+}
+
+func TestValidate_InvalidCredentials(t *testing.T) {
+	f := newFakeIMA(t)
+	cfg := f.config()
+	cfg.Credentials["client_id"] = ""
+
+	err := NewConnector().Validate(context.Background(), cfg)
+	if err == nil {
+		t.Fatal("expected an error for a missing client_id")
+	}
+	if !strings.Contains(err.Error(), "client_id") {
+		t.Errorf("error = %v, want it to name the missing field", err)
+	}
+}
+
+func TestFetchIncremental_RequiresResourceIDs(t *testing.T) {
+	f := newFakeIMA(t)
+	if _, _, err := NewConnector().FetchIncremental(context.Background(), f.config(), nil); err == nil {
+		t.Fatal("expected an error when no knowledge base is selected")
+	}
+}
+
+func TestConnectorIsRegisteredInMetadata(t *testing.T) {
+	meta, ok := datasource.ConnectorMetadataRegistry[types.ConnectorTypeIMA]
+	if !ok {
+		t.Fatal("IMA connector missing from the metadata registry")
+	}
+	if meta.Type != types.ConnectorTypeIMA {
+		t.Errorf("metadata type = %q, want %q", meta.Type, types.ConnectorTypeIMA)
+	}
+}

internal/datasource/connector/ima/fake_ima_test.go

@@ -0,0 +1,359 @@
+package ima
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+	secutils "github.com/Tencent/WeKnora/internal/utils"
+)
+
+// TestMain whitelists loopback for SSRF so the httptest servers (127.0.0.1)
+// are reachable. Production keeps the default strict SSRF policy.
+func TestMain(m *testing.M) {
+	_ = os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost")
+	secutils.ResetSSRFWhitelistForTest()
+	os.Exit(m.Run())
+}
+
+// fakeFile is one entry in a fake knowledge base listing.
+type fakeFile struct {
+	MediaID        string
+	Title          string
+	ParentFolderID string
+
+	// MediaType is what get_media_info reports for this file.
+	MediaType int32
+	// Body is what the download URL serves.
+	Body string
+	// ContentType is the download response's Content-Type ("" omits the header).
+	ContentType string
+	// URLHeaders are the auth headers IMA attaches to url_info.
+	URLHeaders map[string]string
+	// NoURL makes get_media_info return an empty url_info.url.
+	NoURL bool
+
+	// InfoFails makes get_media_info return a 500 for this media_id.
+	InfoFails bool
+	// DownloadFails makes the download URL return a 500.
+	DownloadFails bool
+
+	// NotebookID is reported under notebook_ext_info for notes.
+	NotebookID string
+	// NoteBody is what get_doc_content serves for NotebookID.
+	NoteBody string
+	// NoteFails makes get_doc_content return a business error.
+	NoteFails bool
+}
+
+// fakeFolder is a folder entry in a fake knowledge base listing.
+type fakeFolder struct {
+	FolderID       string
+	Name           string
+	ParentFolderID string
+}
+
+// fakeIMA is an in-process stand-in for the IMA OpenAPI. Only the endpoints
+// the connector calls are implemented.
+type fakeIMA struct {
+	server *httptest.Server
+
+	mu sync.Mutex
+	// files and folders are keyed by knowledge base id, then by parent folder
+	// id ("" for the KB root).
+	files   map[string]map[string][]fakeFile
+	folders map[string]map[string][]fakeFolder
+	// bases is what get_addable_knowledge_base_list returns.
+	bases []addableKnowledgeBaseInfo
+	// searchBases is what search_knowledge_base returns.
+	searchBases []searchedKnowledgeBaseInfo
+
+	// calls counts requests per action, plus "download:<media_id>".
+	calls map[string]int
+	// downloadHeaders records the headers seen by the last download of a media.
+	downloadHeaders map[string]http.Header
+}
+
+func newFakeIMA(t *testing.T) *fakeIMA {
+	t.Helper()
+	f := &fakeIMA{
+		files:           map[string]map[string][]fakeFile{},
+		folders:         map[string]map[string][]fakeFolder{},
+		calls:           map[string]int{},
+		downloadHeaders: map[string]http.Header{},
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc(apiBasePath+"/", f.handleAPI)
+	mux.HandleFunc(noteBasePath+"/", f.handleNoteAPI)
+	mux.HandleFunc("/dl/", f.handleDownload)
+	f.server = httptest.NewServer(mux)
+	t.Cleanup(f.server.Close)
+	return f
+}
+
+// setKB replaces the listing of a knowledge base and registers it as addable.
+func (f *fakeIMA) setKB(kbID string, files []fakeFile, folders ...fakeFolder) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	byParent := map[string][]fakeFile{}
+	for _, file := range files {
+		byParent[file.ParentFolderID] = append(byParent[file.ParentFolderID], file)
+	}
+	f.files[kbID] = byParent
+
+	foldersByParent := map[string][]fakeFolder{}
+	for _, folder := range folders {
+		foldersByParent[folder.ParentFolderID] = append(foldersByParent[folder.ParentFolderID], folder)
+	}
+	f.folders[kbID] = foldersByParent
+
+	for _, b := range f.bases {
+		if b.ID == kbID {
+			return
+		}
+	}
+	f.bases = append(f.bases, addableKnowledgeBaseInfo{ID: kbID, Name: "KB " + kbID})
+}
+
+func (f *fakeIMA) callCount(key string) int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.calls[key]
+}
+
+func (f *fakeIMA) record(key string) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.calls[key]++
+}
+
+// findFile locates a file by media_id across every knowledge base.
+func (f *fakeIMA) findFile(mediaID string) (fakeFile, bool) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	for _, byParent := range f.files {
+		for _, files := range byParent {
+			for _, file := range files {
+				if file.MediaID == mediaID {
+					return file, true
+				}
+			}
+		}
+	}
+	return fakeFile{}, false
+}
+
+func (f *fakeIMA) handleAPI(w http.ResponseWriter, r *http.Request) {
+	action := strings.TrimPrefix(r.URL.Path, apiBasePath+"/")
+	f.record(action)
+
+	var req map[string]interface{}
+	_ = json.NewDecoder(r.Body).Decode(&req)
+	str := func(k string) string {
+		if v, ok := req[k].(string); ok {
+			return v
+		}
+		return ""
+	}
+
+	switch action {
+	case "get_addable_knowledge_base_list":
+		f.mu.Lock()
+		resp := getAddableKnowledgeBaseListResp{AddableKnowledgeBaseList: f.bases, IsEnd: true}
+		f.mu.Unlock()
+		writeEnvelope(w, 0, "", resp)
+
+	case "search_knowledge_base":
+		f.mu.Lock()
+		resp := searchKnowledgeBaseResp{InfoList: f.searchBases, IsEnd: true}
+		f.mu.Unlock()
+		writeEnvelope(w, 0, "", resp)
+
+	case "get_knowledge_base":
+		infos := map[string]knowledgeBaseInfo{}
+		if ids, ok := req["ids"].([]interface{}); ok {
+			for _, raw := range ids {
+				id, _ := raw.(string)
+				infos[id] = knowledgeBaseInfo{ID: id, Name: "KB " + id, Description: "desc " + id}
+			}
+		}
+		writeEnvelope(w, 0, "", getKnowledgeBaseResp{Infos: infos})
+
+	case "get_knowledge_list":
+		kbID, folderID := str("knowledge_base_id"), str("folder_id")
+		f.mu.Lock()
+		files := f.files[kbID][folderID]
+		folders := f.folders[kbID][folderID]
+		f.mu.Unlock()
+
+		var list []json.RawMessage
+		for _, folder := range folders {
+			b, _ := json.Marshal(folderInfo{
+				FolderID:       folder.FolderID,
+				Name:           folder.Name,
+				ParentFolderID: folder.ParentFolderID,
+			})
+			list = append(list, b)
+		}
+		for _, file := range files {
+			// IMA omits media_type in list responses.
+			b, _ := json.Marshal(map[string]interface{}{
+				"media_id":         file.MediaID,
+				"title":            file.Title,
+				"parent_folder_id": file.ParentFolderID,
+			})
+			list = append(list, b)
+		}
+		writeEnvelope(w, 0, "", getKnowledgeListResp{KnowledgeList: list, IsEnd: true})
+
+	case "get_media_info":
+		mediaID := str("media_id")
+		f.record("get_media_info:" + mediaID)
+		file, ok := f.findFile(mediaID)
+		if !ok {
+			writeEnvelope(w, 110001, "unknown media", nil)
+			return
+		}
+		if file.InfoFails {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		resp := getMediaInfoResp{MediaType: file.MediaType}
+		resp.NotebookExtInfo = notebookExtInfo{NotebookID: file.NotebookID}
+		// Notes carry no url_info; the body lives in the note namespace.
+		if !file.NoURL && file.MediaType != mediaTypeNote {
+			resp.URLInfo = urlInfo{
+				URL:     f.server.URL + "/dl/" + mediaID,
+				Headers: file.URLHeaders,
+			}
+		}
+		writeEnvelope(w, 0, "", resp)
+
+	default:
+		writeEnvelope(w, 110001, "unsupported action "+action, nil)
+	}
+}
+
+// handleNoteAPI serves the /openapi/note/v1 namespace.
+func (f *fakeIMA) handleNoteAPI(w http.ResponseWriter, r *http.Request) {
+	action := strings.TrimPrefix(r.URL.Path, noteBasePath+"/")
+	f.record("note/" + action)
+
+	var req map[string]interface{}
+	_ = json.NewDecoder(r.Body).Decode(&req)
+
+	if action != "get_doc_content" {
+		writeEnvelope(w, 110012, "unsupported note action "+action, nil)
+		return
+	}
+
+	noteID, _ := req["note_id"].(string)
+	f.record("get_doc_content:" + noteID)
+
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	for _, byParent := range f.files {
+		for _, files := range byParent {
+			for _, file := range files {
+				if file.NotebookID != noteID || noteID == "" {
+					continue
+				}
+				if file.NoteFails {
+					writeEnvelope(w, 110011, "note read failed", nil)
+					return
+				}
+				writeEnvelope(w, 0, "", getDocContentResp{Content: file.NoteBody})
+				return
+			}
+		}
+	}
+	writeEnvelope(w, 110001, "unknown note", nil)
+}
+
+func (f *fakeIMA) handleDownload(w http.ResponseWriter, r *http.Request) {
+	mediaID := strings.TrimPrefix(r.URL.Path, "/dl/")
+	f.record("download:" + mediaID)
+
+	f.mu.Lock()
+	f.downloadHeaders[mediaID] = r.Header.Clone()
+	f.mu.Unlock()
+
+	file, ok := f.findFile(mediaID)
+	if !ok || file.DownloadFails {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	if file.ContentType != "" {
+		w.Header().Set("Content-Type", file.ContentType)
+	}
+	_, _ = w.Write([]byte(file.Body))
+}
+
+func writeEnvelope(w http.ResponseWriter, code int, msg string, data interface{}) {
+	raw, _ := json.Marshal(data)
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(apiEnvelope{Code: code, Msg: msg, Data: raw})
+}
+
+func (f *fakeIMA) config(resourceIDs ...string) *types.DataSourceConfig {
+	return &types.DataSourceConfig{
+		Type: types.ConnectorTypeIMA,
+		Credentials: map[string]interface{}{
+			"client_id": "cid",
+			"api_key":   "key",
+			"base_url":  f.server.URL,
+		},
+		ResourceIDs: resourceIDs,
+	}
+}
+
+// decodeCursor converts the connector cursor back into its typed form.
+func decodeCursor(t *testing.T, cur *types.SyncCursor) *imaCursor {
+	t.Helper()
+	if cur == nil {
+		t.Fatal("cursor is nil")
+	}
+	b, err := json.Marshal(cur.ConnectorCursor)
+	if err != nil {
+		t.Fatalf("marshal cursor: %v", err)
+	}
+	var out imaCursor
+	if err := json.Unmarshal(b, &out); err != nil {
+		t.Fatalf("unmarshal cursor: %v", err)
+	}
+	return &out
+}
+
+// findItem returns the fetched item with the given external id.
+func findItem(items []types.FetchedItem, externalID string) (types.FetchedItem, bool) {
+	for _, it := range items {
+		if it.ExternalID == externalID {
+			return it, true
+		}
+	}
+	return types.FetchedItem{}, false
+}
+
+func mustFindItem(t *testing.T, items []types.FetchedItem, externalID string) types.FetchedItem {
+	t.Helper()
+	it, ok := findItem(items, externalID)
+	if !ok {
+		t.Fatalf("item %s not found in %s", externalID, describeItems(items))
+	}
+	return it
+}
+
+func describeItems(items []types.FetchedItem) string {
+	var parts []string
+	for _, it := range items {
+		parts = append(parts, fmt.Sprintf("{id=%s title=%q deleted=%t}", it.ExternalID, it.Title, it.IsDeleted))
+	}
+	return "[" + strings.Join(parts, " ") + "]"
+}

internal/datasource/connector/ima/types.go

@@ -0,0 +1,433 @@
+package ima
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/datasource"
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+// DefaultBaseURL is the IMA OpenAPI base URL. The Path prefix is baked into
+// the client (see client.go); users only configure the host.
+const DefaultBaseURL = "https://ima.qq.com"
+
+// Base path for knowledge-base OpenAPI requests.
+const apiBasePath = "/openapi/wiki/v1"
+
+// Base path for the note OpenAPI. Notes live in their own namespace: the wiki
+// endpoints only ever hand back a notebook_id for them, and the body has to be
+// read from here.
+const noteBasePath = "/openapi/note/v1"
+
+// Config holds IMA-specific configuration decoded from
+// DataSourceConfig.Credentials. Both credentials are opaque strings and
+// stored encrypted at rest by DataSourceConfig.ToJSON (see internal/types/datasource.go).
+type Config struct {
+	// ClientID: value of the `ima-openapi-clientid` request header,
+	// obtained from https://ima.qq.com/agent-interface.
+	ClientID string `json:"client_id"`
+
+	// APIKey: value of the `ima-openapi-apikey` request header.
+	APIKey string `json:"api_key"`
+
+	// BaseURL lets users point at an on-prem / test IMA endpoint.
+	// Defaults to https://ima.qq.com when empty.
+	BaseURL string `json:"base_url,omitempty"`
+}
+
+// GetBaseURL returns the normalized base URL (empty → default, no trailing slash).
+func (c *Config) GetBaseURL() string {
+	url := strings.TrimSpace(c.BaseURL)
+	if url == "" {
+		return DefaultBaseURL
+	}
+	if !strings.Contains(url, "://") {
+		url = "https://" + url
+	}
+	return strings.TrimRight(url, "/")
+}
+
+// parseIMAConfig extracts and validates IMA-specific configuration.
+// Uses JSON marshal/unmarshal roundtrip so extra fields are ignored gracefully.
+func parseIMAConfig(config *types.DataSourceConfig) (*Config, error) {
+	if config == nil {
+		return nil, fmt.Errorf("%w: config is nil", datasource.ErrInvalidConfig)
+	}
+	credBytes, err := json.Marshal(config.Credentials)
+	if err != nil {
+		return nil, fmt.Errorf("marshal credentials: %w", err)
+	}
+	var cfg Config
+	if err := json.Unmarshal(credBytes, &cfg); err != nil {
+		return nil, fmt.Errorf("parse ima credentials: %w", err)
+	}
+	if strings.TrimSpace(cfg.ClientID) == "" {
+		return nil, fmt.Errorf("%w: client_id is required", datasource.ErrInvalidCredentials)
+	}
+	if strings.TrimSpace(cfg.APIKey) == "" {
+		return nil, fmt.Errorf("%w: api_key is required", datasource.ErrInvalidCredentials)
+	}
+	if err := datasource.ValidateConnectorBaseURL(cfg.GetBaseURL()); err != nil {
+		return nil, err
+	}
+	return &cfg, nil
+}
+
+// apiEnvelope is the uniform IMA response wrapper: `{ code, msg, data }`.
+// A non-zero code is a business-level error and MUST be surfaced to the user.
+//
+// Some published IMA references spell the same envelope `{ retcode, errmsg }`.
+// Accepting both costs nothing and avoids the failure mode where a mismatch
+// leaves Code at its zero value and every API error is silently read as
+// success.
+type apiEnvelope struct {
+	Code   int    `json:"code"`
+	Msg    string `json:"msg"`
+	RetErr *int   `json:"retcode"`
+	ErrMsg string `json:"errmsg"`
+
+	Data json.RawMessage `json:"data"`
+}
+
+// statusCode returns the business status code under either spelling.
+func (e apiEnvelope) statusCode() int {
+	if e.Code != 0 {
+		return e.Code
+	}
+	if e.RetErr != nil {
+		return *e.RetErr
+	}
+	return 0
+}
+
+// message returns the human-readable error under either spelling.
+func (e apiEnvelope) message() string {
+	if e.Msg != "" {
+		return e.Msg
+	}
+	return e.ErrMsg
+}
+
+// KnowledgeBaseInfo mirrors IMA's KnowledgeBaseInfo (get_knowledge_base).
+type knowledgeBaseInfo struct {
+	ID                   string   `json:"id"`
+	Name                 string   `json:"name"`
+	CoverURL             string   `json:"cover_url"`
+	Description          string   `json:"description"`
+	RecommendedQuestions []string `json:"recommended_questions"`
+}
+
+// searchedKnowledgeBaseInfo — search_knowledge_base returns fewer fields.
+type searchedKnowledgeBaseInfo struct {
+	ID       string `json:"id"`
+	Name     string `json:"name"`
+	CoverURL string `json:"cover_url"`
+}
+
+// searchKnowledgeBaseResp — search_knowledge_base response.
+type searchKnowledgeBaseResp struct {
+	InfoList   []searchedKnowledgeBaseInfo `json:"info_list"`
+	IsEnd      bool                        `json:"is_end"`
+	NextCursor string                      `json:"next_cursor"`
+}
+
+// addableKnowledgeBaseInfo — get_addable_knowledge_base_list item.
+// Contains only id + name (no cover_url), see api.md §7.
+type addableKnowledgeBaseInfo struct {
+	ID   string `json:"id"`
+	Name string `json:"name"`
+}
+
+// getAddableKnowledgeBaseListResp — get_addable_knowledge_base_list response.
+// NOTE: the JSON field is `addable_knowledge_base_list`, not `info_list`.
+type getAddableKnowledgeBaseListResp struct {
+	AddableKnowledgeBaseList []addableKnowledgeBaseInfo `json:"addable_knowledge_base_list"`
+	IsEnd                    bool                       `json:"is_end"`
+	NextCursor               string                     `json:"next_cursor"`
+}
+
+// getKnowledgeBaseResp — get_knowledge_base response (map keyed by id).
+type getKnowledgeBaseResp struct {
+	Infos map[string]knowledgeBaseInfo `json:"infos"`
+}
+
+// knowledgeInfo mirrors IMA's KnowledgeInfo (files, notes, etc.).
+// IMPORTANT: IMA does not expose media_type in the list response, we must call
+// get_media_info to discover it. However, folders are distinguished by presence
+// of a non-empty folder_id — see folderInfo below.
+type knowledgeInfo struct {
+	MediaID        string `json:"media_id"`
+	Title          string `json:"title"`
+	ParentFolderID string `json:"parent_folder_id"`
+	MediaType      int32  `json:"media_type,omitempty"`
+}
+
+// folderInfo mirrors IMA's FolderInfo.
+type folderInfo struct {
+	FolderID       string      `json:"folder_id"`
+	Name           string      `json:"name"`
+	FileNumber     json.Number `json:"file_number"`
+	FolderNumber   json.Number `json:"folder_number"`
+	ParentFolderID string      `json:"parent_folder_id"`
+	IsTop          bool        `json:"is_top"`
+}
+
+// FileCount safely converts FileNumber to int64 (0 on empty/invalid).
+func (f folderInfo) FileCount() int64 {
+	if f.FileNumber == "" {
+		return 0
+	}
+	n, err := f.FileNumber.Int64()
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+// FolderCount safely converts FolderNumber to int64 (0 on empty/invalid).
+func (f folderInfo) FolderCount() int64 {
+	if f.FolderNumber == "" {
+		return 0
+	}
+	n, err := f.FolderNumber.Int64()
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+// getKnowledgeListResp — get_knowledge_list response.
+// According to `references/api.md`, the response returns knowledge_list mixed
+// with folders; we decode a loose JSON view so entries without a folder_id are
+// treated as knowledge and those with folder_id are treated as folders.
+type getKnowledgeListResp struct {
+	// KnowledgeList is a loose slice of raw JSON objects because IMA returns
+	// files and folders in the same array. We iterate and inspect fields.
+	KnowledgeList []json.RawMessage `json:"knowledge_list"`
+	IsEnd         bool              `json:"is_end"`
+	NextCursor    string            `json:"next_cursor"`
+	CurrentPath   []folderInfo      `json:"current_path"`
+}
+
+// urlInfo mirrors IMA's URLInfo (present on get_media_info for URL-backed media).
+type urlInfo struct {
+	URL     string            `json:"url"`
+	Headers map[string]string `json:"headers"`
+}
+
+// notebookExtInfo — set on notes (media_type=11).
+type notebookExtInfo struct {
+	NotebookID string `json:"notebook_id"`
+}
+
+// getMediaInfoResp — get_media_info response.
+type getMediaInfoResp struct {
+	MediaType       int32           `json:"media_type"`
+	URLInfo         urlInfo         `json:"url_info"`
+	NotebookExtInfo notebookExtInfo `json:"notebook_ext_info"`
+}
+
+// getDocContentResp — note/v1 get_doc_content response. Content is the note
+// body; with target_content_format=0 it is plain text.
+type getDocContentResp struct {
+	Content string `json:"content"`
+}
+
+// imaCursor tracks per-knowledge-base state seen during the last sync.
+//
+// KBLogical: { logical_key: media_id } — the authoritative source for both
+// delete detection AND "replacement" detection. IMA reassigns media_id
+// whenever a same-named file is replaced in place, so a logical key (see
+// logicalKey) is what we treat as a document's stable identity.
+// A logical_key present last sync but absent this sync ⇒ delete.
+// A logical_key whose media_id changed ⇒ replacement (re-fetch content; the
+// emitted FetchedItem carries the same ExternalID so the ingest layer's
+// "existing external_id → delete-and-recreate" path (see
+// datasource_service.go applyFetchedItem) treats it as an update.
+//
+// Only items that were successfully synced (or deterministically skipped)
+// this cycle are recorded, so a transient download failure leaves the key
+// absent and the next run retries it instead of treating it as unchanged.
+//
+// KBMedia is a legacy field retained only so cursors persisted by earlier
+// builds still deserialize; it is never written or consulted.
+type imaCursor struct {
+	LastSyncTime time.Time                    `json:"last_sync_time"`
+	KBLogical    map[string]map[string]string `json:"kb_logical,omitempty"`
+	KBMedia      map[string]map[string]string `json:"kb_media,omitempty"`
+}
+
+// logicalKey derives a stable identity for an IMA item that survives
+// same-name replacement (which mints a new media_id server-side).
+//
+// The key is a short SHA-256 hex over (kb_id, parent_folder_id, title).
+// Rationale:
+//
+//   - kb_id keeps identities isolated across knowledge bases (same file
+//     copied into two KBs is two distinct docs from WeKnora's perspective).
+//   - parent_folder_id + title anchor the doc to its folder location and
+//     display name — the tuple IMA's own check_repeated_names uses to gate
+//     uploads (see api.md), so IMA's own uniqueness constraint enforces
+//     this key's uniqueness inside a KB.
+//
+// media_type is deliberately NOT part of the key: get_knowledge_list omits it
+// (it only arrives later from get_media_info), so folding it in would mean
+// every external_id shifts the day IMA starts populating the field in list
+// responses — which reads as "every document deleted and re-added".
+//
+// Emitted as "ima_" + first 32 hex chars (128 bits) — small enough to fit
+// well within Postgres varchar(64) external_id columns, still >> collision
+// resistant for any realistic KB size.
+func logicalKey(kbID, parentFolderID, title string) string {
+	h := sha256.New()
+	// Delimiter can't appear in any component because it's neither a valid
+	// IMA id character nor allowed in titles once we've folded to bytes.
+	_, _ = fmt.Fprintf(h, "%s\x1f%s\x1f%s", kbID, parentFolderID, title)
+	return "ima_" + hex.EncodeToString(h.Sum(nil))[:32]
+}
+
+// IMA MediaType enum (see api.md §MediaType).
+const (
+	mediaTypePDF       int32 = 1
+	mediaTypeWeb       int32 = 2
+	mediaTypeWord      int32 = 3
+	mediaTypePPT       int32 = 4
+	mediaTypeExcel     int32 = 5
+	mediaTypeMPArticle int32 = 6
+	mediaTypeMarkdown  int32 = 7
+	mediaTypeImage     int32 = 9
+	mediaTypeNote      int32 = 11
+	mediaTypeAISession int32 = 12
+	mediaTypeTXT       int32 = 13
+	mediaTypeXmind     int32 = 14
+	mediaTypeAudio     int32 = 15
+	mediaTypeVideo     int32 = 16
+	mediaTypeHTML      int32 = 20
+	mediaTypeEPUB      int32 = 21
+)
+
+// extensionForMediaType maps IMA media types to a file extension.
+// Returns "" when the media type has no downloadable body (web page, note, AI
+// session, video-parse) — callers should skip those or treat them as URL-only.
+func extensionForMediaType(t int32) string {
+	switch t {
+	case mediaTypePDF:
+		return "pdf"
+	case mediaTypeWord:
+		return "docx"
+	case mediaTypePPT:
+		return "pptx"
+	case mediaTypeExcel:
+		return "xlsx"
+	case mediaTypeMarkdown:
+		return "md"
+	case mediaTypeImage:
+		return "png"
+	case mediaTypeTXT:
+		return "txt"
+	case mediaTypeXmind:
+		return "xmind"
+	case mediaTypeAudio:
+		return "mp3"
+	case mediaTypeHTML:
+		return "html"
+	case mediaTypeEPUB:
+		return "epub"
+	default:
+		return ""
+	}
+}
+
+// extensionForContentType maps a response Content-Type back to a file
+// extension, ignoring any "; charset=..." suffix. Returns "" for types we
+// have no better guess for than the media_type-derived default.
+//
+// IMA reports every image as media_type=9 regardless of the real encoding, so
+// the download's Content-Type is the only signal that distinguishes a JPEG
+// from a PNG. Naming a JPEG ".png" breaks the extension-based file-type checks
+// downstream in the ingest pipeline.
+func extensionForContentType(contentType string) string {
+	mediaType := strings.ToLower(strings.TrimSpace(contentType))
+	if i := strings.Index(mediaType, ";"); i >= 0 {
+		mediaType = strings.TrimSpace(mediaType[:i])
+	}
+	switch mediaType {
+	case "image/png":
+		return "png"
+	case "image/jpeg", "image/jpg":
+		return "jpg"
+	case "image/gif":
+		return "gif"
+	case "image/webp":
+		return "webp"
+	case "image/bmp":
+		return "bmp"
+	default:
+		return ""
+	}
+}
+
+// mimeForExtension maps a file extension to a canonical MIME type.
+// Used when the downloaded URL response omits (or misreports) Content-Type.
+func mimeForExtension(ext string) string {
+	switch strings.ToLower(strings.TrimPrefix(ext, ".")) {
+	case "pdf":
+		return "application/pdf"
+	case "doc":
+		return "application/msword"
+	case "docx":
+		return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+	case "ppt":
+		return "application/vnd.ms-powerpoint"
+	case "pptx":
+		return "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+	case "xls":
+		return "application/vnd.ms-excel"
+	case "xlsx":
+		return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+	case "md":
+		return "text/markdown"
+	case "txt":
+		return "text/plain"
+	case "html", "htm":
+		return "text/html"
+	case "png":
+		return "image/png"
+	case "jpg", "jpeg":
+		return "image/jpeg"
+	case "webp":
+		return "image/webp"
+	case "gif":
+		return "image/gif"
+	case "bmp":
+		return "image/bmp"
+	case "epub":
+		return "application/epub+zip"
+	case "xmind":
+		return "application/x-xmind"
+	case "mp3":
+		return "audio/mpeg"
+	case "m4a":
+		return "audio/x-m4a"
+	case "wav":
+		return "audio/wav"
+	case "aac":
+		return "audio/aac"
+	default:
+		return "application/octet-stream"
+	}
+}
+
+// isSkippableMediaType reports whether IMA offers no way to read this type's
+// content. AI sessions are only ever referenced by session_id with no read
+// endpoint, and video parses cannot even be added outside the IMA desktop app.
+// Notes are NOT listed here: they are read through the note namespace, see
+// fetchNote.
+func isSkippableMediaType(t int32) bool {
+	return t == mediaTypeAISession || t == mediaTypeVideo
+}

internal/datasource/connector/ima/types_test.go

@@ -0,0 +1,196 @@
+package ima
+
+import (
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/datasource"
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestLogicalKey_IsStableAndScoped(t *testing.T) {
+	base := logicalKey("kb1", "f1", "Doc")
+
+	if base != logicalKey("kb1", "f1", "Doc") {
+		t.Error("logicalKey must be deterministic")
+	}
+	if !strings.HasPrefix(base, "ima_") || len(base) != 36 {
+		t.Errorf("key = %q, want an ima_ prefix and 36 chars so it fits varchar(64)", base)
+	}
+
+	for _, other := range []string{
+		logicalKey("kb2", "f1", "Doc"),
+		logicalKey("kb1", "f2", "Doc"),
+		logicalKey("kb1", "f1", "Other"),
+	} {
+		if other == base {
+			t.Errorf("keys must differ across kb/folder/title, both were %q", base)
+		}
+	}
+}
+
+// TestLogicalKey_DelimiterIsUnambiguous guards against a key collision from
+// naively concatenating the components.
+func TestLogicalKey_DelimiterIsUnambiguous(t *testing.T) {
+	if logicalKey("kb", "a", "bc") == logicalKey("kb", "ab", "c") {
+		t.Error("component boundaries must not be ambiguous")
+	}
+}
+
+func TestParseIMAConfig(t *testing.T) {
+	tests := []struct {
+		name        string
+		credentials map[string]interface{}
+		wantErr     error
+		wantErrText string
+	}{
+		{
+			name:        "valid",
+			credentials: map[string]interface{}{"client_id": "cid", "api_key": "key"},
+		},
+		{
+			name:        "missing client_id",
+			credentials: map[string]interface{}{"api_key": "key"},
+			wantErr:     datasource.ErrInvalidCredentials,
+		},
+		{
+			name:        "blank api_key",
+			credentials: map[string]interface{}{"client_id": "cid", "api_key": "   "},
+			wantErr:     datasource.ErrInvalidCredentials,
+		},
+		{
+			name:        "loopback base_url rejected by SSRF policy",
+			credentials: map[string]interface{}{"client_id": "cid", "api_key": "key", "base_url": "http://169.254.169.254"},
+			wantErrText: "SSRF",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := parseIMAConfig(&types.DataSourceConfig{Credentials: tt.credentials})
+			switch {
+			case tt.wantErr != nil:
+				if !errors.Is(err, tt.wantErr) {
+					t.Fatalf("err = %v, want %v", err, tt.wantErr)
+				}
+			case tt.wantErrText != "":
+				if err == nil || !strings.Contains(err.Error(), tt.wantErrText) {
+					t.Fatalf("err = %v, want it to mention %q", err, tt.wantErrText)
+				}
+			default:
+				if err != nil {
+					t.Fatalf("unexpected error: %v", err)
+				}
+			}
+		})
+	}
+}
+
+func TestParseIMAConfig_NilConfig(t *testing.T) {
+	if _, err := parseIMAConfig(nil); !errors.Is(err, datasource.ErrInvalidConfig) {
+		t.Fatalf("err = %v, want ErrInvalidConfig", err)
+	}
+}
+
+func TestConfig_GetBaseURL(t *testing.T) {
+	tests := []struct {
+		in   string
+		want string
+	}{
+		{"", DefaultBaseURL},
+		{"   ", DefaultBaseURL},
+		{"https://ima.example.com/", "https://ima.example.com"},
+		{"ima.example.com", "https://ima.example.com"},
+		{"http://ima.example.com//", "http://ima.example.com"},
+	}
+	for _, tt := range tests {
+		if got := (&Config{BaseURL: tt.in}).GetBaseURL(); got != tt.want {
+			t.Errorf("GetBaseURL(%q) = %q, want %q", tt.in, got, tt.want)
+		}
+	}
+}
+
+func TestExtensionForContentType(t *testing.T) {
+	tests := []struct {
+		in   string
+		want string
+	}{
+		{"image/jpeg", "jpg"},
+		{"image/JPEG", "jpg"},
+		{"image/png; charset=binary", "png"},
+		{"image/webp", "webp"},
+		{"application/pdf", ""},
+		{"", ""},
+	}
+	for _, tt := range tests {
+		if got := extensionForContentType(tt.in); got != tt.want {
+			t.Errorf("extensionForContentType(%q) = %q, want %q", tt.in, got, tt.want)
+		}
+	}
+}
+
+func TestSanitizeFileName(t *testing.T) {
+	if got := sanitizeFileName(""); got != "untitled" {
+		t.Errorf("empty name = %q, want untitled", got)
+	}
+	if got := sanitizeFileName(`a/b\c:d*e?f"g<h>i|j`); strings.ContainsAny(got, `/\:*?"<>|`) {
+		t.Errorf("filesystem-hostile characters survived: %q", got)
+	}
+
+	// A long CJK title must be truncated on a rune boundary, not mid-sequence.
+	long := strings.Repeat("知识", 200)
+	got := sanitizeFileName(long)
+	if len(got) > 200 {
+		t.Errorf("len = %d bytes, want <= 200", len(got))
+	}
+	if !strings.HasPrefix(long, got) {
+		t.Errorf("truncation corrupted the prefix: %q", got)
+	}
+}
+
+func TestIsSkippableMediaType(t *testing.T) {
+	for _, mt := range []int32{mediaTypeAISession, mediaTypeVideo} {
+		if !isSkippableMediaType(mt) {
+			t.Errorf("media type %d should be skippable", mt)
+		}
+	}
+	// Notes are read through the note namespace rather than skipped.
+	for _, mt := range []int32{mediaTypePDF, mediaTypeMarkdown, mediaTypeWeb, mediaTypeNote} {
+		if isSkippableMediaType(mt) {
+			t.Errorf("media type %d should not be skippable", mt)
+		}
+	}
+}
+
+// TestAPIEnvelope_AcceptsBothSpellings covers the retcode/errmsg variant some
+// IMA references describe: if only `code` were read, a non-zero retcode would
+// decode as 0 and every API error would be silently treated as success.
+func TestAPIEnvelope_AcceptsBothSpellings(t *testing.T) {
+	tests := []struct {
+		name     string
+		body     string
+		wantCode int
+		wantMsg  string
+	}{
+		{"code/msg", `{"code":110030,"msg":"无权限"}`, 110030, "无权限"},
+		{"retcode/errmsg", `{"retcode":110030,"errmsg":"无权限"}`, 110030, "无权限"},
+		{"success", `{"code":0,"msg":"ok"}`, 0, "ok"},
+		{"retcode success", `{"retcode":0,"errmsg":"成功"}`, 0, "成功"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var env apiEnvelope
+			if err := json.Unmarshal([]byte(tt.body), &env); err != nil {
+				t.Fatalf("unmarshal: %v", err)
+			}
+			if got := env.statusCode(); got != tt.wantCode {
+				t.Errorf("statusCode() = %d, want %d", got, tt.wantCode)
+			}
+			if got := env.message(); got != tt.wantMsg {
+				t.Errorf("message() = %q, want %q", got, tt.wantMsg)
+			}
+		})
+	}
+}