WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/datasource/connector/rss/client.go

@@ -0,0 +1,123 @@
+package rss
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+
+	readability "codeberg.org/readeck/go-readability/v2"
+	"github.com/Tencent/WeKnora/internal/utils"
+)
+
+const (
+	// requestTimeout bounds a single feed or article fetch.
+	requestTimeout = 20 * time.Second
+
+	// maxFeedSize caps the feed body to avoid memory blowups on hostile feeds.
+	maxFeedSize = 10 * 1024 * 1024 // 10 MB
+
+	// maxArticleSize caps a single article page body.
+	maxArticleSize = 5 * 1024 * 1024 // 5 MB
+
+	// defaultUserAgent is sent on every request; some feeds reject empty UAs.
+	defaultUserAgent = "Mozilla/5.0 (compatible; WeKnora-RSS/1.0; +https://weknora.weixin.qq.com)"
+)
+
+// client performs SSRF-safe HTTP fetches with optional custom headers.
+type client struct {
+	httpClient *http.Client
+	headers    map[string]string
+}
+
+func newClient(headers map[string]string) *client {
+	cfg := utils.DefaultSSRFSafeHTTPClientConfig()
+	cfg.Timeout = requestTimeout
+	return &client{
+		httpClient: utils.NewSSRFSafeHTTPClient(cfg),
+		headers:    headers,
+	}
+}
+
+// fetch retrieves rawURL with SSRF validation and size limiting. Custom auth
+// headers are only attached when withAuthHeaders is true (feed fetches); article
+// pages on third-party domains must not receive feed credentials.
+func (c *client) fetch(ctx context.Context, rawURL string, maxSize int64, withAuthHeaders bool) ([]byte, error) {
+	if err := utils.ValidateURLForSSRF(rawURL); err != nil {
+		return nil, fmt.Errorf("URL rejected: %w", err)
+	}
+	if _, err := url.Parse(rawURL); err != nil {
+		return nil, fmt.Errorf("invalid URL: %w", err)
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, requestTimeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if withAuthHeaders {
+		for k, v := range c.headers {
+			req.Header.Set(k, v)
+		}
+	}
+	if req.Header.Get("User-Agent") == "" {
+		req.Header.Set("User-Agent", defaultUserAgent)
+	}
+	if req.Header.Get("Accept") == "" {
+		req.Header.Set("Accept",
+			"application/rss+xml, application/atom+xml, application/xml, text/xml, application/json, text/html;q=0.9, */*;q=0.8")
+	}
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("fetch failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status)
+	}
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, maxSize))
+	if err != nil {
+		return nil, fmt.Errorf("read body failed: %w", err)
+	}
+	return body, nil
+}
+
+// fetchFeed retrieves the raw bytes of a feed document.
+func (c *client) fetchFeed(ctx context.Context, feedURL string) ([]byte, error) {
+	return c.fetch(ctx, feedURL, maxFeedSize, true)
+}
+
+// extractArticle fetches an article page and returns the readability-cleaned
+// main content as HTML, plus the extracted title (may be empty). Returns an
+// error if the page can't be fetched or no readable content is found, so the
+// caller can fall back to feed-provided content.
+func (c *client) extractArticle(ctx context.Context, articleURL string) (contentHTML, title string, err error) {
+	body, err := c.fetch(ctx, articleURL, maxArticleSize, false)
+	if err != nil {
+		return "", "", err
+	}
+
+	pageURL, _ := url.Parse(articleURL)
+	article, err := readability.FromReader(bytes.NewReader(body), pageURL)
+	if err != nil {
+		return "", "", fmt.Errorf("readability parse: %w", err)
+	}
+	if article.Node == nil {
+		return "", "", fmt.Errorf("no readable content extracted")
+	}
+
+	var buf bytes.Buffer
+	if err := article.RenderHTML(&buf); err != nil {
+		return "", "", fmt.Errorf("render article html: %w", err)
+	}
+	return buf.String(), article.Title(), nil
+}

internal/datasource/connector/rss/connector.go

@@ -0,0 +1,366 @@
+package rss
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
+	"github.com/Tencent/WeKnora/internal/datasource"
+	"github.com/Tencent/WeKnora/internal/logger"
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/mmcdole/gofeed"
+)
+
+// Compile-time proof that *Connector satisfies the datasource.Connector interface.
+var _ datasource.Connector = (*Connector)(nil)
+
+// Connector implements datasource.Connector for RSS/Atom/JSON feeds.
+type Connector struct{}
+
+// NewConnector creates a new RSS connector.
+func NewConnector() *Connector { return &Connector{} }
+
+// Type returns the connector type identifier.
+func (c *Connector) Type() string { return types.ConnectorTypeRSS }
+
+// Validate verifies that every configured feed URL is reachable and parses as
+// a valid feed.
+func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
+	cfg, err := parseConfig(config)
+	if err != nil {
+		return err
+	}
+	cli := newClient(cfg.parseHeaders())
+	parser := gofeed.NewParser()
+
+	for _, feedURL := range cfg.feedURLList() {
+		data, err := cli.fetchFeed(ctx, feedURL)
+		if err != nil {
+			return fmt.Errorf("fetch feed %s: %w", feedURL, err)
+		}
+		if _, err := parser.Parse(bytes.NewReader(data)); err != nil {
+			return fmt.Errorf("parse feed %s: %w", feedURL, err)
+		}
+	}
+	return nil
+}
+
+// ResolveResourceAncestors has nothing to do: feeds are a flat list with no
+// nesting, so a selection has no ancestors to reveal.
+func (c *Connector) ResolveResourceAncestors(
+	ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
+) ([]string, error) {
+	return []string{}, nil
+}
+
+// ListResources returns one resource per configured feed URL. The feed is
+// fetched so the resource can carry its real title; a feed that fails to fetch
+// still appears (named by URL) with an error note, so the user can deselect it
+// instead of the whole listing failing.
+func (c *Connector) ListResources(
+	ctx context.Context, config *types.DataSourceConfig, parentID string,
+) ([]types.Resource, error) {
+	// Feeds are flat: a lazy-load request for a specific parent has nothing extra.
+	if parentID != "" {
+		return []types.Resource{}, nil
+	}
+
+	cfg, err := parseConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	cli := newClient(cfg.parseHeaders())
+	parser := gofeed.NewParser()
+
+	feedURLs := cfg.feedURLList()
+	out := make([]types.Resource, 0, len(feedURLs))
+	for _, feedURL := range feedURLs {
+		res := types.Resource{
+			ExternalID: feedURL,
+			Type:       "feed",
+			Name:       feedURL,
+			URL:        feedURL,
+		}
+		data, err := cli.fetchFeed(ctx, feedURL)
+		if err != nil {
+			logger.Warnf(ctx, "[RSS] list: fetch %s failed: %v", feedURL, err)
+			res.Description = "fetch failed: " + err.Error()
+			out = append(out, res)
+			continue
+		}
+		feed, err := parser.Parse(bytes.NewReader(data))
+		if err != nil {
+			logger.Warnf(ctx, "[RSS] list: parse %s failed: %v", feedURL, err)
+			res.Description = "parse failed: " + err.Error()
+			out = append(out, res)
+			continue
+		}
+		if title := strings.TrimSpace(feed.Title); title != "" {
+			res.Name = title
+		}
+		res.Description = strings.TrimSpace(feed.Description)
+		if feed.Link != "" {
+			res.URL = feed.Link
+		}
+		if feed.UpdatedParsed != nil {
+			res.ModifiedAt = *feed.UpdatedParsed
+		}
+		res.Metadata = map[string]interface{}{"item_count": len(feed.Items)}
+		out = append(out, res)
+	}
+	return out, nil
+}
+
+// FetchAll performs a full sync of the specified feeds (or all configured feeds
+// when resourceIDs is empty).
+func (c *Connector) FetchAll(
+	ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
+) ([]types.FetchedItem, error) {
+	items, _, err := c.walk(ctx, config, resourceIDs, nil, false)
+	return items, err
+}
+
+// FetchIncremental returns only items whose content fingerprint changed since
+// the prior cursor. Deletions are intentionally not emitted (feeds drop old
+// items as a matter of course).
+func (c *Connector) FetchIncremental(
+	ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor,
+) ([]types.FetchedItem, *types.SyncCursor, error) {
+	var prev *rssCursor
+	if cursor != nil && cursor.ConnectorCursor != nil {
+		var p rssCursor
+		b, err := json.Marshal(cursor.ConnectorCursor)
+		if err != nil {
+			logger.Warnf(ctx, "[RSS] marshal connector cursor: %v", err)
+		} else if err := json.Unmarshal(b, &p); err != nil {
+			logger.Warnf(ctx, "[RSS] unmarshal connector cursor: %v", err)
+		} else {
+			prev = &p
+		}
+	}
+
+	items, newCursor, err := c.walk(ctx, config, config.ResourceIDs, prev, true)
+	if err != nil && newCursor == nil {
+		return nil, nil, err
+	}
+
+	cursorMap := make(map[string]interface{})
+	if newCursor != nil {
+		b, marshalErr := json.Marshal(newCursor)
+		if marshalErr != nil {
+			logger.Warnf(ctx, "[RSS] marshal new cursor: %v", marshalErr)
+		} else if unmarshalErr := json.Unmarshal(b, &cursorMap); unmarshalErr != nil {
+			logger.Warnf(ctx, "[RSS] unmarshal new cursor to map: %v", unmarshalErr)
+		}
+	}
+
+	var syncCursor *types.SyncCursor
+	if newCursor != nil {
+		syncCursor = &types.SyncCursor{
+			LastSyncTime:    newCursor.LastSyncTime,
+			ConnectorCursor: cursorMap,
+		}
+	}
+
+	return items, syncCursor, err
+}
+
+// walk is the shared implementation for FetchAll / FetchIncremental.
+//
+// When incremental is true, items whose feed signal and content fingerprint are
+// both unchanged are omitted from the result (ingest skipped) without fetching
+// article pages. The returned cursor always reflects the best-known item state
+// so a later sync can detect changes.
+func (c *Connector) walk(
+	ctx context.Context,
+	config *types.DataSourceConfig,
+	resourceIDs []string,
+	prev *rssCursor,
+	incremental bool,
+) ([]types.FetchedItem, *rssCursor, error) {
+	cfg, err := parseConfig(config)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Default to all configured feeds when no explicit selection was made.
+	feedURLs := resourceIDs
+	if len(feedURLs) == 0 {
+		feedURLs = cfg.feedURLList()
+	}
+
+	cli := newClient(cfg.parseHeaders())
+	parser := gofeed.NewParser()
+
+	newCursor := &rssCursor{
+		LastSyncTime: time.Now().UTC(),
+		FeedItems:    make(map[string]map[string]string),
+		FeedSignals:  make(map[string]map[string]string),
+	}
+	var out []types.FetchedItem
+	var feedErrors []string
+
+	for _, feedURL := range feedURLs {
+		data, err := cli.fetchFeed(ctx, feedURL)
+		if err != nil {
+			logger.Warnf(ctx, "[RSS] sync: fetch %s failed: %v", feedURL, err)
+			feedErrors = append(feedErrors, fmt.Sprintf("%s: %v", feedURL, err))
+			copyFeedCursor(newCursor, prev, feedURL)
+			continue
+		}
+		feed, err := parser.Parse(bytes.NewReader(data))
+		if err != nil {
+			logger.Warnf(ctx, "[RSS] sync: parse %s failed: %v", feedURL, err)
+			feedErrors = append(feedErrors, fmt.Sprintf("%s: %v", feedURL, err))
+			copyFeedCursor(newCursor, prev, feedURL)
+			continue
+		}
+
+		newCursor.FeedItems[feedURL] = make(map[string]string)
+		newCursor.FeedSignals[feedURL] = make(map[string]string)
+		var prevItems map[string]string
+		var prevSignals map[string]string
+		if incremental && prev != nil {
+			prevItems = prev.FeedItems[feedURL]
+			if prev.FeedSignals != nil {
+				prevSignals = prev.FeedSignals[feedURL]
+			}
+		}
+
+		var kept, skipped int
+		for _, item := range feed.Items {
+			if item == nil {
+				continue
+			}
+			itemID := firstNonEmpty(item.GUID, item.Link, item.Title)
+			if itemID == "" {
+				continue
+			}
+
+			feedContent := firstNonEmpty(item.Content, item.Description)
+			feedSig := feedSignalFingerprint(item, feedContent)
+
+			if incremental && prevItems != nil {
+				prevFP := prevItems[itemID]
+				prevSig := ""
+				if prevSignals != nil {
+					prevSig = prevSignals[itemID]
+				}
+				if prevFP != "" && feedSig == prevSig {
+					newCursor.FeedItems[feedURL][itemID] = prevFP
+					newCursor.FeedSignals[feedURL][itemID] = feedSig
+					skipped++
+					continue
+				}
+			}
+
+			resolved := c.resolveItem(ctx, cli, feed, item, feedURL, itemID, feedContent)
+			newCursor.FeedItems[feedURL][itemID] = resolved.fingerprint
+			newCursor.FeedSignals[feedURL][itemID] = feedSig
+
+			if incremental && prevItems != nil && prevItems[itemID] == resolved.fingerprint {
+				skipped++
+				continue
+			}
+			kept++
+			out = append(out, resolved.item)
+		}
+
+		logger.Infof(ctx, "[RSS] feed %s: items=%d fetched=%d skipped=%d",
+			feedURL, len(feed.Items), kept, skipped)
+	}
+
+	if len(feedErrors) > 0 {
+		if len(out) == 0 && len(feedErrors) == len(feedURLs) {
+			return nil, newCursor, fmt.Errorf("all feeds failed: %s", strings.Join(feedErrors, "; "))
+		}
+		return out, newCursor, &datasource.PartialFetchError{Details: feedErrors}
+	}
+
+	return out, newCursor, nil
+}
+
+type resolvedFeedItem struct {
+	item        types.FetchedItem
+	fingerprint string
+}
+
+// resolveItem assembles a FetchedItem for a single feed entry, resolving the
+// best available content (full-text article > feed content) and converting it
+// to Markdown.
+func (c *Connector) resolveItem(
+	ctx context.Context,
+	cli *client,
+	feed *gofeed.Feed,
+	item *gofeed.Item,
+	feedURL, itemID, feedContent string,
+) resolvedFeedItem {
+	title := firstNonEmpty(item.Title, "untitled")
+
+	// Prefer full article text; fall back to feed-provided content on failure.
+	contentHTML := feedContent
+	if strings.TrimSpace(item.Link) != "" {
+		if articleHTML, articleTitle, err := cli.extractArticle(ctx, item.Link); err == nil {
+			contentHTML = articleHTML
+			if item.Title == "" && articleTitle != "" {
+				title = articleTitle
+			}
+		} else {
+			logger.Warnf(ctx, "[RSS] full-text fetch failed for %s (using feed content): %v", item.Link, err)
+		}
+	}
+
+	content := htmlToMarkdown(contentHTML)
+
+	updatedAt := time.Now().UTC()
+	switch {
+	case item.UpdatedParsed != nil && !item.UpdatedParsed.IsZero():
+		updatedAt = *item.UpdatedParsed
+	case item.PublishedParsed != nil && !item.PublishedParsed.IsZero():
+		updatedAt = *item.PublishedParsed
+	}
+
+	author := ""
+	if item.Author != nil {
+		author = item.Author.Name
+	}
+
+	return resolvedFeedItem{
+		fingerprint: contentFingerprint(content),
+		item: types.FetchedItem{
+			ExternalID:       itemExternalID(feedURL, itemID),
+			Title:            title,
+			Content:          []byte(content),
+			ContentType:      "text/markdown",
+			FileName:         sanitizeFileName(title) + ".md",
+			URL:              item.Link,
+			UpdatedAt:        updatedAt,
+			SourceResourceID: feedURL,
+			Metadata: map[string]string{
+				"channel":    types.ChannelRSS,
+				"feed_url":   feedURL,
+				"feed_title": feed.Title,
+				"guid":       item.GUID,
+				"link":       item.Link,
+				"author":     author,
+			},
+		},
+	}
+}
+
+// htmlToMarkdown converts HTML to Markdown, returning the trimmed original on
+// conversion failure so we never silently drop content.
+func htmlToMarkdown(html string) string {
+	if strings.TrimSpace(html) == "" {
+		return ""
+	}
+	md, err := htmltomd.ConvertString(html)
+	if err != nil || strings.TrimSpace(md) == "" {
+		return strings.TrimSpace(html)
+	}
+	return strings.TrimSpace(md)
+}

internal/datasource/connector/rss/connector_test.go

@@ -0,0 +1,415 @@
+package rss
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/datasource"
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/Tencent/WeKnora/internal/utils"
+	"github.com/mmcdole/gofeed"
+)
+
+// TestMain whitelists loopback for SSRF so the httptest servers (127.0.0.1)
+// are reachable. Production keeps the default strict SSRF policy.
+func TestMain(m *testing.M) {
+	_ = os.Setenv("SSRF_WHITELIST", "127.0.0.1,::1")
+	utils.ResetSSRFWhitelistForTest()
+	code := m.Run()
+	os.Exit(code)
+}
+
+const longArticleBody = `<p>This is the first paragraph of a reasonably long article that the ` +
+	`readability extractor should detect as the main content of the page. It contains ` +
+	`enough words to clear the minimum content threshold used by the algorithm.</p>` +
+	`<p>The second paragraph continues the discussion with more sentences so that the ` +
+	`scoring heuristics confidently select this block over navigation and footer noise.</p>` +
+	`<p>A third paragraph adds further substance, ensuring the article is unmistakably ` +
+	`the dominant readable region of the document under test.</p>`
+
+// fakeFeed spins up an httptest server serving an RSS feed and article pages.
+type fakeFeed struct {
+	server             *httptest.Server
+	feedTitle          string
+	itemContent        string // optional <description>/content for items
+	articleAuthHeaders []string
+	articleFetches     atomic.Int32
+	failFeed           atomic.Bool
+}
+
+func newFakeFeed(t *testing.T) *fakeFeed {
+	t.Helper()
+	f := &fakeFeed{feedTitle: "Test Feed"}
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("/article/", func(w http.ResponseWriter, r *http.Request) {
+		f.articleFetches.Add(1)
+		for k, vals := range r.Header {
+			if strings.EqualFold(k, "X-Test-Auth") && len(vals) > 0 && vals[0] != "" {
+				f.articleAuthHeaders = append(f.articleAuthHeaders, vals[0])
+			}
+		}
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		fmt.Fprintf(w, `<!DOCTYPE html><html><head><title>%s</title></head>`+
+			`<body><nav>menu</nav><article><h1>Heading</h1>%s</article><footer>foot</footer></body></html>`,
+			"Full "+r.URL.Path, longArticleBody)
+	})
+
+	mux.HandleFunc("/feed.xml", func(w http.ResponseWriter, r *http.Request) {
+		if f.failFeed.Load() {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			return
+		}
+		if auth := r.Header.Get("X-Test-Auth"); f.itemContent == "needs-auth" && auth != "secret" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		w.Header().Set("Content-Type", "application/rss+xml")
+		base := "http://" + r.Host
+		desc := "summary fallback"
+		fmt.Fprintf(w, `<?xml version="1.0"?>
+<rss version="2.0"><channel>
+<title>%s</title>
+<link>%s</link>
+<description>A test feed</description>
+<item>
+  <title>Article One</title>
+  <link>%s/article/a1</link>
+  <guid>guid-1</guid>
+  <pubDate>Mon, 02 Jan 2006 15:04:05 GMT</pubDate>
+  <description>%s</description>
+</item>
+<item>
+  <title>Article Two</title>
+  <link>%s/article/a2</link>
+  <guid>guid-2</guid>
+  <pubDate>Tue, 03 Jan 2006 15:04:05 GMT</pubDate>
+  <description>%s</description>
+</item>
+</channel></rss>`, f.feedTitle, base, base, desc, base, desc)
+	})
+
+	f.server = httptest.NewServer(mux)
+	t.Cleanup(f.server.Close)
+	return f
+}
+
+func (f *fakeFeed) feedURL() string { return f.server.URL + "/feed.xml" }
+
+func makeConfig(feedURLs string, headers string) *types.DataSourceConfig {
+	cfg := &types.DataSourceConfig{
+		Type: types.ConnectorTypeRSS,
+		Settings: map[string]interface{}{
+			"feed_urls": feedURLs,
+		},
+		Credentials: map[string]interface{}{},
+	}
+	if headers != "" {
+		cfg.Credentials["auth_headers"] = headers
+	}
+	return cfg
+}
+
+func TestConnector_Type(t *testing.T) {
+	if NewConnector().Type() != types.ConnectorTypeRSS {
+		t.Fatalf("Type() = %q, want %q", NewConnector().Type(), types.ConnectorTypeRSS)
+	}
+}
+
+func TestParseConfig_RequiresFeedURLs(t *testing.T) {
+	if _, err := parseConfig(makeConfig("   ", "")); err == nil {
+		t.Fatal("expected error when feed_urls is blank")
+	}
+}
+
+func TestParseConfig_FeedURLsFromSettings(t *testing.T) {
+	cfg, err := parseConfig(makeConfig("https://example.com/feed.xml", ""))
+	if err != nil {
+		t.Fatalf("parseConfig error: %v", err)
+	}
+	got := cfg.feedURLList()
+	if len(got) != 1 || got[0] != "https://example.com/feed.xml" {
+		t.Fatalf("feedURLList = %v", got)
+	}
+}
+
+func TestParseConfig_LegacyFeedURLsInCredentials(t *testing.T) {
+	legacy := &types.DataSourceConfig{
+		Type: types.ConnectorTypeRSS,
+		Credentials: map[string]interface{}{
+			"feed_urls": "https://legacy.example/feed.xml",
+		},
+	}
+	cfg, err := parseConfig(legacy)
+	if err != nil {
+		t.Fatalf("parseConfig error: %v", err)
+	}
+	if got := cfg.feedURLList(); len(got) != 1 || got[0] != "https://legacy.example/feed.xml" {
+		t.Fatalf("feedURLList = %v", got)
+	}
+}
+
+func TestConfig_FeedURLList_SplitsAndDedupes(t *testing.T) {
+	cfg := &Config{FeedURLs: "https://a.com/f, https://b.com/f\nhttps://a.com/f\n"}
+	got := cfg.feedURLList()
+	want := []string{"https://a.com/f", "https://b.com/f"}
+	if strings.Join(got, "|") != strings.Join(want, "|") {
+		t.Fatalf("feedURLList = %v, want %v", got, want)
+	}
+}
+
+func TestConfig_ParseHeaders(t *testing.T) {
+	cfg := &Config{AuthHeaders: "Authorization: Bearer x\nX-Foo:  bar \nbroken-line\n: noname"}
+	got := cfg.parseHeaders()
+	if got["Authorization"] != "Bearer x" {
+		t.Errorf("Authorization = %q", got["Authorization"])
+	}
+	if got["X-Foo"] != "bar" {
+		t.Errorf("X-Foo = %q", got["X-Foo"])
+	}
+	if len(got) != 2 {
+		t.Errorf("expected 2 headers, got %d: %v", len(got), got)
+	}
+}
+
+func TestFeedSignalFingerprint(t *testing.T) {
+	item := &gofeed.Item{GUID: "g1", Link: "https://example.com/a", Title: "t"}
+	sig1 := feedSignalFingerprint(item, "body")
+	sig2 := feedSignalFingerprint(item, "body")
+	if sig1 == "" || sig1 != sig2 {
+		t.Fatalf("feed signal unstable: %q vs %q", sig1, sig2)
+	}
+	sig3 := feedSignalFingerprint(item, "changed")
+	if sig1 == sig3 {
+		t.Fatal("expected different signal when feed content changes")
+	}
+}
+
+func TestItemExternalID_ScopesByFeed(t *testing.T) {
+	got := itemExternalID("https://a.com/feed", "guid-1")
+	want := "https://a.com/feed:guid-1"
+	if got != want {
+		t.Fatalf("itemExternalID = %q, want %q", got, want)
+	}
+}
+
+func TestConnector_Validate_Success(t *testing.T) {
+	f := newFakeFeed(t)
+	if err := NewConnector().Validate(context.Background(), makeConfig(f.feedURL(), "")); err != nil {
+		t.Fatalf("Validate error: %v", err)
+	}
+}
+
+func TestConnector_Validate_PrivateFeedWithHeader(t *testing.T) {
+	f := newFakeFeed(t)
+	f.itemContent = "needs-auth"
+
+	// Without the header → 401.
+	if err := NewConnector().Validate(context.Background(), makeConfig(f.feedURL(), "")); err == nil {
+		t.Fatal("expected error without auth header")
+	}
+	// With the header → success.
+	if err := NewConnector().Validate(
+		context.Background(), makeConfig(f.feedURL(), "X-Test-Auth: secret"),
+	); err != nil {
+		t.Fatalf("Validate with header error: %v", err)
+	}
+}
+
+func TestConnector_FetchAll_DoesNotSendAuthHeadersToArticles(t *testing.T) {
+	f := newFakeFeed(t)
+	f.itemContent = "needs-auth"
+	_, err := NewConnector().FetchAll(
+		context.Background(), makeConfig(f.feedURL(), "X-Test-Auth: secret"), nil,
+	)
+	if err != nil {
+		t.Fatalf("FetchAll error: %v", err)
+	}
+	if len(f.articleAuthHeaders) != 0 {
+		t.Fatalf("article requests must not carry feed auth headers, got %v", f.articleAuthHeaders)
+	}
+}
+
+func TestConnector_ListResources(t *testing.T) {
+	f := newFakeFeed(t)
+	res, err := NewConnector().ListResources(context.Background(), makeConfig(f.feedURL(), ""), "")
+	if err != nil {
+		t.Fatalf("ListResources error: %v", err)
+	}
+	if len(res) != 1 {
+		t.Fatalf("expected 1 resource, got %d", len(res))
+	}
+	if res[0].ExternalID != f.feedURL() {
+		t.Errorf("ExternalID = %q, want %q", res[0].ExternalID, f.feedURL())
+	}
+	if res[0].Name != "Test Feed" {
+		t.Errorf("Name = %q, want %q", res[0].Name, "Test Feed")
+	}
+
+	// Non-empty parentID → empty (feeds are flat).
+	children, err := NewConnector().ListResources(context.Background(), makeConfig(f.feedURL(), ""), "feed-x")
+	if err != nil {
+		t.Fatalf("ListResources(parent) error: %v", err)
+	}
+	if len(children) != 0 {
+		t.Fatalf("expected no children, got %d", len(children))
+	}
+}
+
+func TestConnector_FetchAll_FullTextMarkdown(t *testing.T) {
+	f := newFakeFeed(t)
+	items, err := NewConnector().FetchAll(context.Background(), makeConfig(f.feedURL(), ""), nil)
+	if err != nil {
+		t.Fatalf("FetchAll error: %v", err)
+	}
+	if len(items) != 2 {
+		t.Fatalf("expected 2 items, got %d", len(items))
+	}
+
+	it := items[0]
+	wantExternalID := itemExternalID(f.feedURL(), "guid-1")
+	if it.ExternalID != wantExternalID {
+		t.Errorf("ExternalID = %q, want %q", it.ExternalID, wantExternalID)
+	}
+	if it.ContentType != "text/markdown" {
+		t.Errorf("ContentType = %q, want text/markdown", it.ContentType)
+	}
+	if it.Metadata["channel"] != types.ChannelRSS {
+		t.Errorf("channel = %q, want %q", it.Metadata["channel"], types.ChannelRSS)
+	}
+	// Full text from the article page should be present (not the short summary).
+	if !strings.Contains(string(it.Content), "first paragraph") {
+		t.Errorf("expected full article text in content, got: %q", string(it.Content))
+	}
+	if !strings.HasSuffix(it.FileName, ".md") {
+		t.Errorf("FileName = %q, want .md suffix", it.FileName)
+	}
+}
+
+func TestConnector_FetchIncremental_SkipsWithoutArticleFetch(t *testing.T) {
+	f := newFakeFeed(t)
+	cfg := makeConfig(f.feedURL(), "")
+	cfg.ResourceIDs = []string{f.feedURL()}
+
+	items, cursor, err := NewConnector().FetchIncremental(context.Background(), cfg, nil)
+	if err != nil {
+		t.Fatalf("first FetchIncremental error: %v", err)
+	}
+	if len(items) != 2 {
+		t.Fatalf("expected 2 items on first sync, got %d", len(items))
+	}
+	if got := f.articleFetches.Load(); got != 2 {
+		t.Fatalf("first sync article fetches = %d, want 2", got)
+	}
+
+	f.articleFetches.Store(0)
+	items2, _, err := NewConnector().FetchIncremental(context.Background(), cfg, cursor)
+	if err != nil {
+		t.Fatalf("second FetchIncremental error: %v", err)
+	}
+	if len(items2) != 0 {
+		t.Fatalf("expected 0 items on unchanged second sync, got %d", len(items2))
+	}
+	if got := f.articleFetches.Load(); got != 0 {
+		t.Fatalf("unchanged incremental sync must not refetch articles, got %d fetches", got)
+	}
+}
+
+func TestConnector_FetchIncremental_SkipsUnchanged(t *testing.T) {
+	f := newFakeFeed(t)
+	cfg := makeConfig(f.feedURL(), "")
+	cfg.ResourceIDs = []string{f.feedURL()}
+
+	// First sync: everything is new.
+	items, cursor, err := NewConnector().FetchIncremental(context.Background(), cfg, nil)
+	if err != nil {
+		t.Fatalf("first FetchIncremental error: %v", err)
+	}
+	if len(items) != 2 {
+		t.Fatalf("expected 2 items on first sync, got %d", len(items))
+	}
+	if cursor == nil {
+		t.Fatal("expected non-nil cursor")
+	}
+
+	// Second sync with the returned cursor: nothing changed → no items.
+	items2, _, err := NewConnector().FetchIncremental(context.Background(), cfg, cursor)
+	if err != nil {
+		t.Fatalf("second FetchIncremental error: %v", err)
+	}
+	if len(items2) != 0 {
+		t.Fatalf("expected 0 items on unchanged second sync, got %d", len(items2))
+	}
+}
+
+func TestConnector_Walk_PartialFeedFailure(t *testing.T) {
+	f := newFakeFeed(t)
+	cfg := makeConfig(f.feedURL()+", https://invalid.invalid/feed.xml", "")
+	items, err := NewConnector().FetchAll(context.Background(), cfg, nil)
+	var partial *datasource.PartialFetchError
+	if !errors.As(err, &partial) {
+		t.Fatalf("expected PartialFetchError, got %v", err)
+	}
+	if len(partial.Details) != 1 {
+		t.Fatalf("expected 1 feed error, got %v", partial.Details)
+	}
+	if len(items) != 2 {
+		t.Fatalf("expected 2 items from healthy feed, got %d", len(items))
+	}
+}
+
+func TestConnector_Walk_PreservesCursorOnFeedFailure(t *testing.T) {
+	f := newFakeFeed(t)
+	cfg := makeConfig(f.feedURL(), "")
+	cfg.ResourceIDs = []string{f.feedURL()}
+
+	_, cursor, err := NewConnector().FetchIncremental(context.Background(), cfg, nil)
+	if err != nil {
+		t.Fatalf("first sync error: %v", err)
+	}
+
+	f.failFeed.Store(true)
+	_, newCursor, err := NewConnector().FetchIncremental(context.Background(), cfg, cursor)
+	if err == nil {
+		t.Fatal("expected error when sole feed is unavailable")
+	}
+	if newCursor == nil || newCursor.ConnectorCursor == nil {
+		t.Fatal("expected cursor to be preserved on feed failure")
+	}
+
+	var restored rssCursor
+	b, err := json.Marshal(newCursor.ConnectorCursor)
+	if err != nil {
+		t.Fatalf("marshal cursor: %v", err)
+	}
+	if err := json.Unmarshal(b, &restored); err != nil {
+		t.Fatalf("unmarshal cursor: %v", err)
+	}
+	items := restored.FeedItems[f.feedURL()]
+	if len(items) != 2 {
+		t.Fatalf("expected preserved fingerprints for 2 items, got %d", len(items))
+	}
+	if items["guid-1"] == "" || items["guid-2"] == "" {
+		t.Fatalf("expected non-empty preserved fingerprints, got %+v", items)
+	}
+}
+
+func TestConnector_ResolveResourceAncestors_Empty(t *testing.T) {
+	got, err := NewConnector().ResolveResourceAncestors(
+		context.Background(), makeConfig("https://a.com/f", ""), []string{"x"},
+	)
+	if err != nil {
+		t.Fatalf("ResolveResourceAncestors error: %v", err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("expected empty ancestors, got %v", got)
+	}
+}

internal/datasource/connector/rss/types.go

@@ -0,0 +1,249 @@
+// Package rss implements the RSS/Atom data source connector for WeKnora.
+//
+// It syncs articles from one or more RSS/Atom/JSON feeds into a WeKnora
+// knowledge base. Each configured feed URL is treated as a selectable
+// resource; each feed item becomes a knowledge entry whose body is the
+// article's full text rendered as Markdown.
+//
+// Capabilities:
+//   - Private feeds: optional custom HTTP headers (e.g. "Authorization: Bearer …")
+//     are attached only to feed fetches, never to third-party article pages.
+//   - Full text: when an item exposes a link, the article page is fetched and
+//     run through a readability extractor; the cleaned HTML is converted to
+//     Markdown. Feed-provided content (content:encoded / description) is used
+//     as a fallback.
+//   - Incremental: feed-level signals skip full-text article fetches when a feed
+//     entry is unchanged; content fingerprints detect feed-body changes. Article-
+//     only edits without feed updates may be missed until the feed entry changes.
+//     Deletions are NOT synced — feeds routinely drop old items.
+//
+// All outbound requests go through the SSRF-safe HTTP client so a malicious
+// feed cannot redirect WeKnora to internal services.
+package rss
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"maps"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/Tencent/WeKnora/internal/datasource"
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/mmcdole/gofeed"
+)
+
+// Config holds RSS-specific configuration.
+//
+// FeedURLs are stored in DataSourceConfig.Settings (non-secret, editable in
+// the UI without replacing credentials). AuthHeaders live in Credentials
+// because they may carry secrets that must be encrypted at rest. Credentials
+// may still carry feed_urls for backward compatibility with older rows.
+type Config struct {
+	// FeedURLs is a newline- or comma-separated list of feed URLs.
+	FeedURLs string `json:"feed_urls"`
+
+	// AuthHeaders is an optional newline-separated list of custom request
+	// headers in "Name: Value" form, applied only to feed fetches.
+	AuthHeaders string `json:"auth_headers,omitempty"`
+}
+
+// parseConfig extracts and validates RSS configuration.
+func parseConfig(config *types.DataSourceConfig) (*Config, error) {
+	if config == nil {
+		return nil, fmt.Errorf("%w: config is nil", datasource.ErrInvalidConfig)
+	}
+	credBytes, err := json.Marshal(config.Credentials)
+	if err != nil {
+		return nil, fmt.Errorf("marshal credentials: %w", err)
+	}
+	var cfg Config
+	if err := json.Unmarshal(credBytes, &cfg); err != nil {
+		return nil, fmt.Errorf("parse rss credentials: %w", err)
+	}
+	if urls := feedURLsFromSettings(config.Settings); urls != "" {
+		cfg.FeedURLs = urls
+	}
+	if len(cfg.feedURLList()) == 0 {
+		return nil, fmt.Errorf("%w: feed_urls is required", datasource.ErrInvalidCredentials)
+	}
+	return &cfg, nil
+}
+
+func feedURLsFromSettings(settings map[string]interface{}) string {
+	if len(settings) == 0 {
+		return ""
+	}
+	raw, ok := settings["feed_urls"]
+	if !ok {
+		return ""
+	}
+	s, ok := raw.(string)
+	if !ok {
+		return ""
+	}
+	return strings.TrimSpace(s)
+}
+
+// feedURLList splits FeedURLs on newlines and commas, trims, dedupes (order
+// preserved), and drops blanks.
+func (c *Config) feedURLList() []string {
+	if c == nil {
+		return nil
+	}
+	raw := strings.FieldsFunc(c.FeedURLs, func(r rune) bool {
+		return r == '\n' || r == '\r' || r == ','
+	})
+	seen := make(map[string]struct{}, len(raw))
+	out := make([]string, 0, len(raw))
+	for _, u := range raw {
+		u = strings.TrimSpace(u)
+		if u == "" {
+			continue
+		}
+		if _, ok := seen[u]; ok {
+			continue
+		}
+		seen[u] = struct{}{}
+		out = append(out, u)
+	}
+	return out
+}
+
+// parseHeaders turns the newline-separated "Name: Value" AuthHeaders blob into
+// a map. Lines without a colon, or with an empty name, are skipped.
+func (c *Config) parseHeaders() map[string]string {
+	if c == nil || strings.TrimSpace(c.AuthHeaders) == "" {
+		return nil
+	}
+	headers := make(map[string]string)
+	for _, line := range strings.Split(c.AuthHeaders, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		idx := strings.Index(line, ":")
+		if idx <= 0 {
+			continue
+		}
+		name := strings.TrimSpace(line[:idx])
+		value := strings.TrimSpace(line[idx+1:])
+		if name == "" {
+			continue
+		}
+		headers[name] = value
+	}
+	if len(headers) == 0 {
+		return nil
+	}
+	return headers
+}
+
+// rssCursor stores incremental sync state.
+//
+// FeedItems maps feedURL → itemID → fingerprint, where fingerprint is a
+// "h:<sha256-prefix>" hash of the final Markdown body that would be ingested.
+// FeedSignals maps feedURL → itemID → feed-only signal used to skip full-text
+// article fetches when the feed entry itself has not changed.
+type rssCursor struct {
+	LastSyncTime time.Time                    `json:"last_sync_time"`
+	FeedItems    map[string]map[string]string `json:"feed_items,omitempty"`
+	FeedSignals  map[string]map[string]string `json:"feed_signals,omitempty"`
+}
+
+// contentFingerprint hashes ingested Markdown for incremental change detection.
+func contentFingerprint(markdown string) string {
+	sum := sha256.Sum256([]byte(markdown))
+	return "h:" + hex.EncodeToString(sum[:])[:16]
+}
+
+// feedSignalFingerprint hashes feed-visible fields so incremental sync can skip
+// article-page fetches when the entry has not changed in the feed.
+func feedSignalFingerprint(item *gofeed.Item, feedContent string) string {
+	if item == nil {
+		return ""
+	}
+	var b strings.Builder
+	b.WriteString(item.GUID)
+	b.WriteByte('\n')
+	b.WriteString(item.Link)
+	b.WriteByte('\n')
+	b.WriteString(item.Title)
+	b.WriteByte('\n')
+	if item.UpdatedParsed != nil && !item.UpdatedParsed.IsZero() {
+		b.WriteString(item.UpdatedParsed.UTC().Format(time.RFC3339))
+	}
+	b.WriteByte('\n')
+	if item.PublishedParsed != nil && !item.PublishedParsed.IsZero() {
+		b.WriteString(item.PublishedParsed.UTC().Format(time.RFC3339))
+	}
+	b.WriteByte('\n')
+	b.WriteString(feedContent)
+	sum := sha256.Sum256([]byte(b.String()))
+	return "s:" + hex.EncodeToString(sum[:])[:16]
+}
+
+func copyFeedCursor(dst, prev *rssCursor, feedURL string) {
+	if dst == nil || prev == nil {
+		return
+	}
+	if src, ok := prev.FeedItems[feedURL]; ok && len(src) > 0 {
+		dst.FeedItems[feedURL] = maps.Clone(src)
+	}
+	if prev.FeedSignals != nil {
+		if src, ok := prev.FeedSignals[feedURL]; ok && len(src) > 0 {
+			if dst.FeedSignals == nil {
+				dst.FeedSignals = make(map[string]map[string]string)
+			}
+			dst.FeedSignals[feedURL] = maps.Clone(src)
+		}
+	}
+}
+
+// itemExternalID scopes an item ID to its feed so GUIDs cannot collide across feeds.
+func itemExternalID(feedURL, itemID string) string {
+	return feedURL + ":" + itemID
+}
+
+// firstNonEmpty returns the first non-empty trimmed string among the args.
+func firstNonEmpty(values ...string) string {
+	for _, v := range values {
+		if strings.TrimSpace(v) != "" {
+			return v
+		}
+	}
+	return ""
+}
+
+// sanitizeFileName removes characters invalid in filenames and truncates to a
+// safe length at a UTF-8 rune boundary (mirrors the Yuque connector).
+func sanitizeFileName(name string) string {
+	name = strings.TrimSpace(name)
+	if name == "" {
+		return "untitled"
+	}
+	replacer := strings.NewReplacer(
+		"/", "_", "\\", "_", ":", "_", "*", "_",
+		"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
+		"\n", " ", "\r", " ", "\t", " ",
+	)
+	result := strings.TrimSpace(replacer.Replace(name))
+	if result == "" {
+		return "untitled"
+	}
+	const maxBytes = 200
+	if len(result) > maxBytes {
+		result = result[:maxBytes]
+		for len(result) > 0 {
+			r, size := utf8.DecodeLastRuneInString(result)
+			if r != utf8.RuneError || size != 1 {
+				break
+			}
+			result = result[:len(result)-1]
+		}
+	}
+	return result
+}