WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/datasource/connector/rss/client.go

@@ -0,0 +1,123 @@
+package rss
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+
+	readability "codeberg.org/readeck/go-readability/v2"
+	"github.com/Tencent/WeKnora/internal/utils"
+)
+
+const (
+	// requestTimeout bounds a single feed or article fetch.
+	requestTimeout = 20 * time.Second
+
+	// maxFeedSize caps the feed body to avoid memory blowups on hostile feeds.
+	maxFeedSize = 10 * 1024 * 1024 // 10 MB
+
+	// maxArticleSize caps a single article page body.
+	maxArticleSize = 5 * 1024 * 1024 // 5 MB
+
+	// defaultUserAgent is sent on every request; some feeds reject empty UAs.
+	defaultUserAgent = "Mozilla/5.0 (compatible; WeKnora-RSS/1.0; +https://weknora.weixin.qq.com)"
+)
+
+// client performs SSRF-safe HTTP fetches with optional custom headers.
+type client struct {
+	httpClient *http.Client
+	headers    map[string]string
+}
+
+func newClient(headers map[string]string) *client {
+	cfg := utils.DefaultSSRFSafeHTTPClientConfig()
+	cfg.Timeout = requestTimeout
+	return &client{
+		httpClient: utils.NewSSRFSafeHTTPClient(cfg),
+		headers:    headers,
+	}
+}
+
+// fetch retrieves rawURL with SSRF validation and size limiting. Custom auth
+// headers are only attached when withAuthHeaders is true (feed fetches); article
+// pages on third-party domains must not receive feed credentials.
+func (c *client) fetch(ctx context.Context, rawURL string, maxSize int64, withAuthHeaders bool) ([]byte, error) {
+	if err := utils.ValidateURLForSSRF(rawURL); err != nil {
+		return nil, fmt.Errorf("URL rejected: %w", err)
+	}
+	if _, err := url.Parse(rawURL); err != nil {
+		return nil, fmt.Errorf("invalid URL: %w", err)
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, requestTimeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if withAuthHeaders {
+		for k, v := range c.headers {
+			req.Header.Set(k, v)
+		}
+	}
+	if req.Header.Get("User-Agent") == "" {
+		req.Header.Set("User-Agent", defaultUserAgent)
+	}
+	if req.Header.Get("Accept") == "" {
+		req.Header.Set("Accept",
+			"application/rss+xml, application/atom+xml, application/xml, text/xml, application/json, text/html;q=0.9, */*;q=0.8")
+	}
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("fetch failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status)
+	}
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, maxSize))
+	if err != nil {
+		return nil, fmt.Errorf("read body failed: %w", err)
+	}
+	return body, nil
+}
+
+// fetchFeed retrieves the raw bytes of a feed document.
+func (c *client) fetchFeed(ctx context.Context, feedURL string) ([]byte, error) {
+	return c.fetch(ctx, feedURL, maxFeedSize, true)
+}
+
+// extractArticle fetches an article page and returns the readability-cleaned
+// main content as HTML, plus the extracted title (may be empty). Returns an
+// error if the page can't be fetched or no readable content is found, so the
+// caller can fall back to feed-provided content.
+func (c *client) extractArticle(ctx context.Context, articleURL string) (contentHTML, title string, err error) {
+	body, err := c.fetch(ctx, articleURL, maxArticleSize, false)
+	if err != nil {
+		return "", "", err
+	}
+
+	pageURL, _ := url.Parse(articleURL)
+	article, err := readability.FromReader(bytes.NewReader(body), pageURL)
+	if err != nil {
+		return "", "", fmt.Errorf("readability parse: %w", err)
+	}
+	if article.Node == nil {
+		return "", "", fmt.Errorf("no readable content extracted")
+	}
+
+	var buf bytes.Buffer
+	if err := article.RenderHTML(&buf); err != nil {
+		return "", "", fmt.Errorf("render article html: %w", err)
+	}
+	return buf.String(), article.Title(), nil
+}