WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/datasource/connector/gitlab/client.go

@@ -0,0 +1,267 @@
+package gitlab
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/datasource"
+)
+
+type client struct {
+	baseURL, token string
+	http           *http.Client
+}
+type apiError struct {
+	endpoint string
+	status   int
+}
+
+func (e *apiError) Error() string {
+	return fmt.Sprintf("gitlab API %s: status %d", e.endpoint, e.status)
+}
+
+type project struct {
+	ID                int64  `json:"id"`
+	PathWithNamespace string `json:"path_with_namespace"`
+	Name              string `json:"name"`
+	WebURL            string `json:"web_url"`
+	DefaultBranch     string `json:"default_branch"`
+	Namespace         struct {
+		ID int64 `json:"id"`
+	} `json:"namespace"`
+}
+type treeEntry struct {
+	ID   string `json:"id"`
+	Name string `json:"name"`
+	Type string `json:"type"`
+	Path string `json:"path"`
+}
+type comparison struct {
+	Diffs []struct {
+		OldPath     string `json:"old_path"`
+		NewPath     string `json:"new_path"`
+		NewFile     bool   `json:"new_file"`
+		DeletedFile bool   `json:"deleted_file"`
+		RenamedFile bool   `json:"renamed_file"`
+	} `json:"diffs"`
+	CompareTimeout bool `json:"compare_timeout"`
+	CompareSameRef bool `json:"compare_same_ref"`
+}
+
+func newClient(baseURL, token string) (*client, error) {
+	baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
+	if baseURL == "" || strings.TrimSpace(token) == "" {
+		return nil, fmt.Errorf("GitLab platform configuration is missing")
+	}
+	if err := datasource.ValidateConnectorBaseURL(baseURL); err != nil {
+		return nil, err
+	}
+	if !strings.Contains(baseURL, "://") {
+		baseURL = "https://" + baseURL
+	}
+	if !strings.HasSuffix(baseURL, "/api/v4") {
+		baseURL += "/api/v4"
+	}
+	return &client{baseURL: baseURL, token: token, http: datasource.NewConnectorHTTPClient(30 * time.Second)}, nil
+}
+func (c *client) get(ctx context.Context, endpoint string, out interface{}) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+endpoint, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("PRIVATE-TOKEN", c.token)
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return &apiError{endpoint: endpoint, status: resp.StatusCode}
+	}
+	return json.Unmarshal(body, out)
+}
+func (c *client) getRaw(ctx context.Context, endpoint string) ([]byte, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+endpoint, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("PRIVATE-TOKEN", c.token)
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, &apiError{endpoint: endpoint, status: resp.StatusCode}
+	}
+	return io.ReadAll(resp.Body)
+}
+
+// projectPath encodes a GitLab project identifier for URL path segments.
+// Numeric IDs are used verbatim. Namespace paths accept either "group/project"
+// or a once-encoded "group%2Fproject" without double-encoding percent signs.
+func projectPath(id string) string {
+	id = strings.TrimSpace(id)
+	if id == "" {
+		return ""
+	}
+	if _, err := strconv.ParseInt(id, 10, 64); err == nil {
+		return id
+	}
+	decoded := id
+	if strings.Contains(id, "%") {
+		if unescaped, err := url.PathUnescape(id); err == nil {
+			decoded = unescaped
+		}
+	}
+	if strings.Contains(decoded, "/") {
+		parts := strings.Split(decoded, "/")
+		for i, part := range parts {
+			parts[i] = url.PathEscape(part)
+		}
+		return strings.Join(parts, "%2F")
+	}
+	return url.PathEscape(decoded)
+}
+func (c *client) project(ctx context.Context, id string) (*project, error) {
+	var p project
+	err := c.get(ctx, "/projects/"+projectPath(id), &p)
+	return &p, err
+}
+func (c *client) projects(ctx context.Context) ([]project, error) {
+	var p []project
+	err := c.get(ctx, "/projects?membership=true&per_page=100&order_by=path_with_namespace&sort=asc", &p)
+	return p, err
+}
+
+// ping verifies that the supplied private token is accepted by this GitLab
+// instance. It avoids project-list ordering parameters that older GitLab
+// deployments may reject even when the token is valid.
+func (c *client) ping(ctx context.Context) error {
+	var user struct {
+		ID int64 `json:"id"`
+	}
+	return c.get(ctx, "/user", &user)
+}
+func (c *client) commitSHA(ctx context.Context, id, ref string) (string, error) {
+	var v struct {
+		ID string `json:"id"`
+	}
+	err := c.get(ctx, "/projects/"+projectPath(id)+"/repository/commits/"+url.PathEscape(ref), &v)
+	return v.ID, err
+}
+func (c *client) tree(ctx context.Context, id, ref, dir string) ([]treeEntry, error) {
+	q := url.Values{"ref": {ref}, "per_page": {"100"}, "page": {"1"}}
+	if dir != "" {
+		q.Set("path", dir)
+	}
+	endpoint := "/projects/" + projectPath(id) + "/repository/tree"
+	var all []treeEntry
+	for {
+		var page []treeEntry
+		nextPage, err := c.getTreePage(ctx, endpoint, q, &page)
+		if err != nil {
+			return nil, err
+		}
+		all = append(all, page...)
+		if nextPage == "" {
+			return all, nil
+		}
+		q.Set("page", nextPage)
+	}
+}
+
+func (c *client) getTreePage(ctx context.Context, endpoint string, query url.Values, out interface{}) (string, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+endpoint+"?"+query.Encode(), nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("PRIVATE-TOKEN", c.token)
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return "", &apiError{endpoint: endpoint, status: resp.StatusCode}
+	}
+	if err := json.Unmarshal(body, out); err != nil {
+		return "", err
+	}
+	return resp.Header.Get("X-Next-Page"), nil
+}
+func (c *client) raw(ctx context.Context, id, ref, file string) ([]byte, error) {
+	q := url.Values{"ref": {ref}}
+	encodedFile := gitlabFilePathEscape(file)
+	rawEndpoint := "/projects/" + projectPath(id) + "/repository/files/" + encodedFile + "/raw?" + q.Encode()
+	content, err := c.getRaw(ctx, rawEndpoint)
+	if err == nil {
+		return content, nil
+	}
+	var apiErr *apiError
+	if !errors.As(err, &apiErr) || apiErr.status != http.StatusNotFound {
+		return nil, fmt.Errorf("gitlab raw file: %w", err)
+	}
+
+	// Some GitLab deployments expose the file detail endpoint but return 404
+	// for the otherwise standard /raw route. The detail response contains the
+	// same content as base64 and provides a compatible fallback.
+	var detail struct {
+		Encoding string `json:"encoding"`
+		Content  string `json:"content"`
+	}
+	detailEndpoint := "/projects/" + projectPath(id) + "/repository/files/" + encodedFile + "?" + q.Encode()
+	if err := c.get(ctx, detailEndpoint, &detail); err != nil {
+		return nil, fmt.Errorf("gitlab file content: %w", err)
+	}
+	if detail.Encoding != "base64" {
+		return nil, fmt.Errorf("gitlab file content: unsupported encoding %q", detail.Encoding)
+	}
+	content, err = base64.StdEncoding.DecodeString(detail.Content)
+	if err != nil {
+		return nil, fmt.Errorf("gitlab file content: decode base64: %w", err)
+	}
+	return content, nil
+}
+
+// gitlabFilePathEscape mirrors the company GitLab raw-file route: only ASCII
+// letters, digits, hyphen and underscore remain literal. In particular dots,
+// path separators and UTF-8 bytes must be percent-encoded.
+func gitlabFilePathEscape(file string) string {
+	const hex = "0123456789ABCDEF"
+	var b strings.Builder
+	b.Grow(len(file) * 3)
+	for _, c := range []byte(file) {
+		if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || c == '_' {
+			b.WriteByte(c)
+			continue
+		}
+		b.WriteByte('%')
+		b.WriteByte(hex[c>>4])
+		b.WriteByte(hex[c&0x0f])
+	}
+	return b.String()
+}
+func (c *client) compare(ctx context.Context, id, from, to string) (*comparison, error) {
+	q := url.Values{"from": {from}, "to": {to}}
+	var v comparison
+	err := c.get(ctx, "/projects/"+projectPath(id)+"/repository/compare?"+q.Encode(), &v)
+	return &v, err
+}

internal/datasource/connector/gitlab/client_test.go

@@ -0,0 +1,408 @@
+package gitlab
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+	secutils "github.com/Tencent/WeKnora/internal/utils"
+)
+
+func allowLocalGitLabServer(t *testing.T) {
+	t.Helper()
+	t.Setenv("SSRF_WHITELIST", "127.0.0.1,::1,localhost")
+	secutils.ResetSSRFWhitelistForTest()
+	t.Cleanup(secutils.ResetSSRFWhitelistForTest)
+}
+
+func TestConnectorValidateUsesDataSourceCredentials(t *testing.T) {
+	allowLocalGitLabServer(t)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/v4/user" {
+			http.NotFound(w, r)
+			return
+		}
+		if got := r.Header.Get("PRIVATE-TOKEN"); got != "per-source-token" {
+			t.Fatalf("PRIVATE-TOKEN = %q", got)
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"id": 42}`))
+	}))
+	defer server.Close()
+
+	ds := &types.DataSourceConfig{
+		Credentials: map[string]interface{}{
+			"base_url":     server.URL,
+			"access_token": "per-source-token",
+		},
+	}
+
+	if err := NewConnector().Validate(context.Background(), ds); err != nil {
+		t.Fatalf("Validate() error = %v", err)
+	}
+}
+
+func TestConnectorValidateReturnsGitLabAPIError(t *testing.T) {
+	allowLocalGitLabServer(t)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/v4/user" {
+			http.NotFound(w, r)
+			return
+		}
+		http.Error(w, "invalid token", http.StatusUnauthorized)
+	}))
+	defer server.Close()
+
+	err := NewConnector().Validate(context.Background(), &types.DataSourceConfig{
+		Credentials: map[string]interface{}{
+			"base_url":     server.URL,
+			"access_token": "invalid-token",
+		},
+	})
+	var apiErr *apiError
+	if !errors.As(err, &apiErr) {
+		t.Fatalf("Validate() error = %v, want GitLab API error", err)
+	}
+	if apiErr.endpoint != "/user" || apiErr.status != http.StatusUnauthorized {
+		t.Fatalf("apiError = %#v", apiErr)
+	}
+}
+
+func TestConnectorValidateRejectsMissingCredentials(t *testing.T) {
+	err := NewConnector().Validate(context.Background(), &types.DataSourceConfig{
+		Credentials: map[string]interface{}{"base_url": "https://gitlab.example.com"},
+	})
+	if err == nil || err.Error() != "GitLab platform configuration is missing" {
+		t.Fatalf("Validate() error = %v", err)
+	}
+}
+
+func TestConnectorValidateRejectsMissingProjectsOnSave(t *testing.T) {
+	allowLocalGitLabServer(t)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/v4/user" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"id": 42}`))
+	}))
+	defer server.Close()
+
+	err := NewConnector().Validate(context.Background(), &types.DataSourceConfig{
+		Credentials: map[string]interface{}{
+			"base_url":     server.URL,
+			"access_token": "token",
+		},
+		Settings: map[string]interface{}{
+			"projects": []interface{}{},
+		},
+	})
+	if err == nil {
+		t.Fatal("expected missing projects validation error")
+	}
+}
+
+func TestConnectorConfiguredDoesNotReuseRegistryClient(t *testing.T) {
+	allowLocalGitLabServer(t)
+
+	var requestedTokens []string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/v4/user" {
+			http.NotFound(w, r)
+			return
+		}
+		requestedTokens = append(requestedTokens, r.Header.Get("PRIVATE-TOKEN"))
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"id": 42}`))
+	}))
+	defer server.Close()
+
+	connector := NewConnector()
+	first := &types.DataSourceConfig{
+		Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token-a"},
+	}
+	second := &types.DataSourceConfig{
+		Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token-b"},
+	}
+	if err := connector.Validate(context.Background(), first); err != nil {
+		t.Fatalf("first validate: %v", err)
+	}
+	if err := connector.Validate(context.Background(), second); err != nil {
+		t.Fatalf("second validate: %v", err)
+	}
+	if len(requestedTokens) != 2 || requestedTokens[0] != "token-a" || requestedTokens[1] != "token-b" {
+		t.Fatalf("requested tokens = %#v", requestedTokens)
+	}
+}
+
+func TestNewClientNormalizesAPIBaseURL(t *testing.T) {
+	allowLocalGitLabServer(t)
+
+	server := httptest.NewServer(http.NotFoundHandler())
+	defer server.Close()
+
+	c, err := newClient(server.URL+"/", "token")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got, want := c.baseURL, server.URL+"/api/v4"; got != want {
+		t.Fatalf("baseURL = %q, want %q", got, want)
+	}
+}
+
+func TestFetchIncrementalSyncsMultipleProjects(t *testing.T) {
+	allowLocalGitLabServer(t)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		switch r.URL.Path {
+		case "/api/v4/projects/1":
+			_, _ = w.Write([]byte(`{"id":1,"name":"project-one","path_with_namespace":"group/project-one","web_url":"https://gitlab.test/group/project-one","default_branch":"master"}`))
+		case "/api/v4/projects/2":
+			_, _ = w.Write([]byte(`{"id":2,"name":"project-two","path_with_namespace":"group/project-two","web_url":"https://gitlab.test/group/project-two","default_branch":"master"}`))
+		case "/api/v4/projects/1/repository/commits/master":
+			_, _ = w.Write([]byte(`{"id":"commit-1"}`))
+		case "/api/v4/projects/2/repository/commits/master":
+			_, _ = w.Write([]byte(`{"id":"commit-2"}`))
+		case "/api/v4/projects/1/repository/tree":
+			_, _ = w.Write([]byte(`[{"name":"one.md","type":"blob","path":"one.md"}]`))
+		case "/api/v4/projects/2/repository/tree":
+			_, _ = w.Write([]byte(`[{"name":"two.md","type":"blob","path":"two.md"}]`))
+		default:
+			if strings.HasPrefix(r.URL.EscapedPath(), "/api/v4/projects/1/repository/files/one%2Emd/raw") {
+				_, _ = w.Write([]byte("one"))
+				return
+			}
+			if strings.HasPrefix(r.URL.EscapedPath(), "/api/v4/projects/2/repository/files/two%2Emd/raw") {
+				_, _ = w.Write([]byte("two"))
+				return
+			}
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+
+	connector := NewConnector()
+	config := &types.DataSourceConfig{
+		Credentials: map[string]interface{}{
+			"base_url":     server.URL,
+			"access_token": "token",
+		},
+		Settings: map[string]interface{}{
+			"projects": []interface{}{
+				map[string]interface{}{"project_id": "1", "ref": "master", "paths": []interface{}{}},
+				map[string]interface{}{"project_id": "2", "ref": "master", "paths": []interface{}{}},
+			},
+		}}
+
+	items, cursor, err := connector.FetchIncremental(context.Background(), config, nil)
+	if err != nil {
+		t.Fatalf("first sync: %v", err)
+	}
+	if len(items) != 2 {
+		t.Fatalf("first sync items = %d, want 2", len(items))
+	}
+	if cursor == nil || fmt.Sprint(cursor.ConnectorCursor["projects"]) == "" {
+		t.Fatal("first sync did not return per-project cursor state")
+	}
+
+	items, _, err = connector.FetchIncremental(context.Background(), config, cursor)
+	if err != nil {
+		t.Fatalf("second sync: %v", err)
+	}
+	if len(items) != 0 {
+		t.Fatalf("second sync items = %d, want 0", len(items))
+	}
+}
+
+type gitLabStreamRecorder struct {
+	items       []types.FetchedItem
+	checkpoints []*types.SyncCursor
+}
+
+func (h *gitLabStreamRecorder) Emit(_ context.Context, item types.FetchedItem) error {
+	h.items = append(h.items, item)
+	return nil
+}
+
+func (h *gitLabStreamRecorder) Checkpoint(_ context.Context, cursor *types.SyncCursor) error {
+	h.checkpoints = append(h.checkpoints, cursor)
+	return nil
+}
+
+func TestFetchStreamFiltersUnsupportedFilesAndCheckpointsProjects(t *testing.T) {
+	allowLocalGitLabServer(t)
+
+	var rawRequests []string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		switch r.URL.Path {
+		case "/api/v4/projects/1":
+			_, _ = w.Write([]byte(`{"id":1,"name":"docs","path_with_namespace":"group/docs","web_url":"https://gitlab.test/group/docs","default_branch":"main"}`))
+		case "/api/v4/projects/1/repository/commits/main":
+			_, _ = w.Write([]byte(`{"id":"commit-1"}`))
+		case "/api/v4/projects/1/repository/tree":
+			_, _ = w.Write([]byte(`[
+				{"name":"README.md","type":"blob","path":"README.md"},
+				{"name":"server.go","type":"blob","path":"server.go"},
+				{"name":"payload.exe","type":"blob","path":"payload.exe"}
+			]`))
+		default:
+			if strings.Contains(r.URL.EscapedPath(), "/repository/files/") {
+				rawRequests = append(rawRequests, r.URL.EscapedPath())
+				_, _ = w.Write([]byte("# readme"))
+				return
+			}
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+
+	connector := NewConnector()
+	config := &types.DataSourceConfig{
+		Credentials: map[string]interface{}{
+			"base_url":     server.URL,
+			"access_token": "token",
+		},
+		Settings: map[string]interface{}{
+			"projects": []interface{}{map[string]interface{}{"project_id": "1", "paths": []interface{}{}}},
+		},
+	}
+	handler := &gitLabStreamRecorder{}
+	next, err := connector.FetchStream(context.Background(), config, nil, handler)
+	if err != nil {
+		t.Fatalf("FetchStream() error = %v", err)
+	}
+	if len(handler.items) != 1 || handler.items[0].FileName != "docs-main/README.md" {
+		t.Fatalf("emitted items = %#v", handler.items)
+	}
+	if len(rawRequests) != 1 || !strings.Contains(rawRequests[0], "README%2Emd") {
+		t.Fatalf("raw requests = %v, want only README.md", rawRequests)
+	}
+	if len(handler.checkpoints) != 1 || next == nil {
+		t.Fatalf("checkpoints = %d, next = %#v", len(handler.checkpoints), next)
+	}
+	projects, _ := next.ConnectorCursor["projects"].(map[string]string)
+	if projects["1"] != "commit-1" {
+		t.Fatalf("cursor projects = %#v", next.ConnectorCursor["projects"])
+	}
+}
+
+func TestIsSupportedFile(t *testing.T) {
+	for _, tc := range []struct {
+		file string
+		want bool
+	}{
+		{file: "docs/guide.MD", want: true},
+		{file: "docs/guide.mdx", want: true},
+		{file: "report.pdf", want: true},
+		{file: "src/main.go", want: false},
+		{file: "archive.tar.gz", want: false},
+		{file: "LICENSE", want: false},
+	} {
+		if got := isSupportedFile(tc.file); got != tc.want {
+			t.Errorf("isSupportedFile(%q) = %v, want %v", tc.file, got, tc.want)
+		}
+	}
+}
+
+func TestTreeFollowsGitLabPagination(t *testing.T) {
+	allowLocalGitLabServer(t)
+	var requestedPages []string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/v4/projects/1/repository/tree" {
+			http.NotFound(w, r)
+			return
+		}
+		requestedPages = append(requestedPages, r.URL.Query().Get("page"))
+		w.Header().Set("Content-Type", "application/json")
+		switch r.URL.Query().Get("page") {
+		case "1":
+			w.Header().Set("X-Next-Page", "2")
+			_, _ = w.Write([]byte(`[{"name":"one.md","type":"blob","path":"one.md"}]`))
+		case "2":
+			_, _ = w.Write([]byte(`[{"name":"two.md","type":"blob","path":"two.md"}]`))
+		default:
+			http.Error(w, "unexpected page", http.StatusBadRequest)
+		}
+	}))
+	defer server.Close()
+
+	c, err := newClient(server.URL, "token")
+	if err != nil {
+		t.Fatal(err)
+	}
+	entries, err := c.tree(context.Background(), "1", "main", "")
+	if err != nil {
+		t.Fatalf("tree() error = %v", err)
+	}
+	if len(entries) != 2 || entries[1].Path != "two.md" {
+		t.Fatalf("entries = %#v", entries)
+	}
+	if got, want := strings.Join(requestedPages, ","), "1,2"; got != want {
+		t.Fatalf("requested pages = %q, want %q", got, want)
+	}
+}
+
+func TestProjectPathEncodesNamespaceWithoutDoubleEscaping(t *testing.T) {
+	for _, tc := range []struct {
+		in, want string
+	}{
+		{in: "12345", want: "12345"},
+		{in: "group/project", want: "group%2Fproject"},
+		{in: "group%2Fproject", want: "group%2Fproject"},
+		{in: "my group/my project", want: "my%20group%2Fmy%20project"},
+	} {
+		if got := projectPath(tc.in); got != tc.want {
+			t.Errorf("projectPath(%q) = %q, want %q", tc.in, got, tc.want)
+		}
+	}
+}
+
+func TestRawFallsBackToBase64FileDetail(t *testing.T) {
+	t.Setenv("SSRF_WHITELIST", "127.0.0.1,::1,localhost")
+	secutils.ResetSSRFWhitelistForTest()
+	t.Cleanup(secutils.ResetSSRFWhitelistForTest)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.EscapedPath() {
+		case "/api/v4/projects/18724/repository/files/docs%2Finternal%2Freadme%2Emd/raw":
+			http.NotFound(w, r)
+		case "/api/v4/projects/18724/repository/files/docs%2Finternal%2Freadme%2Emd":
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = w.Write([]byte(`{"encoding":"base64","content":"SGVsbG8sIEdpdExhYiE="}`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+
+	c, err := newClient(server.URL, "token")
+	if err != nil {
+		t.Fatal(err)
+	}
+	content, err := c.raw(context.Background(), "18724", "master", "docs/internal/readme.md")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(content) != "Hello, GitLab!" {
+		t.Fatalf("content = %q", content)
+	}
+}
+
+func TestGitlabFilePathEscape(t *testing.T) {
+	got := gitlabFilePathEscape("docs/internal/中文-file.md")
+	want := "docs%2Finternal%2F%E4%B8%AD%E6%96%87-file%2Emd"
+	if got != want {
+		t.Fatalf("gitlabFilePathEscape() = %q, want %q", got, want)
+	}
+}

internal/datasource/connector/gitlab/connector.go

@@ -0,0 +1,436 @@
+package gitlab
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/datasource"
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+var _ datasource.StreamingConnector = (*Connector)(nil)
+
+type Connector struct {
+	client        *client
+	canonicalBase string
+}
+
+// NewConnector creates a stateless connector. Each data source provides its
+// own GitLab URL and access token in its encrypted credentials.
+func NewConnector() *Connector    { return &Connector{} }
+func (c *Connector) Type() string { return types.ConnectorTypeGitLab }
+
+func (c *Connector) configured(ds *types.DataSourceConfig) (*Connector, error) {
+	if ds == nil {
+		return nil, datasource.ErrInvalidConfig
+	}
+	baseURL, _ := ds.Credentials["base_url"].(string)
+	token, _ := ds.Credentials["access_token"].(string)
+	client, err := newClient(baseURL, token)
+	if err != nil {
+		return nil, err
+	}
+	return &Connector{client: client, canonicalBase: client.baseURL}, nil
+}
+
+func (c *Connector) Validate(ctx context.Context, ds *types.DataSourceConfig) error {
+	configured, err := c.configured(ds)
+	if err != nil {
+		return err
+	}
+	if err := configured.client.ping(ctx); err != nil {
+		return err
+	}
+	if ds != nil {
+		if _, ok := ds.Settings["projects"]; ok {
+			if _, err := parseConfig(ds); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+func (c *Connector) ListResources(ctx context.Context, ds *types.DataSourceConfig, parent string) ([]types.Resource, error) {
+	var err error
+	if c, err = c.configured(ds); err != nil {
+		return nil, err
+	}
+	if _, err := parseConfig(ds); err != nil {
+		return nil, err
+	}
+	if parent == "" {
+		ps, err := c.client.projects(ctx)
+		if err != nil {
+			return nil, err
+		}
+		out := make([]types.Resource, 0, len(ps))
+		for _, p := range ps {
+			out = append(out, types.Resource{ExternalID: fmt.Sprint(p.ID), Name: p.PathWithNamespace, Type: "project", URL: p.WebURL, HasChildren: true})
+		}
+		return out, nil
+	}
+	id, dir := splitResourceID(parent)
+	p, err := c.client.project(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+	ref := p.DefaultBranch
+	entries, err := c.client.tree(ctx, id, ref, dir)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]types.Resource, 0, len(entries))
+	for _, e := range entries {
+		if e.Type == "tree" {
+			out = append(out, types.Resource{ExternalID: id + ":" + e.Path, Name: e.Name, Type: "directory", ParentID: parent, HasChildren: true})
+		}
+	}
+	return out, nil
+}
+func (c *Connector) ResolveResourceAncestors(context.Context, *types.DataSourceConfig, []string) ([]string, error) {
+	return []string{}, nil
+}
+func (c *Connector) FetchAll(ctx context.Context, ds *types.DataSourceConfig, _ []string) ([]types.FetchedItem, error) {
+	var err error
+	if c, err = c.configured(ds); err != nil {
+		return nil, err
+	}
+	cfg, err := parseConfig(ds)
+	if err != nil {
+		return nil, err
+	}
+	var out []types.FetchedItem
+	for _, s := range cfg.Projects {
+		p, err := c.client.project(ctx, s.ProjectID)
+		if err != nil {
+			return nil, err
+		}
+		ref := s.Ref
+		if ref == "" {
+			ref = p.DefaultBranch
+		}
+		files, err := c.files(ctx, s.ProjectID, ref, s.Paths)
+		if err != nil {
+			return nil, err
+		}
+		for _, file := range files {
+			item, err := c.item(ctx, p, ref, file)
+			if err != nil {
+				return nil, err
+			}
+			out = append(out, item)
+		}
+	}
+	return out, nil
+}
+
+type cursor struct {
+	Projects map[string]string `json:"projects"`
+}
+
+func (c *Connector) FetchIncremental(ctx context.Context, ds *types.DataSourceConfig, old *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error) {
+	var err error
+	if c, err = c.configured(ds); err != nil {
+		return nil, nil, err
+	}
+	cfg, err := parseConfig(ds)
+	if err != nil {
+		return nil, nil, err
+	}
+	prev := cursor{Projects: map[string]string{}}
+	if old != nil {
+		b, _ := json.Marshal(old.ConnectorCursor)
+		_ = json.Unmarshal(b, &prev)
+		if prev.Projects == nil {
+			prev.Projects = map[string]string{}
+		}
+	}
+	var out []types.FetchedItem
+	next := cursor{Projects: map[string]string{}}
+	for _, s := range cfg.Projects {
+		p, err := c.client.project(ctx, s.ProjectID)
+		if err != nil {
+			return nil, nil, err
+		}
+		ref := s.Ref
+		if ref == "" {
+			ref = p.DefaultBranch
+		}
+		head, err := c.client.commitSHA(ctx, s.ProjectID, ref)
+		if err != nil {
+			return nil, nil, err
+		}
+		previous := prev.Projects[s.ProjectID]
+		if previous == "" {
+			files, err := c.files(ctx, s.ProjectID, ref, s.Paths)
+			if err != nil {
+				return nil, nil, err
+			}
+			for _, f := range files {
+				item, err := c.item(ctx, p, ref, f)
+				if err != nil {
+					return nil, nil, err
+				}
+				out = append(out, item)
+			}
+		} else if previous != head {
+			diff, err := c.client.compare(ctx, s.ProjectID, previous, head)
+			if err != nil || diff.CompareTimeout {
+				// A compare may be unavailable after history rewrites or truncated by
+				// the server. Re-enumerate the configured scope to preserve updates.
+				files, listErr := c.files(ctx, s.ProjectID, ref, s.Paths)
+				if listErr != nil {
+					return nil, nil, fmt.Errorf("gitlab list files %s: %w", s.ProjectID, listErr)
+				}
+				for _, f := range files {
+					item, itemErr := c.item(ctx, p, ref, f)
+					if itemErr != nil {
+						return nil, nil, itemErr
+					}
+					out = append(out, item)
+				}
+			} else {
+				for _, d := range diff.Diffs {
+					if d.DeletedFile {
+						if c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) {
+							out = append(out, c.deleted(p, ref, d.OldPath))
+						}
+						continue
+					}
+					if d.RenamedFile && c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) {
+						out = append(out, c.deleted(p, ref, d.OldPath))
+					}
+					if c.inScope(d.NewPath, s.Paths) && isSupportedFile(d.NewPath) {
+						item, err := c.item(ctx, p, ref, d.NewPath)
+						if err != nil {
+							return nil, nil, err
+						}
+						out = append(out, item)
+					}
+				}
+			}
+		}
+		next.Projects[s.ProjectID] = head
+	}
+	raw, _ := json.Marshal(next)
+	return out, &types.SyncCursor{LastSyncTime: time.Now().UTC(), ConnectorCursor: map[string]interface{}{"projects": next.Projects, "raw": string(raw)}}, nil
+}
+
+// FetchStream is the production sync path. It emits each supported repository
+// file as soon as it is read, so large GitLab projects do not accumulate their
+// contents in memory. The service prefers this method for StreamingConnector
+// implementations; FetchAll and FetchIncremental remain compatibility methods
+// required by the base Connector interface.
+func (c *Connector) FetchStream(
+	ctx context.Context, ds *types.DataSourceConfig, old *types.SyncCursor, h datasource.StreamHandler,
+) (*types.SyncCursor, error) {
+	var err error
+	if c, err = c.configured(ds); err != nil {
+		return nil, err
+	}
+	cfg, err := parseConfig(ds)
+	if err != nil {
+		return nil, err
+	}
+	prev := cursor{Projects: map[string]string{}}
+	if old != nil {
+		b, _ := json.Marshal(old.ConnectorCursor)
+		_ = json.Unmarshal(b, &prev)
+		if prev.Projects == nil {
+			prev.Projects = map[string]string{}
+		}
+	}
+	next := cursor{Projects: make(map[string]string, len(cfg.Projects))}
+	for projectID, commit := range prev.Projects {
+		next.Projects[projectID] = commit
+	}
+
+	for _, selection := range cfg.Projects {
+		project, err := c.client.project(ctx, selection.ProjectID)
+		if err != nil {
+			return nil, err
+		}
+		ref := selection.Ref
+		if ref == "" {
+			ref = project.DefaultBranch
+		}
+		head, err := c.client.commitSHA(ctx, selection.ProjectID, ref)
+		if err != nil {
+			return nil, err
+		}
+
+		previous := prev.Projects[selection.ProjectID]
+		switch {
+		case previous == "":
+			err = c.streamFiles(ctx, project, ref, selection.Paths, h)
+		case previous != head:
+			err = c.streamChanges(ctx, project, ref, previous, head, selection.Paths, h)
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		next.Projects[selection.ProjectID] = head
+		checkpoint := gitLabCursor(next)
+		if err := h.Checkpoint(ctx, checkpoint); err != nil {
+			return nil, err
+		}
+	}
+	return gitLabCursor(next), nil
+}
+
+func gitLabCursor(value cursor) *types.SyncCursor {
+	raw, _ := json.Marshal(value)
+	return &types.SyncCursor{
+		LastSyncTime:    time.Now().UTC(),
+		ConnectorCursor: map[string]interface{}{"projects": value.Projects, "raw": string(raw)},
+	}
+}
+
+func (c *Connector) streamChanges(
+	ctx context.Context, project *project, ref, from, to string, roots []string, h datasource.StreamHandler,
+) error {
+	diff, err := c.client.compare(ctx, fmt.Sprint(project.ID), from, to)
+	if err != nil || diff.CompareTimeout {
+		// A compare can be unavailable after history rewrites or be truncated by
+		// GitLab. Re-enumerating the configured scope preserves file updates.
+		return c.streamFiles(ctx, project, ref, roots, h)
+	}
+	for _, change := range diff.Diffs {
+		if change.DeletedFile {
+			if c.inScope(change.OldPath, roots) && isSupportedFile(change.OldPath) {
+				if err := h.Emit(ctx, c.deleted(project, ref, change.OldPath)); err != nil {
+					return err
+				}
+			}
+			continue
+		}
+		if change.RenamedFile && c.inScope(change.OldPath, roots) && isSupportedFile(change.OldPath) {
+			if err := h.Emit(ctx, c.deleted(project, ref, change.OldPath)); err != nil {
+				return err
+			}
+		}
+		if c.inScope(change.NewPath, roots) && isSupportedFile(change.NewPath) {
+			item, err := c.item(ctx, project, ref, change.NewPath)
+			if err != nil {
+				return err
+			}
+			if err := h.Emit(ctx, item); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (c *Connector) streamFiles(
+	ctx context.Context, project *project, ref string, roots []string, h datasource.StreamHandler,
+) error {
+	return c.walkFiles(ctx, fmt.Sprint(project.ID), ref, roots, func(file string) error {
+		item, err := c.item(ctx, project, ref, file)
+		if err != nil {
+			return err
+		}
+		return h.Emit(ctx, item)
+	})
+}
+
+func (c *Connector) files(ctx context.Context, id, ref string, roots []string) ([]string, error) {
+	var out []string
+	err := c.walkFiles(ctx, id, ref, roots, func(file string) error {
+		out = append(out, file)
+		return nil
+	})
+	return out, err
+}
+
+// walkFiles visits supported blobs while traversing the selected directories.
+// It deliberately does not collect paths, keeping streaming sync memory bounded
+// by the traversal depth plus the current file body.
+func (c *Connector) walkFiles(ctx context.Context, id, ref string, roots []string, visit func(string) error) error {
+	if len(roots) == 0 {
+		roots = []string{""}
+	}
+	var walk func(string) error
+	walk = func(dir string) error {
+		entries, err := c.client.tree(ctx, id, ref, dir)
+		if err != nil {
+			return err
+		}
+		for _, e := range entries {
+			if e.Type == "tree" {
+				if err := walk(e.Path); err != nil {
+					return err
+				}
+			} else if e.Type == "blob" && isSupportedFile(e.Path) {
+				if err := visit(e.Path); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	}
+	for _, r := range roots {
+		if err := walk(r); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func isSupportedFile(file string) bool {
+	_, ok := gitLabSupportedFileExtensions[strings.ToLower(path.Ext(file))]
+	return ok
+}
+
+// gitLabSupportedFileExtensions limits repository sync to formats the current
+// knowledge import pipeline can process. This is intentionally connector-local:
+// GitLab exposes arbitrary repository blobs, unlike document-centric sources.
+var gitLabSupportedFileExtensions = map[string]struct{}{
+	".pdf": {}, ".txt": {}, ".docx": {}, ".doc": {}, ".epub": {},
+	".html": {}, ".htm": {}, ".mhtml": {}, ".md": {}, ".markdown": {}, ".mdx": {},
+	".png": {}, ".jpg": {}, ".jpeg": {}, ".gif": {},
+	".csv": {}, ".xlsx": {}, ".xls": {}, ".pptx": {}, ".ppt": {}, ".json": {},
+	".mp3": {}, ".wav": {}, ".m4a": {}, ".flac": {}, ".ogg": {},
+}
+
+func (c *Connector) item(ctx context.Context, p *project, ref, file string) (types.FetchedItem, error) {
+	body, err := c.client.raw(ctx, fmt.Sprint(p.ID), ref, file)
+	if err != nil {
+		return types.FetchedItem{}, err
+	}
+	id := fmt.Sprintf("gitlab:%s:%d:%s:%s", c.canonicalBase, p.ID, ref, file)
+	return types.FetchedItem{ExternalID: id, Title: p.PathWithNamespace + "/" + file, FileName: knowledgeRelativePath(p.Name, ref, file), Content: body, ContentType: "text/plain", UpdatedAt: time.Now().UTC(), SourceResourceID: fmt.Sprint(p.ID), Metadata: map[string]string{"channel": types.ConnectorTypeGitLab, "source_type": "gitlab", "gitlab_project_id": fmt.Sprint(p.ID), "gitlab_ref": ref, "gitlab_path": file, "gitlab_url": p.WebURL + "/-/blob/" + ref + "/" + file}}, nil
+}
+
+// knowledgeRelativePath maps a repository file to the KB folder convention:
+// <GitLab project name>-<branch>/<repository-relative file path>.
+func knowledgeRelativePath(projectName, ref, file string) string {
+	root := strings.TrimSpace(projectName) + "-" + strings.ReplaceAll(strings.TrimSpace(ref), "/", "-")
+	return path.Join(root, file)
+}
+func (c *Connector) deleted(p *project, ref, file string) types.FetchedItem {
+	return types.FetchedItem{ExternalID: fmt.Sprintf("gitlab:%s:%d:%s:%s", c.canonicalBase, p.ID, ref, file), IsDeleted: true, Metadata: map[string]string{"channel": types.ConnectorTypeGitLab, "gitlab_path": file}}
+}
+func (c *Connector) inScope(file string, roots []string) bool {
+	if len(roots) == 0 {
+		return true
+	}
+	for _, r := range roots {
+		if file == r || strings.HasPrefix(file, r+"/") {
+			return true
+		}
+	}
+	return false
+}
+func splitResourceID(value string) (string, string) {
+	parts := strings.SplitN(value, ":", 2)
+	if len(parts) == 1 {
+		return value, ""
+	}
+	return parts[0], parts[1]
+}

internal/datasource/connector/gitlab/types.go

@@ -0,0 +1,108 @@
+package gitlab
+
+import (
+	"fmt"
+	"path"
+	"sort"
+	"strings"
+
+	"github.com/Tencent/WeKnora/internal/datasource"
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+type projectSelection struct {
+	ProjectID string   `json:"project_id"`
+	Ref       string   `json:"ref"`
+	Paths     []string `json:"paths"`
+}
+
+type config struct {
+	Projects []projectSelection `json:"projects"`
+}
+
+func parseConfig(ds *types.DataSourceConfig) (*config, error) {
+	if ds == nil {
+		return nil, datasource.ErrInvalidConfig
+	}
+	raw, ok := ds.Settings["projects"]
+	if !ok {
+		return nil, fmt.Errorf("%w: settings.projects is required", datasource.ErrInvalidConfig)
+	}
+	items, ok := raw.([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("%w: settings.projects must be an array", datasource.ErrInvalidConfig)
+	}
+	out := &config{Projects: make([]projectSelection, 0, len(items))}
+	seen := map[string]bool{}
+	for _, rawProject := range items {
+		m, ok := rawProject.(map[string]interface{})
+		if !ok {
+			return nil, fmt.Errorf("%w: invalid project selection", datasource.ErrInvalidConfig)
+		}
+		id, _ := m["project_id"].(string)
+		id = strings.TrimSpace(id)
+		if id == "" || seen[id] {
+			return nil, fmt.Errorf("%w: project_id must be unique and non-empty", datasource.ErrInvalidConfig)
+		}
+		seen[id] = true
+		ref, _ := m["ref"].(string)
+		p := projectSelection{ProjectID: id, Ref: strings.TrimSpace(ref)}
+		if rawPaths, exists := m["paths"]; exists {
+			values, ok := rawPaths.([]interface{})
+			if !ok {
+				return nil, fmt.Errorf("%w: paths must be an array", datasource.ErrInvalidConfig)
+			}
+			for _, value := range values {
+				s, ok := value.(string)
+				if !ok {
+					return nil, fmt.Errorf("%w: path must be a string", datasource.ErrInvalidConfig)
+				}
+				normalized, err := normalizePath(s)
+				if err != nil {
+					return nil, err
+				}
+				p.Paths = append(p.Paths, normalized)
+			}
+		}
+		p.Paths = collapsePaths(p.Paths)
+		out.Projects = append(out.Projects, p)
+	}
+	if len(out.Projects) == 0 {
+		return nil, fmt.Errorf("%w: at least one project is required", datasource.ErrInvalidConfig)
+	}
+	return out, nil
+}
+
+func normalizePath(value string) (string, error) {
+	v := strings.Trim(strings.TrimSpace(value), "/")
+	if v == "" {
+		return "", nil
+	}
+	if strings.Contains(v, "\\") {
+		return "", fmt.Errorf("%w: path must use forward slashes", datasource.ErrInvalidConfig)
+	}
+	if path.Clean(v) != v || v == "." || strings.HasPrefix(v, "../") || strings.Contains(v, "/../") {
+		return "", fmt.Errorf("%w: invalid repository path", datasource.ErrInvalidConfig)
+	}
+	return v, nil
+}
+
+func collapsePaths(paths []string) []string {
+	if len(paths) == 0 {
+		return nil
+	}
+	sort.Strings(paths)
+	for _, p := range paths {
+		if p == "" {
+			return nil
+		}
+	}
+	out := make([]string, 0, len(paths))
+	for _, p := range paths {
+		if len(out) > 0 && (p == out[len(out)-1] || strings.HasPrefix(p, out[len(out)-1]+"/")) {
+			continue
+		}
+		out = append(out, p)
+	}
+	return out
+}

internal/datasource/connector/gitlab/types_test.go

@@ -0,0 +1,47 @@
+package gitlab
+
+import (
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestParseConfigCollapsesDirectories(t *testing.T) {
+	cfg, err := parseConfig(&types.DataSourceConfig{Settings: map[string]interface{}{
+		"projects": []interface{}{map[string]interface{}{
+			"project_id": "team/docs", "ref": "main",
+			"paths": []interface{}{"docs/guide", "docs", "docs"},
+		}},
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(cfg.Projects) != 1 || len(cfg.Projects[0].Paths) != 1 || cfg.Projects[0].Paths[0] != "docs" {
+		t.Fatalf("unexpected config: %#v", cfg)
+	}
+}
+
+func TestParseConfigRootMeansWholeProject(t *testing.T) {
+	cfg, err := parseConfig(&types.DataSourceConfig{Settings: map[string]interface{}{
+		"projects": []interface{}{map[string]interface{}{"project_id": "42", "paths": []interface{}{"/"}}},
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(cfg.Projects[0].Paths) != 0 {
+		t.Fatalf("root path must collapse to whole project: %#v", cfg.Projects[0].Paths)
+	}
+}
+
+func TestNormalizePathRejectsTraversal(t *testing.T) {
+	if _, err := normalizePath("docs/../secrets"); err == nil {
+		t.Fatal("expected traversal rejection")
+	}
+}
+
+func TestKnowledgeRelativePathPreservesRepositoryTreeBelowProjectAndBranch(t *testing.T) {
+	got := knowledgeRelativePath("knowledge", "feature/login", "docs/guide/install.md")
+	if got != "knowledge-feature-login/docs/guide/install.md" {
+		t.Fatalf("knowledge relative path = %q", got)
+	}
+}