WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/agent/tools/web_fetch_test.go

@@ -0,0 +1,147 @@
+package tools
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"sync"
+	"testing"
+
+	webfetch "github.com/Tencent/WeKnora/internal/infrastructure/web_fetch"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type stubWebContentFetcher struct {
+	mu        sync.Mutex
+	contents  map[string]string
+	errors    map[string]error
+	callCount map[string]int
+}
+
+func (fetcher *stubWebContentFetcher) Fetch(_ context.Context, rawURL string) (string, error) {
+	fetcher.mu.Lock()
+	defer fetcher.mu.Unlock()
+	fetcher.callCount[rawURL]++
+	if err := fetcher.errors[rawURL]; err != nil {
+		return "", err
+	}
+	return fetcher.contents[rawURL], nil
+}
+
+func TestWebFetchToolSingleURLSuccessSurvivesSummaryFailure(t *testing.T) {
+	const rawURL = "https://example.com/specs"
+	fetcher := newStubWebContentFetcher(map[string]string{rawURL: "official specifications"}, nil)
+	tool := newWebFetchTool(nil, fetcher)
+
+	result, err := tool.Execute(context.Background(), webFetchArgs(
+		WebFetchItem{URL: rawURL, Prompt: "extract specifications"},
+	))
+
+	require.NoError(t, err)
+	require.True(t, result.Success)
+	assert.Equal(t, 1, result.Data["successful_count"])
+	items := result.Data["results"].([]map[string]interface{})
+	assert.Equal(t, "success", items[0]["status"])
+	assert.Equal(t, "failed", items[0]["summary_status"])
+	assert.Equal(t, "official specifications", items[0]["raw_content"])
+}
+
+func TestWebFetchToolPreservesPartialSuccess(t *testing.T) {
+	const successURL = "https://example.com/success"
+	const failedURL = "https://example.com/forbidden"
+	fetcher := newStubWebContentFetcher(
+		map[string]string{successURL: "verified page content"},
+		map[string]error{failedURL: fetchFailure(webfetch.ErrorHTTP403, false, "access denied")},
+	)
+	tool := newWebFetchTool(nil, fetcher)
+
+	result, err := tool.Execute(context.Background(), webFetchArgs(
+		WebFetchItem{URL: successURL, Prompt: "extract facts"},
+		WebFetchItem{URL: failedURL, Prompt: "extract facts"},
+	))
+
+	require.NoError(t, err)
+	require.True(t, result.Success)
+	assert.Equal(t, 1, result.Data["successful_count"])
+	assert.Equal(t, 1, result.Data["failed_count"])
+	assert.Equal(t, false, result.Data["all_failed"])
+	items := result.Data["results"].([]map[string]interface{})
+	assert.Equal(t, "success", items[0]["status"])
+	assert.Equal(t, "failed", items[1]["status"])
+	assert.Equal(t, "http_403", items[1]["error_code"])
+	assert.Equal(t, false, items[1]["retryable"])
+}
+
+func TestWebFetchToolAllFailuresReturnStructuredFallback(t *testing.T) {
+	const firstURL = "https://example.com/dns"
+	const secondURL = "https://example.com/rate-limit"
+	fetcher := newStubWebContentFetcher(nil, map[string]error{
+		firstURL:  fetchFailure(webfetch.ErrorDNS, true, "DNS lookup failed"),
+		secondURL: fetchFailure(webfetch.ErrorHTTP429, true, "rate limited"),
+	})
+	tool := newWebFetchTool(nil, fetcher)
+
+	result, err := tool.Execute(context.Background(), webFetchArgs(
+		WebFetchItem{URL: firstURL, Prompt: "extract facts"},
+		WebFetchItem{URL: secondURL, Prompt: "extract facts"},
+	))
+
+	require.NoError(t, err)
+	require.False(t, result.Success, "all-failed batches should not report tool success")
+	assert.Equal(t, true, result.Data["all_failed"])
+	assert.Equal(t, 0, result.Data["successful_count"])
+	assert.Contains(t, result.Output, "answer from existing web_search titles, URLs, and snippets")
+}
+
+func TestWebFetchToolDeduplicatesURLsWithinBatch(t *testing.T) {
+	const rawURL = "https://example.com/page#section"
+	const duplicateURL = "https://example.com/page"
+	fetcher := newStubWebContentFetcher(map[string]string{rawURL: "page content"}, nil)
+	tool := newWebFetchTool(nil, fetcher)
+
+	result, err := tool.Execute(context.Background(), webFetchArgs(
+		WebFetchItem{URL: rawURL, Prompt: "extract facts"},
+		WebFetchItem{URL: duplicateURL, Prompt: "extract facts"},
+	))
+
+	require.NoError(t, err)
+	assert.Equal(t, 1, fetcher.callCount[rawURL])
+	assert.Equal(t, 0, fetcher.callCount[duplicateURL])
+	assert.Equal(t, 1, result.Data["skipped_count"])
+	items := result.Data["results"].([]map[string]interface{})
+	assert.Equal(t, "duplicate_url", items[1]["error_code"])
+}
+
+func TestWebFetchToolDeduplicatesGitHubBlobAndRawURLs(t *testing.T) {
+	const blobURL = "https://github.com/org/repo/blob/main/README.md"
+	const rawURL = "https://raw.githubusercontent.com/org/repo/main/README.md"
+	fetcher := newStubWebContentFetcher(map[string]string{rawURL: "readme content"}, nil)
+	tool := newWebFetchTool(nil, fetcher)
+
+	result, err := tool.Execute(context.Background(), webFetchArgs(
+		WebFetchItem{URL: blobURL, Prompt: "extract facts"},
+		WebFetchItem{URL: rawURL, Prompt: "extract facts"},
+	))
+
+	require.NoError(t, err)
+	assert.Equal(t, 1, fetcher.callCount[rawURL]+fetcher.callCount[blobURL])
+	assert.Equal(t, 1, result.Data["skipped_count"])
+}
+
+func newStubWebContentFetcher(contents map[string]string, failures map[string]error) *stubWebContentFetcher {
+	return &stubWebContentFetcher{
+		contents:  contents,
+		errors:    failures,
+		callCount: make(map[string]int),
+	}
+}
+
+func fetchFailure(code webfetch.ErrorCode, retryable bool, message string) error {
+	return &webfetch.FetchError{Code: code, Retryable: retryable, Err: errors.New(message)}
+}
+
+func webFetchArgs(items ...WebFetchItem) json.RawMessage {
+	encoded, _ := json.Marshal(WebFetchInput{Items: items})
+	return encoded
+}