94 lines
3.4 KiB
Go
94 lines
3.4 KiB
Go
|
package service
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"encoding/json"
|
||
|
|
"os"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"github.com/Tencent/WeKnora/internal/types"
|
||
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||
|
|
"github.com/stretchr/testify/require"
|
||
|
|
)
|
||
|
|
|
||
|
|
// checkpointsWikiPages serves the real pages of one knowledge base from a fixture.
|
||
|
|
type checkpointsWikiPages struct {
|
||
|
|
interfaces.WikiPageService
|
||
|
|
bySlug map[string]*types.WikiPageLite
|
||
|
|
}
|
||
|
|
|
||
|
|
func (f checkpointsWikiPages) ListBySlugs(ctx context.Context, kbID string, slugs []string) (map[string]*types.WikiPageLite, error) {
|
||
|
|
out := make(map[string]*types.WikiPageLite, len(slugs))
|
||
|
|
for _, slug := range slugs {
|
||
|
|
if page := f.bySlug[slug]; page != nil {
|
||
|
|
out[slug] = page
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return out, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (f checkpointsWikiPages) FindPagesByNormalizedTitles(ctx context.Context, kbID, pageType string, identities []string) ([]*types.WikiPageLite, error) {
|
||
|
|
wanted := make(map[string]bool, len(identities))
|
||
|
|
for _, identity := range identities {
|
||
|
|
wanted[identity] = true
|
||
|
|
}
|
||
|
|
var out []*types.WikiPageLite
|
||
|
|
for _, page := range f.bySlug {
|
||
|
|
if page.PageType == pageType && wanted[normalizeWikiIdentityTitle(page.Title)] {
|
||
|
|
out = append(out, page)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return out, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// Replays the fifteen real candidate payloads of one document through the complete
|
||
|
|
// reading-tail path (normalize, resolve, merge) against the pages that knowledge base
|
||
|
|
// really owns, so a conflict that aborted the document in production is reproduced
|
||
|
|
// and then fixed offline.
|
||
|
|
func TestWikiRealWindowsFullResolution(t *testing.T) {
|
||
|
|
windowsPath, pagesPath := os.Getenv("WIKI_REPLAY_FILE"), os.Getenv("WIKI_PAGES_FILE")
|
||
|
|
if windowsPath == "" || pagesPath == "" {
|
||
|
|
t.Skip("WIKI_REPLAY_FILE and WIKI_PAGES_FILE must be set")
|
||
|
|
}
|
||
|
|
rawWindows, err := os.ReadFile(windowsPath)
|
||
|
|
require.NoError(t, err)
|
||
|
|
var texts []string
|
||
|
|
require.NoError(t, json.Unmarshal(rawWindows, &texts))
|
||
|
|
|
||
|
|
rawPages, err := os.ReadFile(pagesPath)
|
||
|
|
require.NoError(t, err)
|
||
|
|
var pages []*types.WikiPageLite
|
||
|
|
require.NoError(t, json.Unmarshal(rawPages, &pages))
|
||
|
|
bySlug := make(map[string]*types.WikiPageLite, len(pages))
|
||
|
|
for _, page := range pages {
|
||
|
|
bySlug[page.Slug] = page
|
||
|
|
}
|
||
|
|
|
||
|
|
merged := combinedExtraction{}
|
||
|
|
for i, text := range texts {
|
||
|
|
var part combinedExtraction
|
||
|
|
require.NoError(t, json.Unmarshal([]byte(text), &part), "window %d", i)
|
||
|
|
merged.Entities = append(merged.Entities, part.Entities...)
|
||
|
|
merged.Concepts = append(merged.Concepts, part.Concepts...)
|
||
|
|
}
|
||
|
|
|
||
|
|
normalized, err := normalizeWikiFreshCandidates(merged)
|
||
|
|
require.NoError(t, err, "normalization rejected the real candidate set")
|
||
|
|
t.Logf("normalized: E=%d C=%d", len(normalized.Entities), len(normalized.Concepts))
|
||
|
|
|
||
|
|
service := &wikiIngestService{wikiService: checkpointsWikiPages{bySlug: bySlug}}
|
||
|
|
resolved, err := service.resolveWikiSourceIdentities(context.Background(), "kb-real", normalized, nil)
|
||
|
|
require.NoError(t, err, "identity resolution rejected the real candidate set")
|
||
|
|
t.Logf("resolved: E=%d C=%d pruning=%+v", len(resolved.Entities), len(resolved.Concepts), resolved.CandidatePruning)
|
||
|
|
require.Positive(t, len(resolved.Entities)+len(resolved.Concepts), "the document must still publish pages")
|
||
|
|
|
||
|
|
seen := map[string]string{}
|
||
|
|
for _, item := range append(append([]extractedItem{}, resolved.Entities...), resolved.Concepts...) {
|
||
|
|
if other, ok := seen[item.Slug]; ok && other != item.Name {
|
||
|
|
t.Fatalf("slug %q still shared by %q and %q", item.Slug, other, item.Name)
|
||
|
|
}
|
||
|
|
seen[item.Slug] = item.Name
|
||
|
|
}
|
||
|
|
t.Logf("final identity count: %d", len(seen))
|
||
|
|
}
|