package service import ( "crypto/sha256" "fmt" "sort" "strings" "unicode/utf8" "github.com/Tencent/WeKnora/internal/types" ) // Normalize only fresh proposals before citation routing. Existing references // and published pages must continue to use the strict identity validators. func normalizeWikiFreshCandidates(input combinedExtraction) (combinedExtraction, error) { if err := validateWikiCandidates(input); err != nil { return combinedExtraction{}, err } items := make(map[string]extractedItem) kinds := make(map[string]string) reserved := make(map[string]bool) preferText := func(left, right string) string { if utf8.RuneCountInString(right) > utf8.RuneCountInString(left) || (utf8.RuneCountInString(right) == utf8.RuneCountInString(left) && right < left) { return right } return left } for _, group := range []struct { kind string items []extractedItem }{{types.WikiPageTypeEntity, input.Entities}, {types.WikiPageTypeConcept, input.Concepts}} { for i, original := range group.items { if len(original.Aliases) != 0 { return combinedExtraction{}, wikiIdentityFailure("wiki_identity_alias_unsupported", group.kind, i, nil) } item := original item.Name = strings.TrimSpace(item.Name) item.Aliases = []string{} item.SourceChunks = append([]string(nil), item.SourceChunks...) identity := wikiSourceIdentityKey(group.kind, item.Name) reserved[item.Slug] = true if previous, exists := items[identity]; exists { if previous.Slug < item.Slug { item.Slug = previous.Slug } item.Description = preferText(previous.Description, item.Description) item.Details = preferText(previous.Details, item.Details) item.SourceChunks = append(item.SourceChunks, previous.SourceChunks...) } sort.Strings(item.SourceChunks) unique := item.SourceChunks[:0] for _, chunk := range item.SourceChunks { if len(unique) == 0 || unique[len(unique)-1] != chunk { unique = append(unique, chunk) } } item.SourceChunks = unique items[identity], kinds[identity] = item, group.kind } } identities := make([]string, 0, len(items)) slugCount := make(map[string]int) for identity, item := range items { identities = append(identities, identity) slugCount[item.Slug]++ } sort.Strings(identities) result := combinedExtraction{Entities: []extractedItem{}, Concepts: []extractedItem{}} for _, identity := range identities { item := items[identity] if slugCount[item.Slug] > 1 { base := item.Slug digest := fmt.Sprintf("%x", sha256.Sum256([]byte(identity))) for length := 12; ; length += 4 { candidate := "" if length <= len(digest) { candidate = base + "-" + digest[:length] } else { candidate = fmt.Sprintf("%s-%s-%d", base, digest, (length-len(digest))/4) } if !reserved[candidate] { item.Slug, reserved[candidate] = candidate, true break } } } if kinds[identity] == types.WikiPageTypeEntity { result.Entities = append(result.Entities, item) } else { result.Concepts = append(result.Concepts, item) } } return result, validateWikiIdentityCollisions(result) }