xinmi/internal/application/service/wiki_fresh_candidates.go

package service

import (
	"crypto/sha256"
	"fmt"
	"sort"
	"strings"
	"unicode/utf8"

	"github.com/Tencent/WeKnora/internal/types"
)

// Normalize only fresh proposals before citation routing. Existing references
// and published pages must continue to use the strict identity validators.
func normalizeWikiFreshCandidates(input combinedExtraction) (combinedExtraction, error) {
	if err := validateWikiCandidates(input); err != nil {
		return combinedExtraction{}, err
	}
	items := make(map[string]extractedItem)
	kinds := make(map[string]string)
	reserved := make(map[string]bool)
	preferText := func(left, right string) string {
		if utf8.RuneCountInString(right) > utf8.RuneCountInString(left) ||
			(utf8.RuneCountInString(right) == utf8.RuneCountInString(left) && right < left) {
			return right
		}
		return left
	}
	for _, group := range []struct {
		kind  string
		items []extractedItem
	}{{types.WikiPageTypeEntity, input.Entities}, {types.WikiPageTypeConcept, input.Concepts}} {
		for i, original := range group.items {
			if len(original.Aliases) != 0 {
				return combinedExtraction{}, wikiIdentityFailure("wiki_identity_alias_unsupported", group.kind, i, nil)
			}
			item := original
			item.Name = strings.TrimSpace(item.Name)
			item.Aliases = []string{}
			item.SourceChunks = append([]string(nil), item.SourceChunks...)
			identity := wikiSourceIdentityKey(group.kind, item.Name)
			reserved[item.Slug] = true
			if previous, exists := items[identity]; exists {
				if previous.Slug < item.Slug {
					item.Slug = previous.Slug
				}
				item.Description = preferText(previous.Description, item.Description)
				item.Details = preferText(previous.Details, item.Details)
				item.SourceChunks = append(item.SourceChunks, previous.SourceChunks...)
			}
			sort.Strings(item.SourceChunks)
			unique := item.SourceChunks[:0]
			for _, chunk := range item.SourceChunks {
				if len(unique) == 0 || unique[len(unique)-1] != chunk {
					unique = append(unique, chunk)
				}
			}
			item.SourceChunks = unique
			items[identity], kinds[identity] = item, group.kind
		}
	}
	identities := make([]string, 0, len(items))
	slugCount := make(map[string]int)
	for identity, item := range items {
		identities = append(identities, identity)
		slugCount[item.Slug]++
	}
	sort.Strings(identities)
	result := combinedExtraction{Entities: []extractedItem{}, Concepts: []extractedItem{}}
	for _, identity := range identities {
		item := items[identity]
		if slugCount[item.Slug] > 1 {
			base := item.Slug
			digest := fmt.Sprintf("%x", sha256.Sum256([]byte(identity)))
			for length := 12; ; length += 4 {
				candidate := ""
				if length <= len(digest) {
					candidate = base + "-" + digest[:length]
				} else {
					candidate = fmt.Sprintf("%s-%s-%d", base, digest, (length-len(digest))/4)
				}
				if !reserved[candidate] {
					item.Slug, reserved[candidate] = candidate, true
					break
				}
			}
		}
		if kinds[identity] == types.WikiPageTypeEntity {
			result.Entities = append(result.Entities, item)
		} else {
			result.Concepts = append(result.Concepts, item)
		}
	}
	return result, validateWikiIdentityCollisions(result)
}