96 lines
3.1 KiB
Go
96 lines
3.1 KiB
Go
|
package service
|
||
|
|
|
||
|
|
import (
|
||
|
|
"crypto/sha256"
|
||
|
|
"fmt"
|
||
|
|
"sort"
|
||
|
|
"strings"
|
||
|
|
"unicode/utf8"
|
||
|
|
|
||
|
|
"github.com/Tencent/WeKnora/internal/types"
|
||
|
|
)
|
||
|
|
|
||
|
|
// Normalize only fresh proposals before citation routing. Existing references
|
||
|
|
// and published pages must continue to use the strict identity validators.
|
||
|
|
func normalizeWikiFreshCandidates(input combinedExtraction) (combinedExtraction, error) {
|
||
|
|
if err := validateWikiCandidates(input); err != nil {
|
||
|
|
return combinedExtraction{}, err
|
||
|
|
}
|
||
|
|
items := make(map[string]extractedItem)
|
||
|
|
kinds := make(map[string]string)
|
||
|
|
reserved := make(map[string]bool)
|
||
|
|
preferText := func(left, right string) string {
|
||
|
|
if utf8.RuneCountInString(right) > utf8.RuneCountInString(left) ||
|
||
|
|
(utf8.RuneCountInString(right) == utf8.RuneCountInString(left) && right < left) {
|
||
|
|
return right
|
||
|
|
}
|
||
|
|
return left
|
||
|
|
}
|
||
|
|
for _, group := range []struct {
|
||
|
|
kind string
|
||
|
|
items []extractedItem
|
||
|
|
}{{types.WikiPageTypeEntity, input.Entities}, {types.WikiPageTypeConcept, input.Concepts}} {
|
||
|
|
for i, original := range group.items {
|
||
|
|
if len(original.Aliases) != 0 {
|
||
|
|
return combinedExtraction{}, wikiIdentityFailure("wiki_identity_alias_unsupported", group.kind, i, nil)
|
||
|
|
}
|
||
|
|
item := original
|
||
|
|
item.Name = strings.TrimSpace(item.Name)
|
||
|
|
item.Aliases = []string{}
|
||
|
|
item.SourceChunks = append([]string(nil), item.SourceChunks...)
|
||
|
|
identity := wikiSourceIdentityKey(group.kind, item.Name)
|
||
|
|
reserved[item.Slug] = true
|
||
|
|
if previous, exists := items[identity]; exists {
|
||
|
|
if previous.Slug < item.Slug {
|
||
|
|
item.Slug = previous.Slug
|
||
|
|
}
|
||
|
|
item.Description = preferText(previous.Description, item.Description)
|
||
|
|
item.Details = preferText(previous.Details, item.Details)
|
||
|
|
item.SourceChunks = append(item.SourceChunks, previous.SourceChunks...)
|
||
|
|
}
|
||
|
|
sort.Strings(item.SourceChunks)
|
||
|
|
unique := item.SourceChunks[:0]
|
||
|
|
for _, chunk := range item.SourceChunks {
|
||
|
|
if len(unique) == 0 || unique[len(unique)-1] != chunk {
|
||
|
|
unique = append(unique, chunk)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
item.SourceChunks = unique
|
||
|
|
items[identity], kinds[identity] = item, group.kind
|
||
|
|
}
|
||
|
|
}
|
||
|
|
identities := make([]string, 0, len(items))
|
||
|
|
slugCount := make(map[string]int)
|
||
|
|
for identity, item := range items {
|
||
|
|
identities = append(identities, identity)
|
||
|
|
slugCount[item.Slug]++
|
||
|
|
}
|
||
|
|
sort.Strings(identities)
|
||
|
|
result := combinedExtraction{Entities: []extractedItem{}, Concepts: []extractedItem{}}
|
||
|
|
for _, identity := range identities {
|
||
|
|
item := items[identity]
|
||
|
|
if slugCount[item.Slug] > 1 {
|
||
|
|
base := item.Slug
|
||
|
|
digest := fmt.Sprintf("%x", sha256.Sum256([]byte(identity)))
|
||
|
|
for length := 12; ; length += 4 {
|
||
|
|
candidate := ""
|
||
|
|
if length <= len(digest) {
|
||
|
|
candidate = base + "-" + digest[:length]
|
||
|
|
} else {
|
||
|
|
candidate = fmt.Sprintf("%s-%s-%d", base, digest, (length-len(digest))/4)
|
||
|
|
}
|
||
|
|
if !reserved[candidate] {
|
||
|
|
item.Slug, reserved[candidate] = candidate, true
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if kinds[identity] == types.WikiPageTypeEntity {
|
||
|
|
result.Entities = append(result.Entities, item)
|
||
|
|
} else {
|
||
|
|
result.Concepts = append(result.Concepts, item)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return result, validateWikiIdentityCollisions(result)
|
||
|
|
}
|