277 lines
8.0 KiB
Go
277 lines
8.0 KiB
Go
|
package chatpipeline
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"fmt"
|
|||
|
|
"regexp"
|
|||
|
|
"strings"
|
|||
|
|
|
|||
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
var graphPriceLabelPattern = regexp.MustCompile(`(?:终端)?零售(?:价格|价)|批发(?:价格|价)|促销(?:价格|价)|优惠(?:价格|价)|活动(?:价格|价)|会员(?:价格|价)|建议零售价|销售价格|售价|单价|原价|现价|价格`)
|
|||
|
|
var graphPriceAmountPattern = regexp.MustCompile(`^(人民币|[¥￥$])?([0-9]+(?:\.[0-9]+)?)(美元|人民币|元|角|分|盒|箱|瓶|套|支|粒|片|件|袋|kg|mg|ml|g)?(/(?:[0-9一二两俩三四五六七八九十百千]+)?(?:盒|箱|瓶|套|支|粒|片|件|袋|kg|mg|ml|g))?`)
|
|||
|
|
var graphPriceHeaderUnitPattern = regexp.MustCompile(`^[（(](元|美元|元/(?:盒|箱|瓶|套|支|件))[）)]$`)
|
|||
|
|
|
|||
|
|
type graphPriceValue struct {
|
|||
|
|
number string
|
|||
|
|
unit string
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
type graphPriceClaim struct {
|
|||
|
|
label string
|
|||
|
|
values []graphPriceValue
|
|||
|
|
prefix string
|
|||
|
|
hasQuantity bool
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func graphPriceValues(value, headerUnit string) []graphPriceValue {
|
|||
|
|
value = strings.TrimLeft(compactGraphText(value), "：:为是")
|
|||
|
|
var values []graphPriceValue
|
|||
|
|
for value != "" {
|
|||
|
|
match := graphPriceAmountPattern.FindStringSubmatch(value)
|
|||
|
|
if match == nil {
|
|||
|
|
// Do not accept a parsed prefix while silently ignoring a changed
|
|||
|
|
// package quantity or another amount later in the same price claim.
|
|||
|
|
if strings.HasPrefix(value, "/") || graphQuantityPattern.MatchString(value) {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
unit := match[3] + match[4]
|
|||
|
|
if match[1] != "" {
|
|||
|
|
currency := "元"
|
|||
|
|
if match[1] == "$" {
|
|||
|
|
currency = "美元"
|
|||
|
|
}
|
|||
|
|
if unit == "" || strings.HasPrefix(unit, "/") {
|
|||
|
|
unit = currency + unit
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if unit == "" {
|
|||
|
|
unit = headerUnit
|
|||
|
|
}
|
|||
|
|
values = append(values, graphPriceValue{number: match[2], unit: unit})
|
|||
|
|
value = strings.TrimLeft(value[len(match[0]):], "，,、")
|
|||
|
|
}
|
|||
|
|
return values
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func graphPriceClaims(value string) []graphPriceClaim {
|
|||
|
|
value = compactGraphText(value)
|
|||
|
|
labels := graphPriceLabelPattern.FindAllStringIndex(value, -1)
|
|||
|
|
var claims []graphPriceClaim
|
|||
|
|
for i, span := range labels {
|
|||
|
|
end := len(value)
|
|||
|
|
if i+1 < len(labels) {
|
|||
|
|
end = labels[i+1][0]
|
|||
|
|
}
|
|||
|
|
prefix := value[:span[0]]
|
|||
|
|
if index := strings.LastIndexAny(prefix, "，,；;。|：:"); index >= 0 {
|
|||
|
|
prefix = strings.TrimLeft(prefix[index:], "，,；;。|：:")
|
|||
|
|
}
|
|||
|
|
claims = append(claims, graphPriceClaim{
|
|||
|
|
label: graphPriceLabel(value[span[0]:span[1]]),
|
|||
|
|
values: graphPriceValues(value[span[1]:end], ""),
|
|||
|
|
prefix: prefix,
|
|||
|
|
hasQuantity: graphQuantityPattern.MatchString(value[span[1]:end]),
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
return claims
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func graphPriceLabel(value string) string {
|
|||
|
|
return strings.TrimPrefix(strings.ReplaceAll(value, "价格", "价"), "终端")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func graphPriceValuesMatch(want, available []graphPriceValue) bool {
|
|||
|
|
if len(want) == 0 || len(want) > len(available) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
for start := 0; start+len(want) <= len(available); start++ {
|
|||
|
|
matches := true
|
|||
|
|
for i, value := range want {
|
|||
|
|
actual := available[start+i]
|
|||
|
|
if value.number != actual.number || (value.unit != "" && value.unit != actual.unit) {
|
|||
|
|
matches = false
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if matches {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func graphPricePrefixMatches(prefix, name string, inherited bool) bool {
|
|||
|
|
prefix = strings.TrimSuffix(compactGraphText(prefix), "的")
|
|||
|
|
if prefix == compactGraphText(name) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
return inherited && (prefix == "" || prefix == "本品" || prefix == "该产品" || prefix == "其")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func graphPriceSentenceSupports(sentence, name string, inherited bool, want graphPriceClaim) bool {
|
|||
|
|
for _, claim := range graphPriceClaims(sentence) {
|
|||
|
|
if !graphPricePrefixMatches(claim.prefix, name, inherited) {
|
|||
|
|
inherited = false
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
inherited = true
|
|||
|
|
if claim.label == want.label && graphPriceValuesMatch(want.values, claim.values) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func graphPriceTableCells(line string) []string {
|
|||
|
|
line = strings.TrimSpace(line)
|
|||
|
|
if !strings.HasPrefix(line, "|") {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
return strings.Split(strings.TrimSuffix(strings.TrimPrefix(line, "|"), "|"), "|")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func graphPriceProductHeader(value string) bool {
|
|||
|
|
switch compactGraphText(value) {
|
|||
|
|
case "产品名称", "商品名称", "品名", "产品":
|
|||
|
|
return true
|
|||
|
|
default:
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func graphPriceTableRowMatches(cells, headers []string, name string) bool {
|
|||
|
|
name = compactGraphText(name)
|
|||
|
|
if name == "" {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if len(headers) == 0 {
|
|||
|
|
for _, cell := range cells {
|
|||
|
|
if compactGraphText(cell) == name {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if len(cells) > len(headers) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
productColumn, brandColumn := -1, -1
|
|||
|
|
for i, header := range headers {
|
|||
|
|
if graphPriceProductHeader(header) {
|
|||
|
|
if productColumn >= 0 {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
productColumn = i
|
|||
|
|
}
|
|||
|
|
if header = compactGraphText(header); header == "品牌" || header == "品牌名称" {
|
|||
|
|
if brandColumn >= 0 {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
brandColumn = i
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if productColumn < 0 || productColumn >= len(cells) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
product := compactGraphText(cells[productColumn])
|
|||
|
|
if product == "" {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if product == name {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if brandColumn < 0 || brandColumn >= len(cells) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
brand := compactGraphText(cells[brandColumn])
|
|||
|
|
return brand != "" && brand+product == name
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func graphPriceAttributeSupported(content, name string, want graphPriceClaim) bool {
|
|||
|
|
var headers []string
|
|||
|
|
headingMatches := false
|
|||
|
|
for _, line := range strings.Split(content, "\n") {
|
|||
|
|
if cells := graphPriceTableCells(line); cells != nil {
|
|||
|
|
headingMatches = false
|
|||
|
|
isHeader := false
|
|||
|
|
for _, cell := range cells {
|
|||
|
|
if graphPriceProductHeader(cell) {
|
|||
|
|
isHeader = true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if isHeader {
|
|||
|
|
headers = cells
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
// A short row may end midway through its final cell. Only closed
|
|||
|
|
// cells have a known boundary; never validate a truncated price.
|
|||
|
|
if len(cells) < len(headers) && !strings.HasSuffix(strings.TrimSpace(line), "|") {
|
|||
|
|
cells = cells[:len(cells)-1]
|
|||
|
|
}
|
|||
|
|
if !graphPriceTableRowMatches(cells, headers, name) {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
for i, cell := range cells {
|
|||
|
|
if graphPriceSentenceSupports(cell, name, true, want) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
// A chunk may end before the row's final cells; only existing
|
|||
|
|
// leading cells can inherit their corresponding explicit header.
|
|||
|
|
if i >= len(headers) {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
header := compactGraphText(headers[i])
|
|||
|
|
span := graphPriceLabelPattern.FindStringIndex(header)
|
|||
|
|
if span == nil || span[0] != 0 || graphPriceLabel(header[:span[1]]) != want.label {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
unit := ""
|
|||
|
|
if suffix := header[span[1]:]; suffix != "" {
|
|||
|
|
match := graphPriceHeaderUnitPattern.FindStringSubmatch(suffix)
|
|||
|
|
if match == nil {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
unit = match[1]
|
|||
|
|
}
|
|||
|
|
if graphPriceValuesMatch(want.values, graphPriceValues(cell, unit)) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
headers = nil
|
|||
|
|
trimmed := strings.TrimSpace(line)
|
|||
|
|
if strings.HasPrefix(trimmed, "#") || graphProductNameMarkerPattern.MatchString(trimmed) {
|
|||
|
|
heading := strings.TrimLeft(trimmed, "# ")
|
|||
|
|
heading = strings.TrimPrefix(graphProductIdentity(heading), "产品名称")
|
|||
|
|
headingMatches = heading == graphProductIdentity(name)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
for _, sentence := range strings.FieldsFunc(line, func(r rune) bool { return strings.ContainsRune("。！？!?；;", r) }) {
|
|||
|
|
if graphPriceSentenceSupports(sentence, name, headingMatches, want) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func validateGraphPriceAttributes(graph *types.GraphData, content string) error {
|
|||
|
|
for i, node := range graph.Node {
|
|||
|
|
for j, attribute := range node.Attributes {
|
|||
|
|
for _, claim := range graphPriceClaims(attribute) {
|
|||
|
|
if !claim.hasQuantity {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if !graphPriceAttributeSupported(content, node.Name, claim) {
|
|||
|
|
return fmt.Errorf("%w: entity %d attribute %d has an unsupported price label/value (%s); require an explicit label and matching amount/unit in this product's sentence or table column, never infer unheaded prices", ErrGraphNeedsReview, i+1, j+1, claim.label)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return nil
|
|||
|
|
}
|