package chatpipeline import ( "fmt" "regexp" "strings" "github.com/Tencent/WeKnora/internal/types" ) var graphPriceLabelPattern = regexp.MustCompile(`(?:终端)?零售(?:价格|价)|批发(?:价格|价)|促销(?:价格|价)|优惠(?:价格|价)|活动(?:价格|价)|会员(?:价格|价)|建议零售价|销售价格|售价|单价|原价|现价|价格`) var graphPriceAmountPattern = regexp.MustCompile(`^(人民币|[¥¥$])?([0-9]+(?:\.[0-9]+)?)(美元|人民币|元|角|分|盒|箱|瓶|套|支|粒|片|件|袋|kg|mg|ml|g)?(/(?:[0-9一二两俩三四五六七八九十百千]+)?(?:盒|箱|瓶|套|支|粒|片|件|袋|kg|mg|ml|g))?`) var graphPriceHeaderUnitPattern = regexp.MustCompile(`^[((](元|美元|元/(?:盒|箱|瓶|套|支|件))[))]$`) type graphPriceValue struct { number string unit string } type graphPriceClaim struct { label string values []graphPriceValue prefix string hasQuantity bool } func graphPriceValues(value, headerUnit string) []graphPriceValue { value = strings.TrimLeft(compactGraphText(value), "::为是") var values []graphPriceValue for value != "" { match := graphPriceAmountPattern.FindStringSubmatch(value) if match == nil { // Do not accept a parsed prefix while silently ignoring a changed // package quantity or another amount later in the same price claim. if strings.HasPrefix(value, "/") || graphQuantityPattern.MatchString(value) { return nil } break } unit := match[3] + match[4] if match[1] != "" { currency := "元" if match[1] == "$" { currency = "美元" } if unit == "" || strings.HasPrefix(unit, "/") { unit = currency + unit } } if unit == "" { unit = headerUnit } values = append(values, graphPriceValue{number: match[2], unit: unit}) value = strings.TrimLeft(value[len(match[0]):], ",,、") } return values } func graphPriceClaims(value string) []graphPriceClaim { value = compactGraphText(value) labels := graphPriceLabelPattern.FindAllStringIndex(value, -1) var claims []graphPriceClaim for i, span := range labels { end := len(value) if i+1 < len(labels) { end = labels[i+1][0] } prefix := value[:span[0]] if index := strings.LastIndexAny(prefix, ",,;;。|::"); index >= 0 { prefix = strings.TrimLeft(prefix[index:], ",,;;。|::") } claims = append(claims, graphPriceClaim{ label: graphPriceLabel(value[span[0]:span[1]]), values: graphPriceValues(value[span[1]:end], ""), prefix: prefix, hasQuantity: graphQuantityPattern.MatchString(value[span[1]:end]), }) } return claims } func graphPriceLabel(value string) string { return strings.TrimPrefix(strings.ReplaceAll(value, "价格", "价"), "终端") } func graphPriceValuesMatch(want, available []graphPriceValue) bool { if len(want) == 0 || len(want) > len(available) { return false } for start := 0; start+len(want) <= len(available); start++ { matches := true for i, value := range want { actual := available[start+i] if value.number != actual.number || (value.unit != "" && value.unit != actual.unit) { matches = false break } } if matches { return true } } return false } func graphPricePrefixMatches(prefix, name string, inherited bool) bool { prefix = strings.TrimSuffix(compactGraphText(prefix), "的") if prefix == compactGraphText(name) { return true } return inherited && (prefix == "" || prefix == "本品" || prefix == "该产品" || prefix == "其") } func graphPriceSentenceSupports(sentence, name string, inherited bool, want graphPriceClaim) bool { for _, claim := range graphPriceClaims(sentence) { if !graphPricePrefixMatches(claim.prefix, name, inherited) { inherited = false continue } inherited = true if claim.label == want.label && graphPriceValuesMatch(want.values, claim.values) { return true } } return false } func graphPriceTableCells(line string) []string { line = strings.TrimSpace(line) if !strings.HasPrefix(line, "|") { return nil } return strings.Split(strings.TrimSuffix(strings.TrimPrefix(line, "|"), "|"), "|") } func graphPriceProductHeader(value string) bool { switch compactGraphText(value) { case "产品名称", "商品名称", "品名", "产品": return true default: return false } } func graphPriceTableRowMatches(cells, headers []string, name string) bool { name = compactGraphText(name) if name == "" { return false } if len(headers) == 0 { for _, cell := range cells { if compactGraphText(cell) == name { return true } } return false } if len(cells) > len(headers) { return false } productColumn, brandColumn := -1, -1 for i, header := range headers { if graphPriceProductHeader(header) { if productColumn >= 0 { return false } productColumn = i } if header = compactGraphText(header); header == "品牌" || header == "品牌名称" { if brandColumn >= 0 { return false } brandColumn = i } } if productColumn < 0 || productColumn >= len(cells) { return false } product := compactGraphText(cells[productColumn]) if product == "" { return false } if product == name { return true } if brandColumn < 0 || brandColumn >= len(cells) { return false } brand := compactGraphText(cells[brandColumn]) return brand != "" && brand+product == name } func graphPriceAttributeSupported(content, name string, want graphPriceClaim) bool { var headers []string headingMatches := false for _, line := range strings.Split(content, "\n") { if cells := graphPriceTableCells(line); cells != nil { headingMatches = false isHeader := false for _, cell := range cells { if graphPriceProductHeader(cell) { isHeader = true } } if isHeader { headers = cells continue } // A short row may end midway through its final cell. Only closed // cells have a known boundary; never validate a truncated price. if len(cells) < len(headers) && !strings.HasSuffix(strings.TrimSpace(line), "|") { cells = cells[:len(cells)-1] } if !graphPriceTableRowMatches(cells, headers, name) { continue } for i, cell := range cells { if graphPriceSentenceSupports(cell, name, true, want) { return true } // A chunk may end before the row's final cells; only existing // leading cells can inherit their corresponding explicit header. if i >= len(headers) { continue } header := compactGraphText(headers[i]) span := graphPriceLabelPattern.FindStringIndex(header) if span == nil || span[0] != 0 || graphPriceLabel(header[:span[1]]) != want.label { continue } unit := "" if suffix := header[span[1]:]; suffix != "" { match := graphPriceHeaderUnitPattern.FindStringSubmatch(suffix) if match == nil { continue } unit = match[1] } if graphPriceValuesMatch(want.values, graphPriceValues(cell, unit)) { return true } } continue } headers = nil trimmed := strings.TrimSpace(line) if strings.HasPrefix(trimmed, "#") || graphProductNameMarkerPattern.MatchString(trimmed) { heading := strings.TrimLeft(trimmed, "# ") heading = strings.TrimPrefix(graphProductIdentity(heading), "产品名称") headingMatches = heading == graphProductIdentity(name) continue } for _, sentence := range strings.FieldsFunc(line, func(r rune) bool { return strings.ContainsRune("。!?!?;;", r) }) { if graphPriceSentenceSupports(sentence, name, headingMatches, want) { return true } } } return false } func validateGraphPriceAttributes(graph *types.GraphData, content string) error { for i, node := range graph.Node { for j, attribute := range node.Attributes { for _, claim := range graphPriceClaims(attribute) { if !claim.hasQuantity { continue } if !graphPriceAttributeSupported(content, node.Name, claim) { return fmt.Errorf("%w: entity %d attribute %d has an unsupported price label/value (%s); require an explicit label and matching amount/unit in this product's sentence or table column, never infer unheaded prices", ErrGraphNeedsReview, i+1, j+1, claim.label) } } } } return nil }