WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/application/service/chat_pipeline/extract_entity_price.go

@@ -0,0 +1,276 @@
+package chatpipeline
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/Tencent/WeKnora/internal/types"
+)
+
+var graphPriceLabelPattern = regexp.MustCompile(`(?:终端)?零售(?:价格|价)|批发(?:价格|价)|促销(?:价格|价)|优惠(?:价格|价)|活动(?:价格|价)|会员(?:价格|价)|建议零售价|销售价格|售价|单价|原价|现价|价格`)
+var graphPriceAmountPattern = regexp.MustCompile(`^(人民币|[¥￥$])?([0-9]+(?:\.[0-9]+)?)(美元|人民币|元|角|分|盒|箱|瓶|套|支|粒|片|件|袋|kg|mg|ml|g)?(/(?:[0-9一二两俩三四五六七八九十百千]+)?(?:盒|箱|瓶|套|支|粒|片|件|袋|kg|mg|ml|g))?`)
+var graphPriceHeaderUnitPattern = regexp.MustCompile(`^[（(](元|美元|元/(?:盒|箱|瓶|套|支|件))[）)]$`)
+
+type graphPriceValue struct {
+	number string
+	unit   string
+}
+
+type graphPriceClaim struct {
+	label       string
+	values      []graphPriceValue
+	prefix      string
+	hasQuantity bool
+}
+
+func graphPriceValues(value, headerUnit string) []graphPriceValue {
+	value = strings.TrimLeft(compactGraphText(value), "：:为是")
+	var values []graphPriceValue
+	for value != "" {
+		match := graphPriceAmountPattern.FindStringSubmatch(value)
+		if match == nil {
+			// Do not accept a parsed prefix while silently ignoring a changed
+			// package quantity or another amount later in the same price claim.
+			if strings.HasPrefix(value, "/") || graphQuantityPattern.MatchString(value) {
+				return nil
+			}
+			break
+		}
+		unit := match[3] + match[4]
+		if match[1] != "" {
+			currency := "元"
+			if match[1] == "$" {
+				currency = "美元"
+			}
+			if unit == "" || strings.HasPrefix(unit, "/") {
+				unit = currency + unit
+			}
+		}
+		if unit == "" {
+			unit = headerUnit
+		}
+		values = append(values, graphPriceValue{number: match[2], unit: unit})
+		value = strings.TrimLeft(value[len(match[0]):], "，,、")
+	}
+	return values
+}
+
+func graphPriceClaims(value string) []graphPriceClaim {
+	value = compactGraphText(value)
+	labels := graphPriceLabelPattern.FindAllStringIndex(value, -1)
+	var claims []graphPriceClaim
+	for i, span := range labels {
+		end := len(value)
+		if i+1 < len(labels) {
+			end = labels[i+1][0]
+		}
+		prefix := value[:span[0]]
+		if index := strings.LastIndexAny(prefix, "，,；;。|：:"); index >= 0 {
+			prefix = strings.TrimLeft(prefix[index:], "，,；;。|：:")
+		}
+		claims = append(claims, graphPriceClaim{
+			label:       graphPriceLabel(value[span[0]:span[1]]),
+			values:      graphPriceValues(value[span[1]:end], ""),
+			prefix:      prefix,
+			hasQuantity: graphQuantityPattern.MatchString(value[span[1]:end]),
+		})
+	}
+	return claims
+}
+
+func graphPriceLabel(value string) string {
+	return strings.TrimPrefix(strings.ReplaceAll(value, "价格", "价"), "终端")
+}
+
+func graphPriceValuesMatch(want, available []graphPriceValue) bool {
+	if len(want) == 0 || len(want) > len(available) {
+		return false
+	}
+	for start := 0; start+len(want) <= len(available); start++ {
+		matches := true
+		for i, value := range want {
+			actual := available[start+i]
+			if value.number != actual.number || (value.unit != "" && value.unit != actual.unit) {
+				matches = false
+				break
+			}
+		}
+		if matches {
+			return true
+		}
+	}
+	return false
+}
+
+func graphPricePrefixMatches(prefix, name string, inherited bool) bool {
+	prefix = strings.TrimSuffix(compactGraphText(prefix), "的")
+	if prefix == compactGraphText(name) {
+		return true
+	}
+	return inherited && (prefix == "" || prefix == "本品" || prefix == "该产品" || prefix == "其")
+}
+
+func graphPriceSentenceSupports(sentence, name string, inherited bool, want graphPriceClaim) bool {
+	for _, claim := range graphPriceClaims(sentence) {
+		if !graphPricePrefixMatches(claim.prefix, name, inherited) {
+			inherited = false
+			continue
+		}
+		inherited = true
+		if claim.label == want.label && graphPriceValuesMatch(want.values, claim.values) {
+			return true
+		}
+	}
+	return false
+}
+
+func graphPriceTableCells(line string) []string {
+	line = strings.TrimSpace(line)
+	if !strings.HasPrefix(line, "|") {
+		return nil
+	}
+	return strings.Split(strings.TrimSuffix(strings.TrimPrefix(line, "|"), "|"), "|")
+}
+
+func graphPriceProductHeader(value string) bool {
+	switch compactGraphText(value) {
+	case "产品名称", "商品名称", "品名", "产品":
+		return true
+	default:
+		return false
+	}
+}
+
+func graphPriceTableRowMatches(cells, headers []string, name string) bool {
+	name = compactGraphText(name)
+	if name == "" {
+		return false
+	}
+	if len(headers) == 0 {
+		for _, cell := range cells {
+			if compactGraphText(cell) == name {
+				return true
+			}
+		}
+		return false
+	}
+	if len(cells) > len(headers) {
+		return false
+	}
+	productColumn, brandColumn := -1, -1
+	for i, header := range headers {
+		if graphPriceProductHeader(header) {
+			if productColumn >= 0 {
+				return false
+			}
+			productColumn = i
+		}
+		if header = compactGraphText(header); header == "品牌" || header == "品牌名称" {
+			if brandColumn >= 0 {
+				return false
+			}
+			brandColumn = i
+		}
+	}
+	if productColumn < 0 || productColumn >= len(cells) {
+		return false
+	}
+	product := compactGraphText(cells[productColumn])
+	if product == "" {
+		return false
+	}
+	if product == name {
+		return true
+	}
+	if brandColumn < 0 || brandColumn >= len(cells) {
+		return false
+	}
+	brand := compactGraphText(cells[brandColumn])
+	return brand != "" && brand+product == name
+}
+
+func graphPriceAttributeSupported(content, name string, want graphPriceClaim) bool {
+	var headers []string
+	headingMatches := false
+	for _, line := range strings.Split(content, "\n") {
+		if cells := graphPriceTableCells(line); cells != nil {
+			headingMatches = false
+			isHeader := false
+			for _, cell := range cells {
+				if graphPriceProductHeader(cell) {
+					isHeader = true
+				}
+			}
+			if isHeader {
+				headers = cells
+				continue
+			}
+			// A short row may end midway through its final cell. Only closed
+			// cells have a known boundary; never validate a truncated price.
+			if len(cells) < len(headers) && !strings.HasSuffix(strings.TrimSpace(line), "|") {
+				cells = cells[:len(cells)-1]
+			}
+			if !graphPriceTableRowMatches(cells, headers, name) {
+				continue
+			}
+			for i, cell := range cells {
+				if graphPriceSentenceSupports(cell, name, true, want) {
+					return true
+				}
+				// A chunk may end before the row's final cells; only existing
+				// leading cells can inherit their corresponding explicit header.
+				if i >= len(headers) {
+					continue
+				}
+				header := compactGraphText(headers[i])
+				span := graphPriceLabelPattern.FindStringIndex(header)
+				if span == nil || span[0] != 0 || graphPriceLabel(header[:span[1]]) != want.label {
+					continue
+				}
+				unit := ""
+				if suffix := header[span[1]:]; suffix != "" {
+					match := graphPriceHeaderUnitPattern.FindStringSubmatch(suffix)
+					if match == nil {
+						continue
+					}
+					unit = match[1]
+				}
+				if graphPriceValuesMatch(want.values, graphPriceValues(cell, unit)) {
+					return true
+				}
+			}
+			continue
+		}
+		headers = nil
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "#") || graphProductNameMarkerPattern.MatchString(trimmed) {
+			heading := strings.TrimLeft(trimmed, "# ")
+			heading = strings.TrimPrefix(graphProductIdentity(heading), "产品名称")
+			headingMatches = heading == graphProductIdentity(name)
+			continue
+		}
+		for _, sentence := range strings.FieldsFunc(line, func(r rune) bool { return strings.ContainsRune("。！？!?；;", r) }) {
+			if graphPriceSentenceSupports(sentence, name, headingMatches, want) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func validateGraphPriceAttributes(graph *types.GraphData, content string) error {
+	for i, node := range graph.Node {
+		for j, attribute := range node.Attributes {
+			for _, claim := range graphPriceClaims(attribute) {
+				if !claim.hasQuantity {
+					continue
+				}
+				if !graphPriceAttributeSupported(content, node.Name, claim) {
+					return fmt.Errorf("%w: entity %d attribute %d has an unsupported price label/value (%s); require an explicit label and matching amount/unit in this product's sentence or table column, never infer unheaded prices", ErrGraphNeedsReview, i+1, j+1, claim.label)
+				}
+			}
+		}
+	}
+	return nil
+}