WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes
Anydoc / anydoc parser engine (push) Canceled after 0s
App / Format, vet, test, and build (push) Canceled after 0s
CLI / test (macos-latest / go 1.26) (push) Canceled after 0s
CLI / test (ubuntu-latest / go 1.26) (push) Canceled after 0s
CLI / test (windows-latest / go 1.26) (push) Canceled after 0s
Build and Push Docker Image / build-ui (push) Canceled after 0s
Build and Push Docker Image / build-docreader (push) Canceled after 0s
Build and Push Docker Image / build-sandbox (push) Canceled after 0s
Build and Push Docker Image / build-app (amd64, linux/amd64, ubuntu-latest) (push) Canceled after 0s
Build and Push Docker Image / build-app (arm64, linux/arm64, ubuntu-24.04-arm) (push) Canceled after 0s
DocReader / Python tests and Go client tests (push) Canceled after 0s
Build & Publish (dsh-weknora) / Typecheck, test, pack (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (pinned) (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (latest, advisory) (push) Canceled after 0s
Frontend / Test, type-check, and build (push) Canceled after 0s
Go Lint Cache / warm lint cache (.) (push) Canceled after 0s
Go Lint Cache / warm lint cache (cli) (push) Canceled after 0s
Go Lint Cache / warm lint cache (client) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.10) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.11) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.12) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.13) (push) Canceled after 0s
Build and Push Docker Image / merge (push) Canceled after 0s
Build & Publish (dsh-weknora) / Publish to npm (push) Canceled after 0s
Build & Publish (mcp-server) / Build sdist + wheel (push) Canceled after 0s
Build & Publish (mcp-server) / Publish to PyPI (push) Canceled after 0s
Anydoc / anydoc parser engine (push) Canceled after 0s
App / Format, vet, test, and build (push) Canceled after 0s
CLI / test (macos-latest / go 1.26) (push) Canceled after 0s
CLI / test (ubuntu-latest / go 1.26) (push) Canceled after 0s
CLI / test (windows-latest / go 1.26) (push) Canceled after 0s
Build and Push Docker Image / build-ui (push) Canceled after 0s
Build and Push Docker Image / build-docreader (push) Canceled after 0s
Build and Push Docker Image / build-sandbox (push) Canceled after 0s
Build and Push Docker Image / build-app (amd64, linux/amd64, ubuntu-latest) (push) Canceled after 0s
Build and Push Docker Image / build-app (arm64, linux/arm64, ubuntu-24.04-arm) (push) Canceled after 0s
DocReader / Python tests and Go client tests (push) Canceled after 0s
Build & Publish (dsh-weknora) / Typecheck, test, pack (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (pinned) (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (latest, advisory) (push) Canceled after 0s
Frontend / Test, type-check, and build (push) Canceled after 0s
Go Lint Cache / warm lint cache (.) (push) Canceled after 0s
Go Lint Cache / warm lint cache (cli) (push) Canceled after 0s
Go Lint Cache / warm lint cache (client) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.10) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.11) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.12) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.13) (push) Canceled after 0s
Build and Push Docker Image / merge (push) Canceled after 0s
Build & Publish (dsh-weknora) / Publish to npm (push) Canceled after 0s
Build & Publish (mcp-server) / Build sdist + wheel (push) Canceled after 0s
Build & Publish (mcp-server) / Publish to PyPI (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1,455 @@
|
||||
package legacygraph
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
neo4jrepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/neo4j"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Repository preserves source records while exposing verified native chunks to RAG.
|
||||
type Repository struct {
|
||||
db *gorm.DB
|
||||
fallback interfaces.RetrieveGraphRepository
|
||||
}
|
||||
|
||||
func NewRepository(db *gorm.DB, driver neo4j.Driver) *Repository {
|
||||
r := &Repository{db: db}
|
||||
if driver != nil {
|
||||
r.fallback = neo4jrepo.NewNeo4jRepository(driver)
|
||||
}
|
||||
return r
|
||||
}
|
||||
func NewRetrieveRepository(db *gorm.DB, driver neo4j.Driver) interfaces.RetrieveGraphRepository {
|
||||
return NewRepository(db, driver)
|
||||
}
|
||||
func (r *Repository) AddGraph(ctx context.Context, ns types.NameSpace, graphs []*types.GraphData) error {
|
||||
return r.addNativeGraphs(ctx, ns, graphs, nil)
|
||||
}
|
||||
func (r *Repository) DelGraph(ctx context.Context, ns []types.NameSpace) error {
|
||||
return r.deleteNativeGraphs(ctx, ns)
|
||||
}
|
||||
|
||||
type Record struct {
|
||||
ID int64 `json:"id" gorm:"column:legacy_id"`
|
||||
Payload json.RawMessage `json:"payload" gorm:"column:payload"`
|
||||
}
|
||||
type Evidence struct {
|
||||
Record
|
||||
NativeKnowledgeID string `json:"native_knowledge_id"`
|
||||
NativeChunkID string `json:"native_chunk_id"`
|
||||
MappingState string `json:"mapping_state"`
|
||||
}
|
||||
type Overview struct {
|
||||
Counts map[string]int64 `json:"counts"`
|
||||
Sources int64 `json:"sources"`
|
||||
MappedSources int64 `json:"mapped_sources"`
|
||||
Nodes []Record `json:"nodes"`
|
||||
Relations []Record `json:"relations"`
|
||||
Evidence []Evidence `json:"evidence"`
|
||||
NextCursor int64 `json:"next_cursor"`
|
||||
}
|
||||
|
||||
func (r *Repository) scope(ctx context.Context, kb string) (*gorm.DB, error) {
|
||||
tenant, ok := types.TenantIDFromContext(ctx)
|
||||
if !ok || tenant == 0 || kb == "" {
|
||||
return nil, fmt.Errorf("graph tenant and knowledge base are required")
|
||||
}
|
||||
return r.db.WithContext(ctx).Table("native_graph_records").Where("tenant_id = ? AND knowledge_base_id = ?", tenant, kb), nil
|
||||
}
|
||||
func (r *Repository) Browse(ctx context.Context, kb, query string, cursor, entity int64) (*Overview, error) {
|
||||
scope, err := r.scope(ctx, kb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := &Overview{Counts: map[string]int64{}, Nodes: []Record{}, Relations: []Record{}, Evidence: []Evidence{}}
|
||||
var counts []struct {
|
||||
SourceTable string
|
||||
N int64
|
||||
}
|
||||
if err = scope.Session(&gorm.Session{}).Select("source_table, count(*) AS n").Group("source_table").Scan(&counts).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range counts {
|
||||
out.Counts[c.SourceTable] = c.N
|
||||
}
|
||||
tenant := types.MustTenantIDFromContext(ctx)
|
||||
var coverage struct {
|
||||
Sources int64
|
||||
Mapped int64
|
||||
}
|
||||
// Overview coverage records migration-time matches; each returned link is revalidated below.
|
||||
err = r.db.WithContext(ctx).Raw(`SELECT count(*) AS sources, count(native_chunk_id) AS mapped FROM native_graph_sources WHERE tenant_id=? AND knowledge_base_id=?`, tenant, kb).Scan(&coverage).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out.Sources = coverage.Sources
|
||||
out.MappedSources = coverage.Mapped
|
||||
q := scope.Session(&gorm.Session{}).Select("legacy_id, payload").Where("source_table = ? AND legacy_id > ?", "knowledgeentity", cursor)
|
||||
if entity > 0 {
|
||||
q = q.Where("legacy_id = ?", entity)
|
||||
} else if strings.TrimSpace(query) != "" {
|
||||
q = q.Where("strpos(lower(payload->>'canonical_name'), lower(?)) > 0 OR strpos(lower(?), lower(payload->>'canonical_name')) > 0", query, query)
|
||||
}
|
||||
if err = q.Order("legacy_id").Limit(51).Find(&out.Nodes).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(out.Nodes) > 50 {
|
||||
out.NextCursor = out.Nodes[49].ID
|
||||
out.Nodes = out.Nodes[:50]
|
||||
}
|
||||
if err = r.projectNativeEntityAttributes(ctx, types.NameSpace{KnowledgeBase: kb}, out.Nodes, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if entity <= 0 {
|
||||
return out, nil
|
||||
}
|
||||
err = r.db.WithContext(ctx).Raw(`SELECT r.legacy_id, r.payload || jsonb_build_object('from_name',a.payload->>'canonical_name','to_name',b.payload->>'canonical_name') AS payload FROM native_graph_records r LEFT JOIN native_graph_records a ON a.tenant_id=r.tenant_id AND a.knowledge_base_id=r.knowledge_base_id AND a.source_table='knowledgeentity' AND a.legacy_id=(r.payload->>'from_entity_id')::bigint LEFT JOIN native_graph_records b ON b.tenant_id=r.tenant_id AND b.knowledge_base_id=r.knowledge_base_id AND b.source_table='knowledgeentity' AND b.legacy_id=(r.payload->>'to_entity_id')::bigint WHERE r.tenant_id=? AND r.knowledge_base_id=? AND r.source_table='knowledgerelation' AND (r.payload->>'from_entity_id'=? OR r.payload->>'to_entity_id'=?) ORDER BY r.legacy_id LIMIT 100`, tenant, kb, strconv.FormatInt(entity, 10), strconv.FormatInt(entity, 10)).Scan(&out.Relations).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = r.db.WithContext(ctx).Raw(`SELECT r.legacy_id,r.payload, COALESCE(c.knowledge_id,'') AS native_knowledge_id, COALESCE(c.id,'') AS native_chunk_id, CASE WHEN c.id IS NOT NULL THEN s.mapping_state WHEN s.native_chunk_id IS NOT NULL THEN 'stale' ELSE COALESCE(s.mapping_state,'unmapped') END AS mapping_state FROM native_graph_records r LEFT JOIN native_graph_sources s ON s.tenant_id=r.tenant_id AND s.knowledge_base_id=r.knowledge_base_id AND s.source_chunk_id=r.payload->>'source_chunk_id' LEFT JOIN chunks c ON c.id=s.native_chunk_id AND c.tenant_id=s.tenant_id AND c.knowledge_base_id=s.knowledge_base_id AND c.deleted_at IS NULL AND c.is_enabled=true AND encode(sha256(convert_to(c.content,'UTF8')),'hex')=s.native_content_sha256 AND EXISTS (SELECT 1 FROM knowledges k WHERE k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id AND k.deleted_at IS NULL) AND EXISTS (SELECT 1 FROM knowledge_bases kb WHERE kb.id=c.knowledge_base_id AND kb.tenant_id=c.tenant_id AND kb.deleted_at IS NULL) WHERE r.tenant_id=? AND r.knowledge_base_id=? AND r.source_table='knowledgegraphevidence' AND r.payload->>'subject_type'='entity' AND r.payload->>'subject_id'=? ORDER BY r.legacy_id LIMIT 100`, tenant, kb, strconv.FormatInt(entity, 10)).Scan(&out.Evidence).Error
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (r *Repository) SearchNode(ctx context.Context, ns types.NameSpace, terms []string) (*types.GraphData, error) {
|
||||
out := &types.GraphData{Node: []*types.GraphNode{}, Relation: []*types.GraphRelation{}}
|
||||
scope, err := r.scope(ctx, ns.KnowledgeBase)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
terms = boundedTerms(terms)
|
||||
if len(terms) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
clauses := make([]string, 0, len(terms))
|
||||
args := make([]any, 0, len(terms)*2)
|
||||
for _, term := range terms {
|
||||
clauses = append(clauses, "(strpos(lower(payload->>'canonical_name'),lower(?))>0 OR strpos(lower(?),lower(payload->>'canonical_name'))>0)")
|
||||
args = append(args, term, term)
|
||||
}
|
||||
var nodes []Record
|
||||
err = scope.Session(&gorm.Session{}).Select("legacy_id,payload").Where("source_table='knowledgeentity' AND payload->>'status'='active'").Where(strings.Join(clauses, " OR "), args...).Order("legacy_id").Limit(16).Find(&nodes).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tenant := types.MustTenantIDFromContext(ctx)
|
||||
seedIDs := []string{}
|
||||
seenIDs := map[int64]bool{}
|
||||
for _, node := range nodes {
|
||||
seedIDs = append(seedIDs, strconv.FormatInt(node.ID, 10))
|
||||
seenIDs[node.ID] = true
|
||||
}
|
||||
var relations []Record
|
||||
if len(seedIDs) > 0 {
|
||||
q, _ := r.scope(ctx, ns.KnowledgeBase)
|
||||
err = q.Select("legacy_id,payload").Where("source_table='knowledgerelation' AND payload->>'status'='active' AND (payload->>'from_entity_id' IN ? OR payload->>'to_entity_id' IN ?)", seedIDs, seedIDs).Order("legacy_id").Limit(32).Find(&relations).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
neighborIDs := []int64{}
|
||||
for _, rel := range relations {
|
||||
var edge struct {
|
||||
From int64 `json:"from_entity_id"`
|
||||
To int64 `json:"to_entity_id"`
|
||||
}
|
||||
if err = json.Unmarshal(rel.Payload, &edge); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, id := range []int64{edge.From, edge.To} {
|
||||
if !seenIDs[id] && len(neighborIDs) < 16 {
|
||||
seenIDs[id] = true
|
||||
neighborIDs = append(neighborIDs, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(neighborIDs) > 0 {
|
||||
var neighbors []Record
|
||||
q, _ = r.scope(ctx, ns.KnowledgeBase)
|
||||
if err = q.Select("legacy_id,payload").Where("source_table='knowledgeentity' AND payload->>'status'='active' AND legacy_id IN ?", neighborIDs).Order("legacy_id").Find(&neighbors).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes = append(nodes, neighbors...)
|
||||
}
|
||||
}
|
||||
chunksByEntity := map[int64][]string{}
|
||||
chunksByRelation := map[int64][]string{}
|
||||
if len(nodes) > 0 {
|
||||
type subject struct {
|
||||
ID int64 `json:"subject_id"`
|
||||
Type string `json:"subject_type"`
|
||||
}
|
||||
ids := make([]subject, 0, len(nodes)+len(relations))
|
||||
for _, node := range nodes {
|
||||
ids = append(ids, subject{node.ID, "entity"})
|
||||
}
|
||||
for _, relation := range relations {
|
||||
ids = append(ids, subject{relation.ID, "relation"})
|
||||
}
|
||||
encodedIDs, err := json.Marshal(ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var matches []struct {
|
||||
SubjectID int64
|
||||
SubjectType string
|
||||
ChunkID string
|
||||
}
|
||||
// One bounded database request, with the evidence index and four-chunk cap per entity.
|
||||
query := `SELECT seed.subject_id,seed.subject_type,matched.chunk_id FROM jsonb_to_recordset(?::jsonb) AS seed(subject_id bigint,subject_type text) CROSS JOIN LATERAL (
|
||||
SELECT DISTINCT c.id AS chunk_id FROM native_graph_records e
|
||||
JOIN native_graph_sources s ON s.tenant_id=e.tenant_id AND s.knowledge_base_id=e.knowledge_base_id AND s.source_chunk_id=e.payload->>'source_chunk_id'
|
||||
JOIN chunks c ON c.id=s.native_chunk_id AND c.tenant_id=s.tenant_id AND c.knowledge_base_id=s.knowledge_base_id AND c.deleted_at IS NULL AND c.is_enabled=true AND encode(sha256(convert_to(c.content,'UTF8')),'hex')=s.native_content_sha256
|
||||
JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id AND k.deleted_at IS NULL
|
||||
JOIN knowledge_bases kb ON kb.id=c.knowledge_base_id AND kb.tenant_id=c.tenant_id AND kb.deleted_at IS NULL
|
||||
WHERE e.tenant_id=? AND e.knowledge_base_id=? AND e.source_table='knowledgegraphevidence' AND e.payload->>'subject_type'=seed.subject_type AND e.payload->>'subject_id'=seed.subject_id::text AND e.payload->>'status'='active'`
|
||||
args := []any{string(encodedIDs), tenant, ns.KnowledgeBase}
|
||||
if ns.Knowledge != "" {
|
||||
query += " AND c.knowledge_id=?"
|
||||
args = append(args, ns.Knowledge)
|
||||
}
|
||||
query += " ORDER BY c.id LIMIT 4) matched"
|
||||
if err = r.db.WithContext(ctx).Raw(query, args...).Scan(&matches).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, match := range matches {
|
||||
if match.SubjectType == "entity" {
|
||||
chunksByEntity[match.SubjectID] = append(chunksByEntity[match.SubjectID], match.ChunkID)
|
||||
} else {
|
||||
chunksByRelation[match.SubjectID] = append(chunksByRelation[match.SubjectID], match.ChunkID)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err = r.projectNativeEntityAttributes(ctx, ns, nodes, chunksByEntity); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := map[int64]string{}
|
||||
for _, node := range nodes {
|
||||
var entity struct {
|
||||
Name string `json:"canonical_name"`
|
||||
Type string `json:"entity_type"`
|
||||
Attributes []string `json:"attributes"`
|
||||
}
|
||||
if err = json.Unmarshal(node.Payload, &entity); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chunks := chunksByEntity[node.ID]
|
||||
if len(chunks) > 0 {
|
||||
names[node.ID] = entity.Name
|
||||
attributes := []string{entity.Type}
|
||||
identity := fmt.Sprintf("legacy_entity:%d", node.ID)
|
||||
for _, attribute := range entity.Attributes {
|
||||
if attribute != entity.Type && attribute != identity {
|
||||
attributes = append(attributes, attribute)
|
||||
}
|
||||
}
|
||||
attributes = append(attributes, identity)
|
||||
out.Node = append(out.Node, &types.GraphNode{Name: entity.Name, Chunks: chunks, Attributes: attributes})
|
||||
}
|
||||
}
|
||||
for _, rel := range relations {
|
||||
var edge struct {
|
||||
From int64 `json:"from_entity_id"`
|
||||
To int64 `json:"to_entity_id"`
|
||||
Type string `json:"relation_type"`
|
||||
Origin string `json:"origin"`
|
||||
}
|
||||
if err = json.Unmarshal(rel.Payload, &edge); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if edge.Origin == nativeOrigin && len(chunksByRelation[rel.ID]) == 0 {
|
||||
continue
|
||||
}
|
||||
if names[edge.From] != "" && names[edge.To] != "" {
|
||||
out.Relation = append(out.Relation, &types.GraphRelation{Node1: names[edge.From], Node2: names[edge.To], Type: edge.Type})
|
||||
}
|
||||
}
|
||||
if r.fallback != nil {
|
||||
fallback, err := r.fallback.SearchNode(ctx, ns, terms)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fallback != nil {
|
||||
out.Node = append(out.Node, fallback.Node...)
|
||||
out.Relation = append(out.Relation, fallback.Relation...)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Attribute ownership is checked independently of the four returned chunks:
|
||||
// an entity may still have a bare reference after its attribute source expires.
|
||||
func (r *Repository) projectNativeEntityAttributes(ctx context.Context, ns types.NameSpace, nodes []Record, chunks map[int64][]string) error {
|
||||
type subject struct {
|
||||
ID int64 `json:"subject_id"`
|
||||
Knowledge string `json:"knowledge_id"`
|
||||
Type string `json:"entity_type"`
|
||||
Attributes json.RawMessage `json:"attributes"`
|
||||
}
|
||||
seeds := []subject{}
|
||||
payloads := map[int64]map[string]json.RawMessage{}
|
||||
for _, node := range nodes {
|
||||
var entity struct {
|
||||
Origin string `json:"origin"`
|
||||
Knowledge string `json:"native_knowledge_id"`
|
||||
Type string `json:"entity_type"`
|
||||
Attributes json.RawMessage `json:"attributes"`
|
||||
}
|
||||
if err := json.Unmarshal(node.Payload, &entity); err != nil {
|
||||
return err
|
||||
}
|
||||
if entity.Origin != nativeOrigin {
|
||||
continue
|
||||
}
|
||||
var payload map[string]json.RawMessage
|
||||
if err := json.Unmarshal(node.Payload, &payload); err != nil {
|
||||
return err
|
||||
}
|
||||
payloads[node.ID] = payload
|
||||
seeds = append(seeds, subject{node.ID, entity.Knowledge, entity.Type, entity.Attributes})
|
||||
}
|
||||
if len(seeds) == 0 {
|
||||
return nil
|
||||
}
|
||||
encoded, err := json.Marshal(seeds)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var matches []struct {
|
||||
SubjectID int64
|
||||
Snapshots json.RawMessage
|
||||
Chunks json.RawMessage
|
||||
AttributeChunk string
|
||||
}
|
||||
query := `SELECT seed.subject_id,jsonb_agg(DISTINCT e.payload->'entity_attributes') AS snapshots,
|
||||
to_jsonb((array_agg(DISTINCT c.id ORDER BY c.id))[1:4]) AS chunks,
|
||||
COALESCE(min(c.id) FILTER (WHERE e.payload->'entity_attributes'->>0 IS NOT NULL
|
||||
AND (e.payload->'entity_attributes'=seed.attributes OR e.payload->'entity_attributes'->>0=seed.entity_type)),'') AS attribute_chunk
|
||||
FROM jsonb_to_recordset(?::jsonb) AS seed(subject_id bigint,knowledge_id text,entity_type text,attributes jsonb)
|
||||
JOIN native_graph_records e ON e.payload->>'subject_id'=seed.subject_id::text AND e.payload->>'native_knowledge_id'=seed.knowledge_id
|
||||
JOIN native_graph_sources s ON s.tenant_id=e.tenant_id AND s.knowledge_base_id=e.knowledge_base_id AND s.source_chunk_id=e.payload->>'source_chunk_id'
|
||||
JOIN chunks c ON c.id=s.native_chunk_id AND c.tenant_id=s.tenant_id AND c.knowledge_base_id=s.knowledge_base_id
|
||||
AND s.native_knowledge_id=c.knowledge_id AND c.deleted_at IS NULL AND c.is_enabled=true
|
||||
AND encode(sha256(convert_to(c.content,'UTF8')),'hex')=s.native_content_sha256
|
||||
JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id AND k.deleted_at IS NULL
|
||||
JOIN knowledge_bases kb ON kb.id=c.knowledge_base_id AND kb.tenant_id=c.tenant_id AND kb.deleted_at IS NULL
|
||||
WHERE e.tenant_id=? AND e.knowledge_base_id=? AND e.source_table='knowledgegraphevidence' AND e.payload->>'subject_type'='entity'
|
||||
AND e.payload->>'origin'=? AND e.payload->>'status'='active' AND s.mapping_state='native_extraction'
|
||||
AND e.payload->>'source_content_hash'=s.native_content_sha256 AND e.source_snapshot_sha256=s.native_content_sha256
|
||||
AND s.source_content_hash=s.native_content_sha256`
|
||||
args := []any{string(encoded), types.MustTenantIDFromContext(ctx), ns.KnowledgeBase, nativeOrigin}
|
||||
if ns.Knowledge != "" {
|
||||
query += " AND c.knowledge_id=?"
|
||||
args = append(args, ns.Knowledge)
|
||||
}
|
||||
query += " GROUP BY seed.subject_id"
|
||||
if err = r.db.WithContext(ctx).Raw(query, args...).Scan(&matches).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
supported := map[int64][]string{}
|
||||
attributeChunks := map[int64]string{}
|
||||
verifiedChunks := map[int64][]string{}
|
||||
for _, match := range matches {
|
||||
supported[match.SubjectID] = compatibleNativeAttributeSnapshots(match.Snapshots)
|
||||
attributeChunks[match.SubjectID] = match.AttributeChunk
|
||||
var current []string
|
||||
if err = json.Unmarshal(match.Chunks, ¤t); err != nil {
|
||||
return err
|
||||
}
|
||||
verifiedChunks[match.SubjectID] = current
|
||||
}
|
||||
for i, node := range nodes {
|
||||
payload, native := payloads[node.ID]
|
||||
if !native {
|
||||
continue
|
||||
}
|
||||
if chunks != nil {
|
||||
chunks[node.ID] = verifiedChunks[node.ID]
|
||||
}
|
||||
attributes := []string{}
|
||||
entityType := "concept"
|
||||
if snapshot := supported[node.ID]; len(snapshot) > 0 {
|
||||
var stored []string
|
||||
if err := json.Unmarshal(payload["attributes"], &stored); err == nil && slices.Equal(stored, snapshot) {
|
||||
attributes = stored
|
||||
entityType = snapshot[0]
|
||||
}
|
||||
var storedType string
|
||||
if err := json.Unmarshal(payload["entity_type"], &storedType); err == nil && storedType == snapshot[0] {
|
||||
entityType = storedType
|
||||
}
|
||||
}
|
||||
if chunk := attributeChunks[node.ID]; chunks != nil && chunk != "" && (len(attributes) > 0 || entityType != "concept") {
|
||||
verified := []string{chunk}
|
||||
for _, prior := range chunks[node.ID] {
|
||||
if prior != chunk && len(verified) < 4 {
|
||||
verified = append(verified, prior)
|
||||
}
|
||||
}
|
||||
chunks[node.ID] = verified
|
||||
}
|
||||
payload["attributes"], _ = json.Marshal(attributes)
|
||||
payload["entity_type"], _ = json.Marshal(entityType)
|
||||
if nodes[i].Payload, err = json.Marshal(payload); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func compatibleNativeAttributeSnapshots(raw json.RawMessage) []string {
|
||||
var snapshots []json.RawMessage
|
||||
if err := json.Unmarshal(raw, &snapshots); err != nil {
|
||||
return nil
|
||||
}
|
||||
var attributes []string
|
||||
for _, snapshot := range snapshots {
|
||||
var values []*string
|
||||
if err := json.Unmarshal(snapshot, &values); err != nil {
|
||||
return nil
|
||||
}
|
||||
if len(values) == 0 {
|
||||
continue
|
||||
}
|
||||
current := make([]string, len(values))
|
||||
for i, value := range values {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
current[i] = *value
|
||||
}
|
||||
if len(attributes) > 0 && !slices.Equal(attributes, current) {
|
||||
return nil
|
||||
}
|
||||
attributes = current
|
||||
}
|
||||
return attributes
|
||||
}
|
||||
|
||||
func boundedTerms(terms []string) []string {
|
||||
out := []string{}
|
||||
seen := map[string]bool{}
|
||||
for _, term := range terms {
|
||||
term = strings.TrimSpace(term)
|
||||
if term == "" || seen[term] {
|
||||
continue
|
||||
}
|
||||
if len([]rune(term)) > 200 {
|
||||
term = string([]rune(term)[:200])
|
||||
}
|
||||
seen[term] = true
|
||||
out = append(out, term)
|
||||
if len(out) == 8 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package legacygraph
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestSearchNodePreservesStoredAttributes(t *testing.T) {
|
||||
for _, origin := range []string{"native", "legacy"} {
|
||||
for _, stored := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("%s/attributes=%t", origin, stored), func(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
graph := testGraph("chunk")
|
||||
graph.Node[0].Attributes = []string{"concept", "Not a medicine", "No treatment, nursing or health-care effect", "Evidence keeps spaces\nand line breaks", "concept"}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{graph}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if origin == "legacy" {
|
||||
if err := tx.Exec(`UPDATE native_graph_records SET payload=payload-'origin' WHERE source_table='knowledgeentity'`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.Exec(`UPDATE native_graph_sources SET mapping_state='exact_content_hash'`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !stored {
|
||||
if err := tx.Exec(`UPDATE native_graph_records SET payload=payload-'attributes' WHERE source_table='knowledgeentity'`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
id, _ := nativeID("doc", "entity", "trust")
|
||||
want := []string{"concept"}
|
||||
if stored {
|
||||
want = append(want, graph.Node[0].Attributes[1:4]...)
|
||||
}
|
||||
want = append(want, fmt.Sprintf("legacy_entity:%d", id))
|
||||
// The same record must preserve attributes as either a seed or a neighbor.
|
||||
for _, term := range []string{"Trust", "Learning"} {
|
||||
found, err := r.SearchNode(ctx, ns, []string{term})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got *types.GraphNode
|
||||
for _, node := range found.Node {
|
||||
if node.Name == "Trust" {
|
||||
got = node
|
||||
}
|
||||
}
|
||||
if got == nil || !reflect.DeepEqual(got.Attributes, want) || !reflect.DeepEqual(got.Chunks, []string{"chunk"}) {
|
||||
t.Fatalf("%s lost source attributes or compatibility metadata: got=%#v want=%#v", term, got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchNodeAttributesRespectSourceAvailabilityAndOwnership(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
graph := testGraph("chunk")
|
||||
graph.Node[0].Attributes = []string{"concept", "Not a medicine", "No treatment, nursing or health-care effect"}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{graph}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, scope := range []struct {
|
||||
name string
|
||||
tenant uint64
|
||||
ns types.NameSpace
|
||||
}{
|
||||
{"other tenant", 8, ns},
|
||||
{"other knowledge base", 7, types.NameSpace{KnowledgeBase: "other-kb"}},
|
||||
{"other document", 7, types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc2"}},
|
||||
} {
|
||||
t.Run(scope.name, func(t *testing.T) {
|
||||
otherCtx := context.WithValue(ctx, types.TenantIDContextKey, scope.tenant)
|
||||
got, err := r.SearchNode(otherCtx, scope.ns, []string{"Trust"})
|
||||
if err != nil || len(got.Node) != 0 || len(got.Relation) != 0 {
|
||||
t.Fatalf("out-of-scope attributes returned: %#v %v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, change := range []struct{ name, sql string }{
|
||||
{"edited source", `UPDATE chunks SET content=content || ' changed' WHERE id='chunk'`},
|
||||
{"disabled source", `UPDATE chunks SET is_enabled=false WHERE id='chunk'`},
|
||||
{"deleted source", `UPDATE chunks SET deleted_at=now() WHERE id='chunk'`},
|
||||
{"deleted document", `UPDATE knowledges SET deleted_at=now() WHERE id='doc' AND tenant_id=7`},
|
||||
{"deleted knowledge base", `UPDATE knowledge_bases SET deleted_at=now() WHERE id='kb' AND tenant_id=7`},
|
||||
} {
|
||||
t.Run(change.name, func(t *testing.T) {
|
||||
if err := tx.SavePoint("attribute_source_check").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.RollbackTo("attribute_source_check")
|
||||
if err := tx.Exec(change.sql).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := r.SearchNode(ctx, ns, []string{"Trust"})
|
||||
if err != nil || len(got.Node) != 0 || len(got.Relation) != 0 {
|
||||
t.Fatalf("unavailable source attributes returned: %#v %v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,450 @@
|
||||
package legacygraph
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestMissingGraphScope(t *testing.T) {
|
||||
r := NewRepository(nil, nil)
|
||||
if _, err := r.Browse(context.Background(), "kb", "", 0, 0); err == nil {
|
||||
t.Fatal("missing tenant accepted")
|
||||
}
|
||||
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(10059))
|
||||
if _, err := r.SearchNode(ctx, types.NameSpace{}, []string{"term"}); err == nil {
|
||||
t.Fatal("missing KB accepted")
|
||||
}
|
||||
}
|
||||
func TestBoundedTerms(t *testing.T) {
|
||||
terms := boundedTerms([]string{"", " ", "a", "a", strings.Repeat("中", 240), "b", "c", "d", "e", "f", "g", "h"})
|
||||
if len(terms) != 8 || terms[0] != "a" || len([]rune(terms[1])) != 200 {
|
||||
t.Fatalf("unbounded terms: %#v", terms)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchNodeKeepsGenericSourcesIsolated(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
if err := tx.Exec(`INSERT INTO chunks VALUES('sibling',7,'kb','doc','Evidence about trust and learning.',NULL,true)`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
for chunk, target := range map[string]string{"chunk": "First target", "sibling": "Second target"} {
|
||||
graph := &types.GraphData{
|
||||
Node: []*types.GraphNode{
|
||||
{Name: "本品", Chunks: []string{chunk}},
|
||||
{Name: target, Chunks: []string{chunk}},
|
||||
},
|
||||
Relation: []*types.GraphRelation{{Node1: "本品", Node2: target, Type: "关联"}},
|
||||
}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{graph}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for chunk, target := range map[string]string{"chunk": "First target", "sibling": "Second target"} {
|
||||
graph, err := r.SearchNode(ctx, ns, []string{target})
|
||||
if err != nil || len(graph.Node) != 2 || len(graph.Relation) != 1 {
|
||||
t.Fatalf("unexpected single-product result: %#v %v", graph, err)
|
||||
}
|
||||
for _, node := range graph.Node {
|
||||
if len(node.Chunks) != 1 || node.Chunks[0] != chunk {
|
||||
t.Fatalf("same-name neighbor expanded another product: %#v", node)
|
||||
}
|
||||
}
|
||||
if edge := graph.Relation[0]; edge.Node1 != "本品" || edge.Node2 != target {
|
||||
t.Fatalf("unexpected relation projection: %#v", edge)
|
||||
}
|
||||
}
|
||||
graph, err := r.SearchNode(ctx, ns, []string{"本品"})
|
||||
if err != nil || len(graph.Node) != 4 || len(graph.Relation) != 2 {
|
||||
t.Fatalf("same-name nodes were collapsed: %#v %v", graph, err)
|
||||
}
|
||||
sources := map[string]bool{}
|
||||
identities := map[string]bool{}
|
||||
for _, node := range graph.Node {
|
||||
if node.Name != "本品" {
|
||||
continue
|
||||
}
|
||||
if len(node.Chunks) != 1 || len(node.Attributes) != 2 {
|
||||
t.Fatalf("generic source identity was lost: %#v", node)
|
||||
}
|
||||
sources[node.Chunks[0]] = true
|
||||
identities[node.Attributes[1]] = true
|
||||
}
|
||||
if len(sources) != 2 || !sources["chunk"] || !sources["sibling"] || len(identities) != 2 {
|
||||
t.Fatalf("generic products share retrieval identity: sources=%v identities=%v", sources, identities)
|
||||
}
|
||||
}
|
||||
|
||||
func assertReaderTrustAttributes(t *testing.T, r *Repository, ctx context.Context, want []string) {
|
||||
t.Helper()
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
id, _ := nativeID("doc", "entity", "trust")
|
||||
wantType := "concept"
|
||||
if len(want) > 0 {
|
||||
wantType = want[0]
|
||||
}
|
||||
wantGraph := append([]string{wantType}, want...)
|
||||
if len(want) > 0 {
|
||||
wantGraph = wantGraph[1:]
|
||||
}
|
||||
wantGraph = append(wantGraph, fmt.Sprintf("legacy_entity:%d", id))
|
||||
for _, term := range []string{"Trust", "Learning"} {
|
||||
graph, err := r.SearchNode(ctx, ns, []string{term})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var found *types.GraphNode
|
||||
for _, node := range graph.Node {
|
||||
if node.Name == "Trust" {
|
||||
found = node
|
||||
}
|
||||
}
|
||||
if found == nil || !slices.Equal(found.Attributes, wantGraph) {
|
||||
t.Fatalf("%s returned unsupported native attributes: got=%#v want=%v", term, found, wantGraph)
|
||||
}
|
||||
}
|
||||
for _, entityID := range []int64{0, id} {
|
||||
page, err := r.Browse(ctx, "kb", "", 0, entityID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, node := range page.Nodes {
|
||||
if node.ID != id {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
var payload struct {
|
||||
Attributes []string `json:"attributes"`
|
||||
Type string `json:"entity_type"`
|
||||
}
|
||||
if err := json.Unmarshal(node.Payload, &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(payload.Attributes, want) || payload.Type != wantType {
|
||||
t.Fatalf("browse returned unsupported native attributes: %#v; want %v / %s", payload, want, wantType)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("browse lost the entity's surviving bare reference")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeReadersRevalidateSharedAttributeSources(t *testing.T) {
|
||||
for name, invalidate := range map[string]string{
|
||||
"changed source": `UPDATE chunks SET content=content || ' changed' WHERE id='chunk' AND tenant_id=7`,
|
||||
"disabled source": `UPDATE chunks SET is_enabled=false WHERE id='chunk' AND tenant_id=7`,
|
||||
"deleted source": `UPDATE chunks SET deleted_at=now() WHERE id='chunk' AND tenant_id=7`,
|
||||
"moved source document": `UPDATE chunks SET knowledge_id='doc2' WHERE id='chunk' AND tenant_id=7`,
|
||||
"wrong evidence hash": `UPDATE native_graph_records SET payload=jsonb_set(payload,'{source_content_hash}',to_jsonb(repeat('a',64))) WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"wrong row hash": `UPDATE native_graph_records SET source_snapshot_sha256=repeat('a',64) WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"inactive evidence": `UPDATE native_graph_records SET payload=jsonb_set(payload,'{status}','"inactive"'::jsonb) WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"wrong origin": `UPDATE native_graph_records SET payload=jsonb_set(payload,'{origin}','"foreign"'::jsonb) WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"wrong tenant": `UPDATE native_graph_records SET tenant_id=8 WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"wrong knowledge base": `UPDATE native_graph_records SET knowledge_base_id='other-kb' WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"wrong document": `UPDATE native_graph_records SET payload=jsonb_set(payload,'{native_knowledge_id}','"doc2"'::jsonb) WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"wrong mapping document": `UPDATE native_graph_sources SET native_knowledge_id='doc2' WHERE source_chunk_id='native:chunk'`,
|
||||
"wrong mapping hash": `UPDATE native_graph_sources SET source_content_hash=repeat('a',64) WHERE source_chunk_id='native:chunk'`,
|
||||
"inactive mapping": `UPDATE native_graph_sources SET mapping_state='stale' WHERE source_chunk_id='native:chunk'`,
|
||||
"missing old snapshot": `UPDATE native_graph_records SET payload=payload-'entity_attributes' WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"null snapshot": `UPDATE native_graph_records SET payload=jsonb_set(payload,'{entity_attributes}','null'::jsonb) WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk' AND payload->>'subject_type'='entity'`,
|
||||
"null attribute": `UPDATE native_graph_records SET payload=jsonb_set(payload,'{entity_attributes}','[null]'::jsonb) WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk' AND payload->>'subject_type'='entity'`,
|
||||
"conflicting snapshot": `UPDATE native_graph_records SET payload=jsonb_set(payload,'{entity_attributes}','["contradiction"]'::jsonb) WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:sibling' AND payload->>'subject_type'='entity'`,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
if err := tx.Exec(`INSERT INTO chunks VALUES('sibling',7,'kb','doc','Another source about trust and learning.',NULL,true)`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
owner, bare := testGraph("chunk"), testGraph("sibling")
|
||||
owner.Node[0].Attributes = []string{"restriction", "Cannot be taken orally"}
|
||||
bare.Node[0].Attributes = nil
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{owner, bare}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertReaderTrustAttributes(t, r, ctx, owner.Node[0].Attributes)
|
||||
if err := tx.Exec(invalidate).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := nativeGraphWriteState(t, tx)
|
||||
assertReaderTrustAttributes(t, r, ctx, nil)
|
||||
if after := nativeGraphWriteState(t, tx); after != before {
|
||||
t.Fatal("reader projection changed stored graph records or mappings")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeReadersKeepVerifiedAttributesBeyondChunkCap(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
if err := tx.Exec(`INSERT INTO chunks SELECT id,7,'kb','doc','Another source about trust and learning.',NULL,true FROM unnest(ARRAY['aa','ab','ac','ad']) id`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
owner := testGraph("chunk")
|
||||
owner.Node[0].Attributes = []string{"restriction", "Cannot be taken orally"}
|
||||
graphs := []*types.GraphData{owner}
|
||||
for _, id := range []string{"aa", "ab", "ac", "ad"} {
|
||||
bare := testGraph(id)
|
||||
bare.Node[0].Attributes = nil
|
||||
graphs = append(graphs, bare)
|
||||
}
|
||||
if err := r.AddGraph(ctx, ns, graphs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An old bare source cannot erase attributes proved by an explicit source.
|
||||
if err := tx.Exec(`UPDATE native_graph_records SET payload=payload-'entity_attributes' WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:aa'`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertReaderTrustAttributes(t, r, ctx, owner.Node[0].Attributes)
|
||||
graph, err := r.SearchNode(ctx, ns, []string{"Trust"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range graph.Node {
|
||||
if len(node.Chunks) != 4 || (node.Name == "Trust" && !slices.Contains(node.Chunks, "chunk")) {
|
||||
t.Fatalf("attribute owner was omitted from the bounded references: %#v", node)
|
||||
}
|
||||
}
|
||||
if err := tx.Exec(`UPDATE chunks SET is_enabled=false WHERE id='chunk' AND tenant_id=7`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertReaderTrustAttributes(t, r, ctx, nil)
|
||||
}
|
||||
|
||||
func TestNativeReadersRespectCopiedSourceMappings(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
graph := testGraph("chunk")
|
||||
graph.Node[0].Attributes = []string{"restriction", "Cannot be taken orally"}
|
||||
if err := r.AddGraph(ctx, types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}, []*types.GraphData{graph}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, sql := range []string{
|
||||
`INSERT INTO knowledges VALUES('copy-doc',7,'other-kb','copy.md',NULL)`,
|
||||
`INSERT INTO chunks VALUES('copy-chunk',7,'other-kb','copy-doc','Evidence about trust and learning.',NULL,true)`,
|
||||
`INSERT INTO native_graph_records SELECT tenant_id,'other-kb',source_table,legacy_id,graph_version_id,payload,source_snapshot_sha256,imported_at FROM native_graph_records WHERE tenant_id=7 AND knowledge_base_id='kb'`,
|
||||
`INSERT INTO native_graph_sources SELECT tenant_id,'other-kb',source_chunk_id,source_file,source_content_hash,excerpt,'copy-doc','copy-chunk',native_content_sha256,mapping_state FROM native_graph_sources WHERE tenant_id=7 AND knowledge_base_id='kb'`,
|
||||
`UPDATE chunks SET is_enabled=false WHERE id='chunk' AND tenant_id=7`,
|
||||
} {
|
||||
if err := tx.Exec(sql).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ns := types.NameSpace{KnowledgeBase: "other-kb", Knowledge: "copy-doc"}
|
||||
id, _ := nativeID("doc", "entity", "trust")
|
||||
found, err := r.SearchNode(ctx, ns, []string{"Trust"})
|
||||
if err != nil || len(found.Node) != 2 {
|
||||
t.Fatalf("copied native graph was not retrieved: %#v %v", found, err)
|
||||
}
|
||||
for _, node := range found.Node {
|
||||
if !slices.Equal(node.Chunks, []string{"copy-chunk"}) {
|
||||
t.Fatalf("copied graph retained an original chunk reference: %#v", node)
|
||||
}
|
||||
if node.Name == "Trust" && !slices.Equal(node.Attributes, []string{"restriction", "Cannot be taken orally", fmt.Sprintf("legacy_entity:%d", id)}) {
|
||||
t.Fatalf("copied source snapshot lost its attributes: %#v", node)
|
||||
}
|
||||
}
|
||||
wrong, err := r.SearchNode(ctx, types.NameSpace{KnowledgeBase: "other-kb", Knowledge: "doc"}, []string{"Trust"})
|
||||
if err != nil || len(wrong.Node) != 0 {
|
||||
t.Fatalf("copied graph leaked through the original document scope: %#v %v", wrong, err)
|
||||
}
|
||||
page, err := r.Browse(ctx, "other-kb", "", 0, id)
|
||||
if err != nil || len(page.Nodes) != 1 {
|
||||
t.Fatalf("copied entity was not browsable: %#v %v", page, err)
|
||||
}
|
||||
var payload struct {
|
||||
Attributes []string `json:"attributes"`
|
||||
}
|
||||
if err := json.Unmarshal(page.Nodes[0].Payload, &payload); err != nil || !slices.Equal(payload.Attributes, graph.Node[0].Attributes) {
|
||||
t.Fatalf("copied browse attributes were not revalidated: %#v %v", payload, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Fixture mutations are confined to a rolled-back transaction.
|
||||
func TestAcceptancePreservedGraph(t *testing.T) {
|
||||
dsn := os.Getenv("GRAPH_TEST_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("GRAPH_TEST_DSN is required for acceptance integration")
|
||||
}
|
||||
db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tx := db.Begin()
|
||||
defer tx.Rollback()
|
||||
if tx.Error != nil {
|
||||
t.Fatal(tx.Error)
|
||||
}
|
||||
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(10059))
|
||||
r := NewRepository(tx, nil)
|
||||
var fixture struct {
|
||||
KB string
|
||||
Entity int64
|
||||
Name string
|
||||
Chunk string
|
||||
Knowledge string
|
||||
}
|
||||
err = tx.Raw(`SELECT r.knowledge_base_id AS kb,r.legacy_id AS entity,r.payload->>'canonical_name' AS name,s.native_chunk_id AS chunk,s.native_knowledge_id AS knowledge FROM native_graph_records r JOIN native_graph_records e ON e.tenant_id=r.tenant_id AND e.knowledge_base_id=r.knowledge_base_id AND e.source_table='knowledgegraphevidence' AND e.payload->>'subject_type'='entity' AND e.payload->>'subject_id'=r.legacy_id::text JOIN native_graph_sources s ON s.tenant_id=e.tenant_id AND s.knowledge_base_id=e.knowledge_base_id AND s.source_chunk_id=e.payload->>'source_chunk_id' WHERE r.tenant_id=10059 AND r.source_table='knowledgeentity' AND r.knowledge_base_id='16c3e59c-4bb6-41cf-ae0a-6d0941b3d032' AND s.native_chunk_id IS NOT NULL ORDER BY r.legacy_id LIMIT 1`).Scan(&fixture).Error
|
||||
if err != nil || fixture.Chunk == "" {
|
||||
t.Fatalf("missing mapped fixture: %v", err)
|
||||
}
|
||||
overview, err := r.Browse(ctx, fixture.KB, "", 0, fixture.Entity)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var preservedEntities, preservedMappings int64
|
||||
if err := tx.Raw(`SELECT count(*) FROM native_graph_records WHERE tenant_id=10059 AND knowledge_base_id=? AND source_table='knowledgeentity' AND payload->>'origin' IS DISTINCT FROM 'native_extraction_v1'`, fixture.KB).Scan(&preservedEntities).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.Raw(`SELECT count(*) FROM native_graph_sources WHERE tenant_id=10059 AND knowledge_base_id=? AND mapping_state<>'native_extraction' AND native_chunk_id IS NOT NULL`, fixture.KB).Scan(&preservedMappings).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(overview.Nodes) != 1 || len(overview.Evidence) == 0 || preservedEntities != 2 || preservedMappings != 3 || overview.Counts["knowledgeentity"] < preservedEntities || overview.MappedSources < preservedMappings {
|
||||
t.Fatalf("preserved graph changed: entities=%d mappings=%d counts=%v", preservedEntities, preservedMappings, overview.Counts)
|
||||
}
|
||||
found := false
|
||||
for _, e := range overview.Evidence {
|
||||
if e.NativeChunkID == fixture.Chunk {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("mapped evidence was not returned")
|
||||
}
|
||||
graph, err := r.SearchNode(ctx, types.NameSpace{KnowledgeBase: fixture.KB, Knowledge: fixture.Knowledge}, []string{fixture.Name})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(graph.Node) == 0 {
|
||||
t.Fatal("native graph retrieval produced no nodes")
|
||||
}
|
||||
for _, n := range graph.Node {
|
||||
for _, c := range n.Chunks {
|
||||
if strings.HasPrefix(c, "chunk-") {
|
||||
t.Fatal("legacy chunk ID leaked into native retrieval")
|
||||
}
|
||||
}
|
||||
}
|
||||
other := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(999999999))
|
||||
denied, err := r.Browse(other, fixture.KB, "", 0, fixture.Entity)
|
||||
if err != nil || len(denied.Nodes) != 0 || len(denied.Evidence) != 0 {
|
||||
t.Fatal("cross-tenant graph leak", err)
|
||||
}
|
||||
wrongFile, err := r.SearchNode(ctx, types.NameSpace{KnowledgeBase: fixture.KB, Knowledge: "other-file"}, []string{fixture.Name})
|
||||
if err != nil || len(wrongFile.Node) != 0 {
|
||||
t.Fatal("file scope was ignored", err)
|
||||
}
|
||||
for _, change := range []struct {
|
||||
name, sql, id string
|
||||
}{
|
||||
{"edited chunk", "UPDATE chunks SET content=content || ' stale-check' WHERE id=? AND tenant_id=10059", fixture.Chunk},
|
||||
{"disabled chunk", "UPDATE chunks SET is_enabled=false WHERE id=? AND tenant_id=10059", fixture.Chunk},
|
||||
{"deleted chunk", "UPDATE chunks SET deleted_at=now() WHERE id=? AND tenant_id=10059", fixture.Chunk},
|
||||
{"deleted document", "UPDATE knowledges SET deleted_at=now() WHERE id=? AND tenant_id=10059", fixture.Knowledge},
|
||||
{"deleted knowledge base", "UPDATE knowledge_bases SET deleted_at=now() WHERE id=? AND tenant_id=10059", fixture.KB},
|
||||
} {
|
||||
t.Run(change.name, func(t *testing.T) {
|
||||
if err := tx.SavePoint("resource_check").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.RollbackTo("resource_check")
|
||||
if err := tx.Exec(change.sql, change.id).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stale, err := r.Browse(ctx, fixture.KB, "", 0, fixture.Entity)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range stale.Evidence {
|
||||
if e.NativeChunkID == fixture.Chunk {
|
||||
t.Fatal("unavailable chunk remained linked")
|
||||
}
|
||||
}
|
||||
staleGraph, err := r.SearchNode(ctx, types.NameSpace{KnowledgeBase: fixture.KB}, []string{fixture.Name})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range staleGraph.Node {
|
||||
for _, c := range n.Chunks {
|
||||
if c == fixture.Chunk {
|
||||
t.Fatal("unavailable chunk leaked into retrieval")
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptanceLargestGraph(t *testing.T) {
|
||||
dsn := os.Getenv("GRAPH_TEST_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("GRAPH_TEST_DSN is required for acceptance integration")
|
||||
}
|
||||
db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(10059))
|
||||
kb := "bb86b44a-576b-4bf8-a6bf-8f1baca3a845"
|
||||
r := NewRepository(db, nil)
|
||||
started := time.Now()
|
||||
overview, err := r.Browse(ctx, kb, "", 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var preserved struct{ Sources, Mapped int64 }
|
||||
if err := db.Raw(`SELECT count(*) AS sources,count(native_chunk_id) AS mapped FROM native_graph_sources WHERE tenant_id=10059 AND knowledge_base_id=? AND mapping_state<>'native_extraction'`, kb).Scan(&preserved).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(overview.Nodes) != 50 || preserved.Sources != 66204 || preserved.Mapped != 7000 || overview.Sources < preserved.Sources || overview.MappedSources < preserved.Mapped {
|
||||
t.Fatalf("unexpected largest graph overview: preserved=%+v sources=%d mapped=%d", preserved, overview.Sources, overview.MappedSources)
|
||||
}
|
||||
t.Logf("largest KB overview: %s", time.Since(started))
|
||||
started = time.Now()
|
||||
graph, err := r.SearchNode(ctx, types.NameSpace{KnowledgeBase: kb}, []string{"销售", "客户", "信任", "心法"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(graph.Node) == 0 || len(graph.Node) > 32 || len(graph.Relation) > 32 {
|
||||
t.Fatalf("invalid bounded retrieval: %d nodes, %d relations", len(graph.Node), len(graph.Relation))
|
||||
}
|
||||
for _, node := range graph.Node {
|
||||
if len(node.Chunks) == 0 || len(node.Chunks) > 4 {
|
||||
t.Fatalf("invalid chunk cap: %#v", node)
|
||||
}
|
||||
}
|
||||
t.Logf("largest KB graph retrieval: %s; %d nodes, %d relations", time.Since(started), len(graph.Node), len(graph.Relation))
|
||||
|
||||
var relationFixture struct {
|
||||
KB, From, To, Relation string
|
||||
}
|
||||
err = db.Raw(`SELECT r.knowledge_base_id AS kb,a.payload->>'canonical_name' AS "from",b.payload->>'canonical_name' AS "to",r.payload->>'relation_type' AS relation FROM native_graph_records r JOIN native_graph_records a ON a.tenant_id=r.tenant_id AND a.knowledge_base_id=r.knowledge_base_id AND a.source_table='knowledgeentity' AND a.legacy_id=(r.payload->>'from_entity_id')::bigint JOIN native_graph_records b ON b.tenant_id=r.tenant_id AND b.knowledge_base_id=r.knowledge_base_id AND b.source_table='knowledgeentity' AND b.legacy_id=(r.payload->>'to_entity_id')::bigint WHERE r.tenant_id=10059 AND r.source_table='knowledgerelation' AND r.legacy_id=1`).Scan(&relationFixture).Error
|
||||
if err != nil || relationFixture.KB == "" {
|
||||
t.Fatalf("missing relation fixture: %v", err)
|
||||
}
|
||||
expanded, err := r.SearchNode(ctx, types.NameSpace{KnowledgeBase: relationFixture.KB}, []string{relationFixture.From})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, relation := range expanded.Relation {
|
||||
if relation.Node1 == relationFixture.From && relation.Node2 == relationFixture.To && relation.Type == relationFixture.Relation {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("verified relation neighbor was not expanded")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,522 @@
|
||||
package legacygraph
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const nativeOrigin = "native_extraction_v1"
|
||||
|
||||
type sourceChunk struct {
|
||||
ID, KnowledgeID, KnowledgeBaseID, Content, FileName string
|
||||
}
|
||||
|
||||
func contentHash(content string) string {
|
||||
h := sha256.Sum256([]byte(content))
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
// IDs stay exactly representable in the browser, outside historical IDs. The
|
||||
// full identity is also checked on conflict, so a hash collision fails closed.
|
||||
func nativeID(parts ...string) (int64, string) {
|
||||
identity, _ := json.Marshal(parts)
|
||||
h := sha256.Sum256(identity)
|
||||
return int64(binary.BigEndian.Uint64(h[:8])&((1<<52)-1)) | 1<<52, hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
func normalizedName(name string) string { return strings.ToLower(strings.TrimSpace(name)) }
|
||||
|
||||
func sourceScopedEntity(name string) bool {
|
||||
name = strings.Map(func(r rune) rune {
|
||||
if unicode.IsSpace(r) {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, normalizedName(name))
|
||||
switch name {
|
||||
case "本品", "该产品", "本产品", "产品", "本商品", "此产品", "本公司", "该公司", "本文", "本方法", "该方法", "thisproduct", "theproduct":
|
||||
return true
|
||||
default:
|
||||
// The extractor uses these names to separate unresolved local products.
|
||||
return strings.HasSuffix(name, "未名产品") || strings.HasPrefix(name, "本品(")
|
||||
}
|
||||
}
|
||||
|
||||
func nativeEntityIdentity(document string, node *types.GraphNode) (int64, string, error) {
|
||||
key := normalizedName(node.Name)
|
||||
parts := []string{document, "entity", key}
|
||||
if sourceScopedEntity(key) {
|
||||
// An unnamed referent is not a document-wide entity. Multiple sources
|
||||
// require the extractor to supply an unambiguous, specifically named node.
|
||||
if len(node.Chunks) == 0 || node.Chunks[0] == "" {
|
||||
return 0, "", fmt.Errorf("generic graph entity requires one source chunk")
|
||||
}
|
||||
for _, chunk := range node.Chunks {
|
||||
if chunk != node.Chunks[0] {
|
||||
return 0, "", fmt.Errorf("generic graph entity cannot merge distinct source chunks")
|
||||
}
|
||||
}
|
||||
parts = append(parts, "source", node.Chunks[0])
|
||||
}
|
||||
id, identity := nativeID(parts...)
|
||||
return id, identity, nil
|
||||
}
|
||||
|
||||
// HasChunkGraph accepts only current checkpoints with extracted relations.
|
||||
// Legacy empty results and excerpt mappings require coverage review.
|
||||
func (r *Repository) HasChunkGraph(ctx context.Context, chunk *types.Chunk) (bool, error) {
|
||||
tenant, ok := types.TenantIDFromContext(ctx)
|
||||
if !ok || tenant == 0 || chunk == nil || chunk.TenantID != tenant || r.db == nil {
|
||||
return false, fmt.Errorf("graph checkpoint requires source tenant and chunk")
|
||||
}
|
||||
var count int64
|
||||
err := r.db.WithContext(ctx).Table("native_graph_records").Where("tenant_id=? AND knowledge_base_id=? AND source_table='knowledgegraphchunkstate' AND payload->>'origin'=? AND payload->>'native_chunk_id'=? AND payload->>'native_knowledge_id'=? AND payload->>'source_content_hash'=? AND payload->>'extraction_status'='complete' AND CAST(COALESCE(payload->>'relation_count','0') AS INTEGER)>0", tenant, chunk.KnowledgeBaseID, nativeOrigin, chunk.ID, chunk.KnowledgeID, contentHash(chunk.Content)).Count(&count).Error
|
||||
return count > 0, err
|
||||
}
|
||||
|
||||
// AddChunkGraph rejects unreviewed empty coverage and results from changed sources.
|
||||
func (r *Repository) AddChunkGraph(ctx context.Context, ns types.NameSpace, chunk *types.Chunk, graph *types.GraphData) error {
|
||||
if chunk == nil || chunk.ID == "" || graph == nil {
|
||||
return fmt.Errorf("graph extraction requires its source chunk and model result")
|
||||
}
|
||||
if len(graph.Relation) == 0 {
|
||||
return fmt.Errorf("graph extraction without relations requires coverage review")
|
||||
}
|
||||
return r.addNativeGraphs(ctx, ns, []*types.GraphData{graph}, map[string]string{chunk.ID: contentHash(chunk.Content)})
|
||||
}
|
||||
|
||||
func (r *Repository) addNativeGraphs(ctx context.Context, ns types.NameSpace, graphs []*types.GraphData, expected map[string]string) error {
|
||||
tenant, ok := types.TenantIDFromContext(ctx)
|
||||
if !ok || tenant == 0 || ns.KnowledgeBase == "" || ns.Knowledge == "" || r.db == nil {
|
||||
return fmt.Errorf("graph writes require tenant, knowledge base, document and database")
|
||||
}
|
||||
ids := map[string]bool{}
|
||||
for id := range expected {
|
||||
ids[id] = true
|
||||
}
|
||||
for _, graph := range graphs {
|
||||
if graph == nil {
|
||||
return fmt.Errorf("nil graph extraction")
|
||||
}
|
||||
for _, node := range graph.Node {
|
||||
if node == nil || normalizedName(node.Name) == "" || len(node.Chunks) == 0 {
|
||||
return fmt.Errorf("extracted graph node requires name and source chunks")
|
||||
}
|
||||
for _, id := range node.Chunks {
|
||||
if id == "" || (expected != nil && expected[id] == "") {
|
||||
return fmt.Errorf("graph node refers to a chunk outside extraction scope")
|
||||
}
|
||||
ids[id] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return fmt.Errorf("graph extraction has no source chunks")
|
||||
}
|
||||
chunkIDs := make([]string, 0, len(ids))
|
||||
for id := range ids {
|
||||
chunkIDs = append(chunkIDs, id)
|
||||
}
|
||||
sort.Strings(chunkIDs)
|
||||
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if err := lockGraph(tx, tenant, ns.KnowledgeBase); err != nil {
|
||||
return err
|
||||
}
|
||||
var chunks []sourceChunk
|
||||
if err := tx.Raw(`SELECT c.id,c.knowledge_id,c.knowledge_base_id,c.content,k.file_name FROM chunks c
|
||||
JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id AND k.deleted_at IS NULL
|
||||
JOIN knowledge_bases kb ON kb.id=c.knowledge_base_id AND kb.tenant_id=c.tenant_id AND kb.deleted_at IS NULL
|
||||
WHERE c.tenant_id=? AND c.knowledge_base_id=? AND c.knowledge_id=? AND c.id IN ? AND c.deleted_at IS NULL AND c.is_enabled=true
|
||||
ORDER BY c.id FOR SHARE OF c,k,kb`, tenant, ns.KnowledgeBase, ns.Knowledge, chunkIDs).Scan(&chunks).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if len(chunks) != len(chunkIDs) {
|
||||
return fmt.Errorf("graph source chunks are unavailable or outside document/tenant scope")
|
||||
}
|
||||
byID := map[string]sourceChunk{}
|
||||
for _, chunk := range chunks {
|
||||
if expected != nil && expected[chunk.ID] != contentHash(chunk.Content) {
|
||||
return fmt.Errorf("graph source chunk %s changed during extraction", chunk.ID)
|
||||
}
|
||||
byID[chunk.ID] = chunk
|
||||
}
|
||||
// Replace only this extraction's evidence. Other documents, chunks and
|
||||
// archived migration records are untouched, including on retries.
|
||||
sources := make([]string, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
source := "native:" + chunk.ID
|
||||
sources = append(sources, source)
|
||||
excerpt := []rune(chunk.Content)
|
||||
if len(excerpt) > 2000 {
|
||||
excerpt = excerpt[:2000]
|
||||
}
|
||||
if err := tx.Exec(`INSERT INTO native_graph_sources(tenant_id,knowledge_base_id,source_chunk_id,source_file,source_content_hash,excerpt,native_knowledge_id,native_chunk_id,native_content_sha256,mapping_state)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,'native_extraction') ON CONFLICT(tenant_id,knowledge_base_id,source_chunk_id) DO UPDATE SET source_file=excluded.source_file,source_content_hash=excluded.source_content_hash,excerpt=excluded.excerpt,native_knowledge_id=excluded.native_knowledge_id,native_chunk_id=excluded.native_chunk_id,native_content_sha256=excluded.native_content_sha256,mapping_state=excluded.mapping_state`, tenant, ns.KnowledgeBase, source, chunk.FileName, contentHash(chunk.Content), string(excerpt), ns.Knowledge, chunk.ID, contentHash(chunk.Content)).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
type pendingEntity struct {
|
||||
identity string
|
||||
payload map[string]any
|
||||
}
|
||||
entities := map[int64]pendingEntity{}
|
||||
// Removed nodes also lose their source's attributes, even when a bare
|
||||
// reference from another chunk keeps the entity alive.
|
||||
var replacedEntities []Record
|
||||
if err := tx.Raw(`SELECT r.legacy_id,r.payload FROM native_graph_records r
|
||||
WHERE r.tenant_id=? AND r.knowledge_base_id=? AND r.source_table='knowledgeentity'
|
||||
AND r.payload->>'origin'=? AND r.payload->>'native_knowledge_id'=?
|
||||
AND EXISTS (SELECT 1 FROM native_graph_records e WHERE e.tenant_id=r.tenant_id AND e.knowledge_base_id=r.knowledge_base_id
|
||||
AND e.source_table='knowledgegraphevidence' AND e.payload->>'origin'=?
|
||||
AND e.payload->>'subject_type'='entity' AND e.payload->>'subject_id'=r.legacy_id::text AND e.payload->>'source_chunk_id' IN ?)`, tenant, ns.KnowledgeBase, nativeOrigin, ns.Knowledge, nativeOrigin, sources).Scan(&replacedEntities).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, record := range replacedEntities {
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(record.Payload, &payload); err != nil {
|
||||
return err
|
||||
}
|
||||
identity, _ := payload["native_identity"].(string)
|
||||
entities[record.ID] = pendingEntity{identity, payload}
|
||||
}
|
||||
if err := tx.Exec(`DELETE FROM native_graph_records WHERE tenant_id=? AND knowledge_base_id=? AND source_table='knowledgegraphevidence' AND payload->>'origin'=? AND payload->>'source_chunk_id' IN ?`, tenant, ns.KnowledgeBase, nativeOrigin, sources).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
counts := map[string]map[string]int{}
|
||||
entitySources := map[int64]map[string][]string{}
|
||||
for _, id := range chunkIDs {
|
||||
counts[id] = map[string]int{"entity": 0, "relation": 0}
|
||||
}
|
||||
for _, graph := range graphs {
|
||||
nodes := map[string]*types.GraphNode{}
|
||||
entityIDs := map[string]int64{}
|
||||
for _, node := range graph.Node {
|
||||
key := normalizedName(node.Name)
|
||||
id, identity, err := nativeEntityIdentity(ns.Knowledge, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if priorID, exists := entityIDs[key]; exists && priorID != id {
|
||||
return fmt.Errorf("generic graph entity is ambiguous within extraction")
|
||||
}
|
||||
name := strings.TrimSpace(node.Name)
|
||||
payload := map[string]any{"canonical_name": name, "entity_type": "concept", "attributes": node.Attributes}
|
||||
if len(node.Attributes) > 0 {
|
||||
payload["entity_type"] = node.Attributes[0]
|
||||
}
|
||||
if prior, exists := entities[id]; exists && prior.identity != identity {
|
||||
return fmt.Errorf("native graph identity collision; preserved graph was not modified")
|
||||
}
|
||||
if _, exists := entitySources[id]; !exists {
|
||||
entities[id] = pendingEntity{identity, payload}
|
||||
entitySources[id] = map[string][]string{}
|
||||
} else if name < entities[id].payload["canonical_name"].(string) {
|
||||
entities[id].payload["canonical_name"] = name
|
||||
}
|
||||
if prior := nodes[key]; prior != nil {
|
||||
prior.Chunks = append(prior.Chunks, node.Chunks...)
|
||||
} else {
|
||||
copyNode := *node
|
||||
copyNode.Chunks = append([]string(nil), node.Chunks...)
|
||||
nodes[key] = ©Node
|
||||
}
|
||||
entityIDs[key] = id
|
||||
seen := map[string]bool{}
|
||||
for _, chunkID := range node.Chunks {
|
||||
if seen[chunkID] {
|
||||
continue
|
||||
}
|
||||
seen[chunkID] = true
|
||||
if attributes, exists := entitySources[id][chunkID]; exists {
|
||||
if !slices.Equal(attributes, node.Attributes) {
|
||||
return fmt.Errorf("duplicate native graph entity source attributes conflict; extraction requires review")
|
||||
}
|
||||
continue
|
||||
}
|
||||
entitySources[id][chunkID] = node.Attributes
|
||||
if err := putEvidence(tx, tenant, ns, "entity", id, byID[chunkID], "", node.Attributes); err != nil {
|
||||
return err
|
||||
}
|
||||
counts[chunkID]["entity"]++
|
||||
}
|
||||
}
|
||||
for _, rel := range graph.Relation {
|
||||
if rel == nil || strings.TrimSpace(rel.Type) == "" {
|
||||
return fmt.Errorf("invalid extracted relation")
|
||||
}
|
||||
from, to := normalizedName(rel.Node1), normalizedName(rel.Node2)
|
||||
if nodes[from] == nil || nodes[to] == nil {
|
||||
return fmt.Errorf("relation references an entity outside extraction")
|
||||
}
|
||||
shared := []string{}
|
||||
fromChunks := map[string]bool{}
|
||||
for _, id := range nodes[from].Chunks {
|
||||
fromChunks[id] = true
|
||||
}
|
||||
for _, id := range nodes[to].Chunks {
|
||||
if fromChunks[id] {
|
||||
shared = append(shared, id)
|
||||
delete(fromChunks, id)
|
||||
}
|
||||
}
|
||||
if len(shared) == 0 {
|
||||
return fmt.Errorf("relation has no shared source evidence")
|
||||
}
|
||||
if rel.Evidence != "" {
|
||||
grounded := shared[:0]
|
||||
for _, chunkID := range shared {
|
||||
if strings.Contains(byID[chunkID].Content, rel.Evidence) {
|
||||
grounded = append(grounded, chunkID)
|
||||
}
|
||||
}
|
||||
shared = grounded
|
||||
if len(shared) == 0 {
|
||||
return fmt.Errorf("relation quotation is absent from current source evidence")
|
||||
}
|
||||
}
|
||||
identityParts := []string{ns.Knowledge, "relation", from, to, normalizedName(rel.Type)}
|
||||
if sourceScopedEntity(from) || sourceScopedEntity(to) {
|
||||
identityParts = append(identityParts, "scoped_endpoints", fmt.Sprint(entityIDs[from]), fmt.Sprint(entityIDs[to]))
|
||||
}
|
||||
id, identity := nativeID(identityParts...)
|
||||
if err := putNativeRecord(tx, tenant, ns, "knowledgerelation", id, identity, map[string]any{"from_entity_id": entityIDs[from], "to_entity_id": entityIDs[to], "relation_type": strings.TrimSpace(rel.Type), "evidence": rel.Evidence}, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, chunkID := range shared {
|
||||
if err := putEvidence(tx, tenant, ns, "relation", id, byID[chunkID], rel.Evidence); err != nil {
|
||||
return err
|
||||
}
|
||||
counts[chunkID]["relation"]++
|
||||
}
|
||||
}
|
||||
}
|
||||
// Reconcile after every source snapshot is present, so batch ordering
|
||||
// cannot decide which source owns the entity's retained attributes.
|
||||
entityIDs := make([]int64, 0, len(entities))
|
||||
for id := range entities {
|
||||
entityIDs = append(entityIDs, id)
|
||||
}
|
||||
slices.Sort(entityIDs)
|
||||
for _, id := range entityIDs {
|
||||
entity := entities[id]
|
||||
if err := putNativeRecord(tx, tenant, ns, "knowledgeentity", id, entity.identity, entity.payload, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, chunk := range chunks {
|
||||
id, identity := nativeID(ns.Knowledge, "chunkstate", chunk.ID)
|
||||
status := "complete"
|
||||
if counts[chunk.ID]["relation"] == 0 {
|
||||
status = "needs_review"
|
||||
}
|
||||
if err := putNativeRecord(tx, tenant, ns, "knowledgegraphchunkstate", id, identity, map[string]any{"source_chunk_id": "native:" + chunk.ID, "native_chunk_id": chunk.ID, "source_content_hash": contentHash(chunk.Content), "extraction_status": status, "entity_count": counts[chunk.ID]["entity"], "relation_count": counts[chunk.ID]["relation"]}, contentHash(chunk.Content)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return pruneNativeOrphans(tx, tenant, ns)
|
||||
})
|
||||
}
|
||||
|
||||
func lockGraph(tx *gorm.DB, tenant uint64, kb string) error {
|
||||
return tx.Exec("SELECT pg_advisory_xact_lock(hashtextextended(?,0))", fmt.Sprintf("native-graph:%d:%s", tenant, kb)).Error
|
||||
}
|
||||
|
||||
func putNativeRecord(tx *gorm.DB, tenant uint64, ns types.NameSpace, table string, id int64, identity string, payload map[string]any, hash string) error {
|
||||
if table == "knowledgeentity" {
|
||||
if err := reconcileNativeEntityAttributes(tx, tenant, ns, id, identity, payload); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
payload["origin"], payload["native_identity"], payload["native_knowledge_id"], payload["status"] = nativeOrigin, identity, ns.Knowledge, "active"
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if hash == "" {
|
||||
hash = contentHash(string(data))
|
||||
}
|
||||
result := tx.Exec(`INSERT INTO native_graph_records(tenant_id,knowledge_base_id,source_table,legacy_id,graph_version_id,payload,source_snapshot_sha256)
|
||||
VALUES(?,?,?,?,0,?::jsonb,?) ON CONFLICT(tenant_id,knowledge_base_id,source_table,legacy_id) DO UPDATE SET payload=excluded.payload,source_snapshot_sha256=excluded.source_snapshot_sha256
|
||||
WHERE native_graph_records.payload->>'native_identity'=excluded.payload->>'native_identity' AND native_graph_records.payload->>'origin'=?`, tenant, ns.KnowledgeBase, table, id, string(data), hash, nativeOrigin)
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected != 1 {
|
||||
return fmt.Errorf("native graph identity collision; preserved graph was not modified")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func reconcileNativeEntityAttributes(tx *gorm.DB, tenant uint64, ns types.NameSpace, id int64, identity string, payload map[string]any) error {
|
||||
var evidence []struct {
|
||||
Payload json.RawMessage
|
||||
}
|
||||
// Evidence for the replaced sources has already been rewritten with raw
|
||||
// node attributes. Only still-current sources may preserve shared facts.
|
||||
if err := tx.Raw(`SELECT e.payload FROM native_graph_records e
|
||||
JOIN native_graph_sources s ON s.tenant_id=e.tenant_id AND s.knowledge_base_id=e.knowledge_base_id AND s.source_chunk_id=e.payload->>'source_chunk_id'
|
||||
JOIN chunks c ON c.id=s.native_chunk_id AND c.tenant_id=s.tenant_id AND c.knowledge_base_id=s.knowledge_base_id
|
||||
AND c.deleted_at IS NULL AND c.is_enabled=true AND encode(sha256(convert_to(c.content,'UTF8')),'hex')=s.native_content_sha256
|
||||
JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id AND k.deleted_at IS NULL
|
||||
JOIN knowledge_bases kb ON kb.id=c.knowledge_base_id AND kb.tenant_id=c.tenant_id AND kb.deleted_at IS NULL
|
||||
WHERE e.tenant_id=? AND e.knowledge_base_id=? AND e.source_table='knowledgegraphevidence'
|
||||
AND e.payload->>'subject_type'='entity' AND e.payload->>'subject_id'=?
|
||||
AND e.payload->>'origin'=? AND e.payload->>'status'='active' AND e.payload->>'native_knowledge_id'=?
|
||||
AND c.knowledge_id=? AND s.native_knowledge_id=c.knowledge_id AND s.mapping_state='native_extraction'
|
||||
AND e.payload->>'source_content_hash'=s.native_content_sha256 AND s.source_content_hash=s.native_content_sha256
|
||||
AND e.source_snapshot_sha256=s.native_content_sha256
|
||||
ORDER BY c.id,e.legacy_id FOR SHARE OF c,k,kb`, tenant, ns.KnowledgeBase, fmt.Sprint(id), nativeOrigin, ns.Knowledge, ns.Knowledge).Scan(&evidence).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
attributes := []string{}
|
||||
unknown := false
|
||||
for _, row := range evidence {
|
||||
var source struct {
|
||||
Attributes json.RawMessage `json:"entity_attributes"`
|
||||
}
|
||||
if err := json.Unmarshal(row.Payload, &source); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(source.Attributes) == 0 {
|
||||
unknown = true
|
||||
continue
|
||||
}
|
||||
var snapshot []*string
|
||||
if err := json.Unmarshal(source.Attributes, &snapshot); err != nil {
|
||||
return fmt.Errorf("invalid native graph entity attribute evidence: %w", err)
|
||||
}
|
||||
if snapshot == nil {
|
||||
unknown = true
|
||||
continue
|
||||
}
|
||||
values := make([]string, len(snapshot))
|
||||
for i, value := range snapshot {
|
||||
if value == nil {
|
||||
return fmt.Errorf("invalid native graph entity attribute evidence: null attribute")
|
||||
}
|
||||
values[i] = *value
|
||||
}
|
||||
if len(values) == 0 {
|
||||
continue
|
||||
}
|
||||
if len(attributes) > 0 && !slices.Equal(attributes, values) {
|
||||
return fmt.Errorf("shared native graph entity attributes conflict; extraction requires review")
|
||||
}
|
||||
attributes = values
|
||||
}
|
||||
entityType := "concept"
|
||||
if len(attributes) > 0 {
|
||||
entityType = attributes[0]
|
||||
}
|
||||
if unknown {
|
||||
var existing Record
|
||||
result := tx.Raw(`SELECT payload FROM native_graph_records WHERE tenant_id=? AND knowledge_base_id=?
|
||||
AND source_table='knowledgeentity' AND legacy_id=? AND payload->>'origin'=? AND payload->>'native_identity'=?`, tenant, ns.KnowledgeBase, id, nativeOrigin, identity).Scan(&existing)
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
var prior struct {
|
||||
Attributes []string `json:"attributes"`
|
||||
Type string `json:"entity_type"`
|
||||
}
|
||||
if result.RowsAffected != 1 {
|
||||
return fmt.Errorf("shared native graph entity attribute provenance is unavailable; extraction requires review")
|
||||
}
|
||||
if err := json.Unmarshal(existing.Payload, &prior); err != nil {
|
||||
return err
|
||||
}
|
||||
// Legacy evidence cannot prove which surviving source owns a fact.
|
||||
// Refuse changing its aggregate until those sources are re-extracted.
|
||||
if !slices.Equal(prior.Attributes, attributes) || prior.Type != entityType {
|
||||
return fmt.Errorf("shared native graph entity attribute provenance is unavailable; extraction requires review")
|
||||
}
|
||||
attributes = prior.Attributes
|
||||
}
|
||||
payload["attributes"], payload["entity_type"] = attributes, entityType
|
||||
return nil
|
||||
}
|
||||
|
||||
func putEvidence(tx *gorm.DB, tenant uint64, ns types.NameSpace, subject string, subjectID int64, chunk sourceChunk, quotation string, attributes ...[]string) error {
|
||||
id, identity := nativeID(ns.Knowledge, "evidence", subject, fmt.Sprint(subjectID), chunk.ID)
|
||||
excerpt := []rune(chunk.Content)
|
||||
if len(excerpt) > 2000 {
|
||||
excerpt = excerpt[:2000]
|
||||
}
|
||||
if quotation != "" {
|
||||
excerpt = []rune(quotation)
|
||||
}
|
||||
payload := map[string]any{"subject_type": subject, "subject_id": subjectID, "source_chunk_id": "native:" + chunk.ID, "source_file": chunk.FileName, "source_content_hash": contentHash(chunk.Content), "excerpt": string(excerpt)}
|
||||
if subject == "entity" {
|
||||
if len(attributes) != 1 {
|
||||
return fmt.Errorf("native graph entity evidence requires source attributes")
|
||||
}
|
||||
payload["entity_attributes"] = append([]string{}, attributes[0]...)
|
||||
}
|
||||
return putNativeRecord(tx, tenant, ns, "knowledgegraphevidence", id, identity, payload, contentHash(chunk.Content))
|
||||
}
|
||||
|
||||
func pruneNativeOrphans(tx *gorm.DB, tenant uint64, ns types.NameSpace) error {
|
||||
query := `DELETE FROM native_graph_records r WHERE r.tenant_id=? AND r.knowledge_base_id=? AND r.payload->>'origin'=? AND r.source_table IN ('knowledgeentity','knowledgerelation')`
|
||||
args := []any{tenant, ns.KnowledgeBase, nativeOrigin}
|
||||
if ns.Knowledge != "" {
|
||||
query += " AND r.payload->>'native_knowledge_id'=?"
|
||||
args = append(args, ns.Knowledge)
|
||||
}
|
||||
query += ` AND NOT EXISTS (SELECT 1 FROM native_graph_records e WHERE e.tenant_id=r.tenant_id AND e.knowledge_base_id=r.knowledge_base_id AND e.source_table='knowledgegraphevidence' AND e.payload->>'subject_id'=r.legacy_id::text AND e.payload->>'subject_type'=CASE r.source_table WHEN 'knowledgeentity' THEN 'entity' ELSE 'relation' END)`
|
||||
return tx.Exec(query, args...).Error
|
||||
}
|
||||
|
||||
func (r *Repository) deleteNativeGraphs(ctx context.Context, namespaces []types.NameSpace) error {
|
||||
tenant, ok := types.TenantIDFromContext(ctx)
|
||||
if !ok || tenant == 0 || r.db == nil {
|
||||
return fmt.Errorf("graph deletion requires tenant and database")
|
||||
}
|
||||
for _, ns := range namespaces {
|
||||
if ns.KnowledgeBase == "" {
|
||||
return fmt.Errorf("graph deletion requires knowledge base scope")
|
||||
}
|
||||
}
|
||||
for _, ns := range namespaces {
|
||||
if err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if err := lockGraph(tx, tenant, ns.KnowledgeBase); err != nil {
|
||||
return err
|
||||
}
|
||||
query := tx.Table("native_graph_records").Where("tenant_id=? AND knowledge_base_id=? AND payload->>'origin'=?", tenant, ns.KnowledgeBase, nativeOrigin)
|
||||
if ns.Knowledge != "" {
|
||||
// Version copies preserve original payloads but remap source
|
||||
// pointers. Scope deletions through those current pointers too.
|
||||
query = query.Where("source_table IN ('knowledgegraphevidence','knowledgegraphchunkstate') AND (payload->>'native_knowledge_id'=? OR payload->>'source_chunk_id' IN (SELECT source_chunk_id FROM native_graph_sources WHERE tenant_id=? AND knowledge_base_id=? AND native_knowledge_id=?))", ns.Knowledge, tenant, ns.KnowledgeBase, ns.Knowledge)
|
||||
}
|
||||
if err := query.Delete(&Record{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if ns.Knowledge != "" {
|
||||
if err := pruneNativeOrphans(tx, tenant, types.NameSpace{KnowledgeBase: ns.KnowledgeBase}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
query = tx.Table("native_graph_sources").Where("tenant_id=? AND knowledge_base_id=? AND source_chunk_id LIKE 'native:%'", tenant, ns.KnowledgeBase)
|
||||
if ns.Knowledge != "" {
|
||||
query = query.Where("native_knowledge_id=?", ns.Knowledge)
|
||||
}
|
||||
return query.Delete(&Record{}).Error
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if r.fallback != nil {
|
||||
return r.fallback.DelGraph(ctx, namespaces)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,657 @@
|
||||
package legacygraph
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestNativeIdentity(t *testing.T) {
|
||||
a, identity := nativeID("doc", "entity", "concept")
|
||||
b, _ := nativeID("other-doc", "entity", "concept")
|
||||
if a < 1<<52 || a > (1<<53)-1 || a == b || len(identity) != 64 {
|
||||
t.Fatal("invalid stable document-scoped browser-safe ID")
|
||||
}
|
||||
c, again := nativeID("doc", "entity", "concept")
|
||||
if a != c || identity != again {
|
||||
t.Fatal("unstable identity")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeGenericEntityIdentityIsSourceScoped(t *testing.T) {
|
||||
for _, name := range []string{"本品", "本商品", "前段未名产品", "后段未名产品", "本品(前段)", "前段 未名产品", "本 品 (前段)", "This Product"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
first, _, err := nativeEntityIdentity("doc", &types.GraphNode{Name: name, Chunks: []string{"first"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, _, err := nativeEntityIdentity("doc", &types.GraphNode{Name: name, Chunks: []string{"second"}})
|
||||
if err != nil || first == second {
|
||||
t.Fatal("unnamed products from separate chunks were merged", err)
|
||||
}
|
||||
for _, chunks := range [][]string{nil, {""}, {"first", "second"}} {
|
||||
if _, _, err := nativeEntityIdentity("doc", &types.GraphNode{Name: name, Chunks: chunks}); err == nil {
|
||||
t.Fatalf("missing or ambiguous source accepted: %v", chunks)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, name := range []string{"Named product", "修复凝胶", "本品专用护理套装", "未名产品研究中心"} {
|
||||
for _, chunk := range []string{"first", "second"} {
|
||||
id, _, err := nativeEntityIdentity("doc", &types.GraphNode{Name: name, Chunks: []string{chunk}})
|
||||
expected, _ := nativeID("doc", "entity", normalizedName(name))
|
||||
if err != nil || id != expected {
|
||||
t.Fatal("specific product lost its stable document identity", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeGenericProductsDoNotShareRelations(t *testing.T) {
|
||||
for _, name := range []string{"本品", "本商品", "前段未名产品", "后段未名产品", "本品(前段)"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
if err := tx.Exec(`INSERT INTO chunks VALUES('sibling',7,'kb','doc','Evidence about trust and learning.',NULL,true)`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
for _, chunkID := range []string{"chunk", "sibling"} {
|
||||
graph := testGraph(chunkID)
|
||||
graph.Node[0].Name = name
|
||||
graph.Relation[0].Node1 = name
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{graph}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
overview, err := r.Browse(ctx, "kb", "", 0, 0)
|
||||
if err != nil || overview.Counts["knowledgeentity"] != 3 || overview.Counts["knowledgerelation"] != 2 {
|
||||
t.Fatalf("separate unnamed products conflated: %#v %v", overview, err)
|
||||
}
|
||||
for _, chunkID := range []string{"chunk", "sibling"} {
|
||||
id, _, _ := nativeEntityIdentity("doc", &types.GraphNode{Name: name, Chunks: []string{chunkID}})
|
||||
detail, err := r.Browse(ctx, "kb", "", 0, id)
|
||||
if err != nil || len(detail.Relations) != 1 || len(detail.Evidence) != 1 || detail.Evidence[0].NativeChunkID != chunkID {
|
||||
t.Fatalf("unnamed product points to another source: %#v %v", detail, err)
|
||||
}
|
||||
}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{{Node: []*types.GraphNode{{Name: name, Chunks: []string{"chunk", "sibling"}}}}}); err == nil {
|
||||
t.Fatal("ambiguous multi-source write accepted")
|
||||
}
|
||||
after, err := r.Browse(ctx, "kb", "", 0, 0)
|
||||
if err != nil || after.Counts["knowledgerelation"] != 2 {
|
||||
t.Fatal("failed ambiguous write damaged committed graph", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func nativeWriteFixture(t *testing.T) (*gorm.DB, *Repository, context.Context) {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("GRAPH_TEST_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("GRAPH_TEST_DSN required for temporary-table integration")
|
||||
}
|
||||
db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { sqlDB.Close() })
|
||||
tx := db.Begin()
|
||||
if tx.Error != nil {
|
||||
t.Fatal(tx.Error)
|
||||
}
|
||||
t.Cleanup(func() { tx.Rollback() })
|
||||
// Temporary tables shadow live names on this transaction's connection;
|
||||
// tests never mutate application records or permanent schemas.
|
||||
for _, sql := range []string{
|
||||
`CREATE TEMP TABLE knowledge_bases(id text,tenant_id bigint,deleted_at timestamptz) ON COMMIT DROP`,
|
||||
`CREATE TEMP TABLE knowledges(id text,tenant_id bigint,knowledge_base_id text,file_name text,deleted_at timestamptz) ON COMMIT DROP`,
|
||||
`CREATE TEMP TABLE chunks(id text,tenant_id bigint,knowledge_base_id text,knowledge_id text,content text,deleted_at timestamptz,is_enabled boolean) ON COMMIT DROP`,
|
||||
`CREATE TEMP TABLE native_graph_records(tenant_id bigint,knowledge_base_id text,source_table text,legacy_id bigint,graph_version_id bigint,payload jsonb,source_snapshot_sha256 text,imported_at timestamptz DEFAULT now(),PRIMARY KEY(tenant_id,knowledge_base_id,source_table,legacy_id)) ON COMMIT DROP`,
|
||||
`CREATE TEMP TABLE native_graph_sources(tenant_id bigint,knowledge_base_id text,source_chunk_id text,source_file text,source_content_hash text,excerpt text,native_knowledge_id text,native_chunk_id text,native_content_sha256 text,mapping_state text,PRIMARY KEY(tenant_id,knowledge_base_id,source_chunk_id)) ON COMMIT DROP`,
|
||||
`INSERT INTO knowledge_bases VALUES('kb',7,NULL),('kb',8,NULL),('other-kb',7,NULL)`,
|
||||
`INSERT INTO knowledges VALUES('doc',7,'kb','source.md',NULL),('doc2',7,'kb','other.md',NULL),('doc',8,'kb','private.md',NULL)`,
|
||||
`INSERT INTO chunks VALUES('chunk',7,'kb','doc','Evidence about trust and learning.',NULL,true),('chunk2',7,'kb','doc2','Evidence about trust and learning.',NULL,true),('foreign',8,'kb','doc','Private source.',NULL,true)`,
|
||||
} {
|
||||
if err := tx.Exec(sql).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))
|
||||
return tx, NewRepository(tx, nil), ctx
|
||||
}
|
||||
|
||||
func testGraph(chunk string) *types.GraphData {
|
||||
return &types.GraphData{Node: []*types.GraphNode{{Name: "Trust", Chunks: []string{chunk}, Attributes: []string{"concept", "Earned through reliable evidence"}}, {Name: "Learning", Chunks: []string{chunk}, Attributes: []string{"process"}}}, Relation: []*types.GraphRelation{{Node1: "Trust", Node2: "Learning", Type: "supports"}}}
|
||||
}
|
||||
|
||||
func TestNativeGraphWriteReplayBrowseAndIsolation(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
chunk := &types.Chunk{ID: "chunk", TenantID: 7, KnowledgeBaseID: "kb", KnowledgeID: "doc", Content: "Evidence about trust and learning."}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := r.AddChunkGraph(ctx, ns, chunk, testGraph("chunk")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
overview, err := r.Browse(ctx, "kb", "", 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if overview.Counts["knowledgeentity"] != 2 || overview.Counts["knowledgerelation"] != 1 || overview.Counts["knowledgegraphevidence"] != 3 || overview.Counts["knowledgegraphchunkstate"] != 1 || overview.MappedSources != 1 {
|
||||
t.Fatalf("not durable/idempotent: %#v", overview)
|
||||
}
|
||||
id, _ := nativeID("doc", "entity", "trust")
|
||||
detail, err := r.Browse(ctx, "kb", "", 0, id)
|
||||
if err != nil || len(detail.Relations) != 1 || len(detail.Evidence) != 1 || detail.Evidence[0].NativeChunkID != "chunk" {
|
||||
t.Fatalf("detail missing actual evidence: %#v %v", detail, err)
|
||||
}
|
||||
found, err := r.SearchNode(ctx, ns, []string{"Trust"})
|
||||
if err != nil || len(found.Node) != 2 || len(found.Relation) != 1 {
|
||||
t.Fatalf("native RAG cannot find persisted graph: %#v %v", found, err)
|
||||
}
|
||||
complete, err := r.HasChunkGraph(ctx, chunk)
|
||||
if err != nil || !complete {
|
||||
t.Fatalf("missing checkpoint: %v", err)
|
||||
}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{testGraph("foreign")}); err == nil {
|
||||
t.Fatal("cross-tenant source accepted")
|
||||
}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{testGraph("chunk2")}); err == nil {
|
||||
t.Fatal("cross-document source accepted")
|
||||
}
|
||||
changed := *chunk
|
||||
changed.Content = "old model input"
|
||||
if err := r.AddChunkGraph(ctx, ns, &changed, testGraph("chunk")); err == nil {
|
||||
t.Fatal("stale model output accepted")
|
||||
}
|
||||
bad := testGraph("chunk")
|
||||
bad.Relation[0].Node2 = "invented"
|
||||
if err := r.AddChunkGraph(ctx, ns, chunk, bad); err == nil {
|
||||
t.Fatal("ungrounded relation accepted")
|
||||
}
|
||||
after, err := r.Browse(ctx, "kb", "", 0, id)
|
||||
if err != nil || len(after.Evidence) != 1 {
|
||||
t.Fatal("failed extraction destroyed committed evidence", err)
|
||||
}
|
||||
if err := r.AddGraph(ctx, types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc2"}, []*types.GraphData{testGraph("chunk2")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.Exec(`INSERT INTO native_graph_records VALUES(7,'kb','knowledgeentity',1,1,'{"canonical_name":"Archived","status":"active"}',repeat('a',64),now())`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.DelGraph(ctx, []types.NameSpace{ns}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := r.Browse(ctx, "kb", "", 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if left.Counts["knowledgeentity"] != 3 || left.Sources != 1 {
|
||||
t.Fatalf("deletion damaged other document/archive: %#v", left)
|
||||
}
|
||||
wrong := context.WithValue(ctx, types.TenantIDContextKey, uint64(8))
|
||||
if err := r.DelGraph(wrong, []types.NameSpace{{KnowledgeBase: "kb"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err = r.Browse(ctx, "kb", "", 0, 0)
|
||||
if err != nil || left.Counts["knowledgeentity"] != 3 {
|
||||
t.Fatal("cross-tenant deletion", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeEmptyResultAndReextraction(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
chunk := &types.Chunk{ID: "chunk", TenantID: 7, KnowledgeBaseID: "kb", KnowledgeID: "doc", Content: "Evidence about trust and learning."}
|
||||
if err := r.AddChunkGraph(ctx, ns, chunk, testGraph("chunk")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.AddChunkGraph(ctx, ns, chunk, &types.GraphData{}); err == nil {
|
||||
t.Fatal("empty extraction accepted without review")
|
||||
}
|
||||
overview, err := r.Browse(ctx, "kb", "", 0, 0)
|
||||
if err != nil || overview.Counts["knowledgerelation"] == 0 || overview.Counts["knowledgegraphchunkstate"] != 1 {
|
||||
t.Fatalf("rejected result replaced prior graph: %#v %v", overview, err)
|
||||
}
|
||||
complete, err := r.HasChunkGraph(ctx, chunk)
|
||||
if err != nil || !complete {
|
||||
t.Fatal("prior verified graph was lost", err)
|
||||
}
|
||||
if err := tx.Exec(`UPDATE native_graph_records SET payload=jsonb_set(payload,'{relation_count}','0'::jsonb) WHERE source_table='knowledgegraphchunkstate'`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
complete, err = r.HasChunkGraph(ctx, chunk)
|
||||
if err != nil || complete {
|
||||
t.Fatal("legacy empty checkpoint was accepted", err)
|
||||
}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{{}}); err == nil {
|
||||
t.Fatal("empty extraction without source accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRelationQuotationIsPersistedAndChecked(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
graph := testGraph("chunk")
|
||||
graph.Relation[0].Evidence = "trust and learning"
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{graph}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var quote, excerpt string
|
||||
if err := tx.Raw(`SELECT payload->>'evidence' FROM native_graph_records WHERE source_table='knowledgerelation'`).Scan("e).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.Raw(`SELECT payload->>'excerpt' FROM native_graph_records WHERE source_table='knowledgegraphevidence' AND payload->>'subject_type'='relation'`).Scan(&excerpt).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if quote != graph.Relation[0].Evidence || excerpt != quote {
|
||||
t.Fatalf("relation quotation not retained: %q %q", quote, excerpt)
|
||||
}
|
||||
graph.Relation[0].Evidence = "invented quotation"
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{graph}); err == nil {
|
||||
t.Fatal("relation quotation absent from source accepted")
|
||||
}
|
||||
if err := tx.Raw(`SELECT payload->>'evidence' FROM native_graph_records WHERE source_table='knowledgerelation'`).Scan("e).Error; err != nil || quote != "trust and learning" {
|
||||
t.Fatal("invalid quotation damaged prior graph", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeRelationRequiresCurrentSourceEvidence(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
if err := tx.Exec(`INSERT INTO chunks VALUES('sibling',7,'kb','doc','Another source discussing trust and learning.',NULL,true)`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{testGraph("chunk")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
graph := testGraph("sibling")
|
||||
graph.Relation = nil
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{graph}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.Exec(`UPDATE chunks SET content='The original relation is no longer supported.' WHERE id='chunk' AND tenant_id=7`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found, err := r.SearchNode(ctx, ns, []string{"Trust"})
|
||||
if err != nil || len(found.Node) != 2 || len(found.Relation) != 0 {
|
||||
t.Fatalf("stale relation remained available through separately supported entities: %#v %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeCopiedVersionDeletionUsesRemappedSources(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{testGraph("chunk")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, sql := range []string{
|
||||
`INSERT INTO knowledges VALUES('copy-doc',7,'other-kb','copy.md',NULL)`,
|
||||
`INSERT INTO chunks VALUES('copy-chunk',7,'other-kb','copy-doc','Evidence about trust and learning.',NULL,true)`,
|
||||
`INSERT INTO native_graph_records SELECT tenant_id,'other-kb',source_table,legacy_id,graph_version_id,payload,source_snapshot_sha256,imported_at FROM native_graph_records WHERE tenant_id=7 AND knowledge_base_id='kb'`,
|
||||
`INSERT INTO native_graph_sources SELECT tenant_id,'other-kb',source_chunk_id,source_file,source_content_hash,excerpt,'copy-doc','copy-chunk',native_content_sha256,mapping_state FROM native_graph_sources WHERE tenant_id=7 AND knowledge_base_id='kb'`,
|
||||
} {
|
||||
if err := tx.Exec(sql).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := r.DelGraph(ctx, []types.NameSpace{{KnowledgeBase: "other-kb", Knowledge: "copy-doc"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
copy, err := r.Browse(ctx, "other-kb", "", 0, 0)
|
||||
if err != nil || len(copy.Counts) != 0 || copy.Sources != 0 {
|
||||
t.Fatalf("copied graph was not deleted via its current document: %#v %v", copy, err)
|
||||
}
|
||||
original, err := r.Browse(ctx, "kb", "", 0, 0)
|
||||
if err != nil || original.Counts["knowledgeentity"] != 2 || original.Sources != 1 {
|
||||
t.Fatal("copy deletion changed original graph", err)
|
||||
}
|
||||
}
|
||||
|
||||
func nativeGraphWriteState(t *testing.T, tx *gorm.DB) string {
|
||||
t.Helper()
|
||||
var state string
|
||||
if err := tx.Raw(`SELECT jsonb_build_object(
|
||||
'records',COALESCE((SELECT jsonb_agg(to_jsonb(r) ORDER BY tenant_id,knowledge_base_id,source_table,legacy_id) FROM native_graph_records r),'[]'::jsonb),
|
||||
'sources',COALESCE((SELECT jsonb_agg(to_jsonb(s) ORDER BY tenant_id,knowledge_base_id,source_chunk_id) FROM native_graph_sources s),'[]'::jsonb))::text`).Scan(&state).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
func assertNativeTrustAttributes(t *testing.T, tx *gorm.DB, want []string) string {
|
||||
t.Helper()
|
||||
id, _ := nativeID("doc", "entity", "trust")
|
||||
var record struct {
|
||||
Payload json.RawMessage
|
||||
Hash string
|
||||
State string
|
||||
}
|
||||
if err := tx.Raw(`SELECT payload,source_snapshot_sha256 AS hash,to_jsonb(r)::text AS state FROM native_graph_records r
|
||||
WHERE tenant_id=7 AND knowledge_base_id='kb' AND source_table='knowledgeentity' AND legacy_id=?`, id).Scan(&record).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var entity struct {
|
||||
Attributes []string `json:"attributes"`
|
||||
Type string `json:"entity_type"`
|
||||
}
|
||||
if err := json.Unmarshal(record.Payload, &entity); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantType := "concept"
|
||||
if len(want) > 0 {
|
||||
wantType = want[0]
|
||||
}
|
||||
if !slices.Equal(entity.Attributes, want) || entity.Type != wantType {
|
||||
t.Fatalf("unexpected aggregate attributes/type: %#v; want %v / %s", entity, want, wantType)
|
||||
}
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(record.Payload, &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil || record.Hash != contentHash(string(encoded)) {
|
||||
t.Fatalf("entity payload hash is stale: %s %v", record.Hash, err)
|
||||
}
|
||||
return record.State
|
||||
}
|
||||
|
||||
func TestNativeSharedEntityAttributesFollowTheirSources(t *testing.T) {
|
||||
for _, siblingHasAttributes := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("sibling_attributes_%t", siblingHasAttributes), func(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
if err := tx.Exec(`INSERT INTO chunks VALUES('sibling',7,'kb','doc','Another source about trust and learning.',NULL,true)`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
original := testGraph("chunk")
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{original}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := assertNativeTrustAttributes(t, tx, original.Node[0].Attributes)
|
||||
sibling := testGraph("sibling")
|
||||
if !siblingHasAttributes {
|
||||
sibling.Node[0].Attributes = nil
|
||||
}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{sibling}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after := assertNativeTrustAttributes(t, tx, original.Node[0].Attributes); after != before {
|
||||
t.Fatal("a second source changed the existing entity row")
|
||||
}
|
||||
var sourceAttributes string
|
||||
id, _ := nativeID("doc", "entity", "trust")
|
||||
if err := tx.Raw(`SELECT payload->'entity_attributes' FROM native_graph_records WHERE source_table='knowledgegraphevidence'
|
||||
AND payload->>'subject_type'='entity' AND payload->>'subject_id'=? AND payload->>'source_chunk_id'='native:sibling'`, fmt.Sprint(id)).Scan(&sourceAttributes).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !siblingHasAttributes && sourceAttributes != "[]" {
|
||||
t.Fatalf("bare source inherited aggregate attributes: %s", sourceAttributes)
|
||||
}
|
||||
original.Node[0].Attributes = nil
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{original}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertNativeTrustAttributes(t, tx, sibling.Node[0].Attributes)
|
||||
sibling.Node[0].Attributes = []string{}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{sibling}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertNativeTrustAttributes(t, tx, nil)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeSharedEntityAttributeConflictRollsBack(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
if err := tx.Exec(`INSERT INTO chunks VALUES('sibling',7,'kb','doc','Another source about trust and learning.',NULL,true)`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{testGraph("chunk")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := nativeGraphWriteState(t, tx)
|
||||
conflicting := testGraph("sibling")
|
||||
conflicting.Node[0].Attributes = []string{"concept", "Never earned through reliable evidence"}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{conflicting}); err == nil || !strings.Contains(err.Error(), "attributes conflict") {
|
||||
t.Fatalf("conflicting source did not require review: %v", err)
|
||||
}
|
||||
if after := nativeGraphWriteState(t, tx); after != before {
|
||||
t.Fatal("failed conflict changed records, evidence, mapping or checkpoint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeSharedEntityAttributesAreRemovedWithTheirNode(t *testing.T) {
|
||||
for _, siblingHasAttributes := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("sibling_attributes_%t", siblingHasAttributes), func(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
if err := tx.Exec(`INSERT INTO chunks VALUES('sibling',7,'kb','doc','Another source about trust and learning.',NULL,true)`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
original, sibling := testGraph("chunk"), testGraph("sibling")
|
||||
if !siblingHasAttributes {
|
||||
sibling.Node[0].Attributes = nil
|
||||
}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{original, sibling}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original.Node[0].Name = "Practice"
|
||||
original.Relation[0].Node1 = "Practice"
|
||||
chunk := &types.Chunk{ID: "chunk", TenantID: 7, KnowledgeBaseID: "kb", KnowledgeID: "doc", Content: "Evidence about trust and learning."}
|
||||
if err := r.AddChunkGraph(ctx, ns, chunk, original); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertNativeTrustAttributes(t, tx, sibling.Node[0].Attributes)
|
||||
sibling.Node[0].Name = "Practice"
|
||||
sibling.Node[0].Attributes = original.Node[0].Attributes
|
||||
sibling.Relation[0].Node1 = "Practice"
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{sibling}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, _ := nativeID("doc", "entity", "trust")
|
||||
var count int64
|
||||
if err := tx.Table("native_graph_records").Where("source_table='knowledgeentity' AND legacy_id=?", id).Count(&count).Error; err != nil || count != 0 {
|
||||
t.Fatalf("entity without surviving sources was not pruned: %d %v", count, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeLegacyEntityAttributeProvenanceRequiresReview(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
if err := tx.Exec(`INSERT INTO chunks VALUES('sibling',7,'kb','doc','Another source about trust and learning.',NULL,true)`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{testGraph("chunk")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, _ := nativeID("doc", "entity", "trust")
|
||||
if err := tx.Exec(`UPDATE native_graph_records SET payload=payload-'entity_attributes' WHERE source_table='knowledgegraphevidence'
|
||||
AND payload->>'subject_type'='entity' AND payload->>'subject_id'=?`, fmt.Sprint(id)).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := nativeGraphWriteState(t, tx)
|
||||
bare := testGraph("sibling")
|
||||
bare.Node[0].Attributes = nil
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{bare}); err == nil || !strings.Contains(err.Error(), "provenance is unavailable") {
|
||||
t.Fatalf("legacy unknown provenance treated as known empty: %v", err)
|
||||
}
|
||||
if after := nativeGraphWriteState(t, tx); after != before {
|
||||
t.Fatal("unknown provenance failure changed prior graph")
|
||||
}
|
||||
// A matching explicit source can coexist with unknown older evidence.
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{testGraph("sibling")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original := testGraph("chunk")
|
||||
original.Node[0].Attributes = nil
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{original}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertNativeTrustAttributes(t, tx, testGraph("sibling").Node[0].Attributes)
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{bare}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertNativeTrustAttributes(t, tx, nil)
|
||||
}
|
||||
|
||||
func TestNativeNullEntityAttributeEvidenceRequiresReview(t *testing.T) {
|
||||
for _, snapshot := range []string{"null", "[null]"} {
|
||||
t.Run(snapshot, func(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
if err := tx.Exec(`INSERT INTO chunks VALUES('sibling',7,'kb','doc','Another source about trust and learning.',NULL,true)`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{testGraph("chunk")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, _ := nativeID("doc", "entity", "trust")
|
||||
if err := tx.Exec(`UPDATE native_graph_records SET payload=jsonb_set(payload,'{entity_attributes}',?::jsonb)
|
||||
WHERE source_table='knowledgegraphevidence' AND payload->>'subject_type'='entity' AND payload->>'subject_id'=?`, snapshot, fmt.Sprint(id)).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := nativeGraphWriteState(t, tx)
|
||||
bare := testGraph("sibling")
|
||||
bare.Node[0].Attributes = nil
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{bare}); err == nil {
|
||||
t.Fatal("null attribute evidence was accepted as known empty")
|
||||
}
|
||||
if after := nativeGraphWriteState(t, tx); after != before {
|
||||
t.Fatal("null attribute evidence failure changed prior graph")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeSharedEntityIgnoresInvalidAttributeEvidence(t *testing.T) {
|
||||
for name, invalidate := range map[string]string{
|
||||
"changed source": `UPDATE chunks SET content='Changed source.' WHERE id='chunk' AND tenant_id=7`,
|
||||
"disabled source": `UPDATE chunks SET is_enabled=false WHERE id='chunk' AND tenant_id=7`,
|
||||
"deleted source": `UPDATE chunks SET deleted_at=now() WHERE id='chunk' AND tenant_id=7`,
|
||||
"wrong evidence hash": `UPDATE native_graph_records SET payload=jsonb_set(payload,'{source_content_hash}',to_jsonb(repeat('a',64))) WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"wrong row hash": `UPDATE native_graph_records SET source_snapshot_sha256=repeat('a',64) WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"inactive evidence": `UPDATE native_graph_records SET payload=jsonb_set(payload,'{status}','"inactive"'::jsonb) WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"wrong origin": `UPDATE native_graph_records SET payload=jsonb_set(payload,'{origin}','"foreign"'::jsonb) WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"wrong tenant": `UPDATE native_graph_records SET tenant_id=8 WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"wrong knowledge base": `UPDATE native_graph_records SET knowledge_base_id='other-kb' WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"wrong document": `UPDATE native_graph_records SET payload=jsonb_set(payload,'{native_knowledge_id}','"doc2"'::jsonb) WHERE source_table='knowledgegraphevidence' AND payload->>'source_chunk_id'='native:chunk'`,
|
||||
"wrong mapping document": `UPDATE native_graph_sources SET native_knowledge_id='doc2' WHERE source_chunk_id='native:chunk'`,
|
||||
"inactive mapping": `UPDATE native_graph_sources SET mapping_state='stale' WHERE source_chunk_id='native:chunk'`,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
if err := tx.Exec(`INSERT INTO chunks VALUES('sibling',7,'kb','doc','Another source about trust and learning.',NULL,true)`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{testGraph("chunk")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.Exec(invalidate).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bare := testGraph("sibling")
|
||||
bare.Node[0].Attributes = nil
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{bare}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertNativeTrustAttributes(t, tx, nil)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeEntityAttributeBatchOrderAndDuplicates(t *testing.T) {
|
||||
for _, reverse := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("reverse_%t", reverse), func(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
if err := tx.Exec(`INSERT INTO chunks VALUES('sibling',7,'kb','doc','Another source about trust and learning.',NULL,true)`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
first, second := testGraph("chunk"), testGraph("sibling")
|
||||
first.Node = append(first.Node, &types.GraphNode{Name: " trust ", Chunks: []string{"chunk", "chunk"}, Attributes: first.Node[0].Attributes})
|
||||
second.Node[0].Name = "TRUST"
|
||||
second.Node[0].Attributes = nil
|
||||
graphs := []*types.GraphData{first, second}
|
||||
if reverse {
|
||||
slices.Reverse(graphs)
|
||||
}
|
||||
if err := r.AddGraph(ctx, ns, graphs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entityBefore := assertNativeTrustAttributes(t, tx, first.Node[0].Attributes)
|
||||
slices.Reverse(graphs)
|
||||
if err := r.AddGraph(ctx, ns, graphs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if entityAfter := assertNativeTrustAttributes(t, tx, first.Node[0].Attributes); entityAfter != entityBefore {
|
||||
t.Fatal("batch order changed the entity payload or hash")
|
||||
}
|
||||
var count int
|
||||
if err := tx.Raw(`SELECT (payload->>'entity_count')::int FROM native_graph_records WHERE source_table='knowledgegraphchunkstate' AND payload->>'native_chunk_id'='chunk'`).Scan(&count).Error; err != nil || count != 2 {
|
||||
t.Fatalf("duplicate entity evidence inflated checkpoint: %d %v", count, err)
|
||||
}
|
||||
before := nativeGraphWriteState(t, tx)
|
||||
first.Node[2].Attributes = []string{"conflicting duplicate"}
|
||||
if err := r.AddGraph(ctx, ns, graphs); err == nil || !strings.Contains(err.Error(), "attributes conflict") {
|
||||
t.Fatalf("duplicate source attributes silently overwritten: %v", err)
|
||||
}
|
||||
if after := nativeGraphWriteState(t, tx); after != before {
|
||||
t.Fatal("failed duplicate source write changed the committed graph")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeLegacyEntityAttributeBatchOrder(t *testing.T) {
|
||||
for _, reverse := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("reverse_%t", reverse), func(t *testing.T) {
|
||||
tx, r, ctx := nativeWriteFixture(t)
|
||||
if err := tx.Exec(`INSERT INTO chunks VALUES('bare',7,'kb','doc','A bare reference to trust.',NULL,true),('explicit',7,'kb','doc','An explicit source about trust.',NULL,true)`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ns := types.NameSpace{KnowledgeBase: "kb", Knowledge: "doc"}
|
||||
original := testGraph("chunk")
|
||||
if err := r.AddGraph(ctx, ns, []*types.GraphData{original}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.Exec(`UPDATE native_graph_records SET payload=payload-'entity_attributes' WHERE source_table='knowledgegraphevidence' AND payload->>'subject_type'='entity'`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bare, explicit := testGraph("bare"), testGraph("explicit")
|
||||
bare.Node[0].Attributes = nil
|
||||
graphs := []*types.GraphData{bare, explicit}
|
||||
if reverse {
|
||||
slices.Reverse(graphs)
|
||||
}
|
||||
if err := r.AddGraph(ctx, ns, graphs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertNativeTrustAttributes(t, tx, original.Node[0].Attributes)
|
||||
original.Node[0].Attributes = nil
|
||||
explicit.Node[0].Attributes = nil
|
||||
graphs = append(graphs, original)
|
||||
if err := r.AddGraph(ctx, ns, graphs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertNativeTrustAttributes(t, tx, nil)
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user