243 lines
9.5 KiB
Go
243 lines
9.5 KiB
Go
|
// Package types defines data structures and types used throughout the system
|
|||
|
|
// These types are shared across different service modules to ensure data consistency
|
|||
|
|
package types
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"strings"
|
|||
|
|
"time"
|
|||
|
|
|
|||
|
|
"gorm.io/gorm"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
// ChunkType 定义了不同类型的 Chunk
|
|||
|
|
type ChunkType = string
|
|||
|
|
|
|||
|
|
const (
|
|||
|
|
// ChunkTypeText 表示普通的文本 Chunk
|
|||
|
|
ChunkTypeText ChunkType = "text"
|
|||
|
|
// ChunkTypeParentText 表示父子分块策略中的父文本 Chunk（仅用于上下文，不参与向量索引）
|
|||
|
|
ChunkTypeParentText ChunkType = "parent_text"
|
|||
|
|
// ChunkTypeImageOCR 表示图片 OCR 文本的 Chunk
|
|||
|
|
ChunkTypeImageOCR ChunkType = "image_ocr"
|
|||
|
|
// ChunkTypeImageCaption 表示图片描述的 Chunk
|
|||
|
|
ChunkTypeImageCaption ChunkType = "image_caption"
|
|||
|
|
// ChunkTypeSummary 表示摘要类型的 Chunk
|
|||
|
|
ChunkTypeSummary = "summary"
|
|||
|
|
// ChunkTypeEntity 表示实体类型的 Chunk
|
|||
|
|
ChunkTypeEntity ChunkType = "entity"
|
|||
|
|
// ChunkTypeRelationship 表示关系类型的 Chunk
|
|||
|
|
ChunkTypeRelationship ChunkType = "relationship"
|
|||
|
|
// ChunkTypeFAQ 表示 FAQ 条目 Chunk
|
|||
|
|
ChunkTypeFAQ ChunkType = "faq"
|
|||
|
|
// ChunkTypeWebSearch 表示 Web 搜索结果的 Chunk
|
|||
|
|
ChunkTypeWebSearch ChunkType = "web_search"
|
|||
|
|
// ChunkTypeTableSummary 表示数据表摘要的 Chunk
|
|||
|
|
ChunkTypeTableSummary ChunkType = "table_summary"
|
|||
|
|
// ChunkTypeTableColumn 表示数据表列描述的 Chunk
|
|||
|
|
ChunkTypeTableColumn ChunkType = "table_column"
|
|||
|
|
// ChunkTypeWikiPage 表示 Wiki 页面同步的 Chunk，用于将 wiki 页面接入现有检索管线
|
|||
|
|
ChunkTypeWikiPage ChunkType = "wiki_page"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
// ChunkStatus 定义了不同状态的 Chunk
|
|||
|
|
type ChunkStatus int
|
|||
|
|
|
|||
|
|
const (
|
|||
|
|
ChunkStatusDefault ChunkStatus = 0
|
|||
|
|
// ChunkStatusStored 表示已存储的 Chunk
|
|||
|
|
ChunkStatusStored ChunkStatus = 1
|
|||
|
|
// ChunkStatusIndexed 表示已索引的 Chunk
|
|||
|
|
ChunkStatusIndexed ChunkStatus = 2
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
// ChunkFlags 定义 Chunk 的标志位，用于管理多个布尔状态
|
|||
|
|
type ChunkFlags int
|
|||
|
|
|
|||
|
|
const (
|
|||
|
|
// ChunkFlagRecommended 表示可推荐状态（1 << 0 = 1）
|
|||
|
|
// 当设置此标志时，该 Chunk 可以被推荐给用户
|
|||
|
|
ChunkFlagRecommended ChunkFlags = 1 << 0
|
|||
|
|
// 未来可扩展更多标志位：
|
|||
|
|
// ChunkFlagPinned ChunkFlags = 1 << 1 // 置顶
|
|||
|
|
// ChunkFlagHot ChunkFlags = 1 << 2 // 热门
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
// HasFlag 检查是否设置了指定标志
|
|||
|
|
func (f ChunkFlags) HasFlag(flag ChunkFlags) bool {
|
|||
|
|
return f&flag != 0
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// SetFlag 设置指定标志
|
|||
|
|
func (f ChunkFlags) SetFlag(flag ChunkFlags) ChunkFlags {
|
|||
|
|
return f | flag
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ClearFlag 清除指定标志
|
|||
|
|
func (f ChunkFlags) ClearFlag(flag ChunkFlags) ChunkFlags {
|
|||
|
|
return f &^ flag
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ToggleFlag 切换指定标志
|
|||
|
|
func (f ChunkFlags) ToggleFlag(flag ChunkFlags) ChunkFlags {
|
|||
|
|
return f ^ flag
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ImageInfo 表示与 Chunk 关联的图片信息
|
|||
|
|
type ImageInfo struct {
|
|||
|
|
// 图片URL（COS）
|
|||
|
|
URL string `json:"url" gorm:"type:text"`
|
|||
|
|
// 原始图片URL
|
|||
|
|
OriginalURL string `json:"original_url" gorm:"type:text"`
|
|||
|
|
// 图片在文本中的开始位置
|
|||
|
|
StartPos int `json:"start_pos"`
|
|||
|
|
// 图片在文本中的结束位置
|
|||
|
|
EndPos int `json:"end_pos"`
|
|||
|
|
// 图片描述
|
|||
|
|
Caption string `json:"caption"`
|
|||
|
|
// 图片OCR文本
|
|||
|
|
OCRText string `json:"ocr_text"`
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// VideoInfo 表示与 Chunk 关联的视频信息
|
|||
|
|
type VideoInfo struct {
|
|||
|
|
// 视频URL
|
|||
|
|
URL string `json:"url" gorm:"type:text"`
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Chunk represents a document chunk
|
|||
|
|
// Chunks are meaningful text segments extracted from original documents
|
|||
|
|
// and are the basic units of knowledge base retrieval
|
|||
|
|
// Each chunk contains a portion of the original content
|
|||
|
|
// and maintains its positional relationship with the original text
|
|||
|
|
// Chunks can be independently embedded as vectors and retrieved, supporting precise content localization
|
|||
|
|
type Chunk struct {
|
|||
|
|
// Unique identifier of the chunk, using UUID format
|
|||
|
|
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
|||
|
|
// SeqID is an auto-increment integer ID for external API usage (FAQ entries)
|
|||
|
|
SeqID int64 `json:"seq_id" gorm:"type:bigint;uniqueIndex;autoIncrement"`
|
|||
|
|
// Tenant ID, used for multi-tenant isolation
|
|||
|
|
TenantID uint64 `json:"tenant_id"`
|
|||
|
|
// ID of the parent knowledge, associated with the Knowledge model
|
|||
|
|
KnowledgeID string `json:"knowledge_id"`
|
|||
|
|
// ID of the knowledge base, for quick location
|
|||
|
|
KnowledgeBaseID string `json:"knowledge_base_id"`
|
|||
|
|
// Optional tag ID for categorization within a knowledge base (used for FAQ)
|
|||
|
|
TagID string `json:"tag_id" gorm:"type:varchar(36);index"`
|
|||
|
|
// Actual text content of the chunk
|
|||
|
|
Content string `json:"content"`
|
|||
|
|
// SourceContent is the immutable parser output. Legacy rows are lazily
|
|||
|
|
// backfilled from Content on the first manual edit.
|
|||
|
|
SourceContent string `json:"-"`
|
|||
|
|
// ContentRevision is incremented for every user edit or rollback.
|
|||
|
|
ContentRevision int `json:"content_revision" gorm:"not null;default:0"`
|
|||
|
|
// IndexStatus reports whether the current content is reflected in the
|
|||
|
|
// retrieval stores: ready | processing | failed.
|
|||
|
|
IndexStatus string `json:"index_status" gorm:"type:varchar(16);not null;default:'ready'"`
|
|||
|
|
// LastEditorID records the actor that produced the current revision.
|
|||
|
|
LastEditorID string `json:"last_editor_id" gorm:"type:varchar(64);not null;default:''"`
|
|||
|
|
// Index position of the chunk in the original document
|
|||
|
|
ChunkIndex int `json:"chunk_index"`
|
|||
|
|
// Whether the chunk is enabled, can be used to temporarily disable certain chunks
|
|||
|
|
IsEnabled bool `json:"is_enabled" gorm:"default:true"`
|
|||
|
|
// Flags 存储多个布尔状态的位标志（如推荐状态等）
|
|||
|
|
// 默认值为 ChunkFlagRecommended (1)，表示默认可推荐
|
|||
|
|
Flags ChunkFlags `json:"flags" gorm:"default:1"`
|
|||
|
|
// Status of the chunk
|
|||
|
|
Status int `json:"status" gorm:"default:0"`
|
|||
|
|
// Starting character position in the original text
|
|||
|
|
StartAt int `json:"start_at"`
|
|||
|
|
// Ending character position in the original text
|
|||
|
|
EndAt int `json:"end_at"`
|
|||
|
|
// Previous chunk ID
|
|||
|
|
PreChunkID string `json:"pre_chunk_id"`
|
|||
|
|
// Next chunk ID
|
|||
|
|
NextChunkID string `json:"next_chunk_id"`
|
|||
|
|
// Chunk 类型，用于区分不同类型的 Chunk
|
|||
|
|
ChunkType ChunkType `json:"chunk_type" gorm:"type:varchar(20);default:'text'"`
|
|||
|
|
// 父 Chunk ID，用于关联图片 Chunk 和原始文本 Chunk
|
|||
|
|
ParentChunkID string `json:"parent_chunk_id" gorm:"type:varchar(36);index"`
|
|||
|
|
// 关系 Chunk ID，用于关联关系 Chunk 和原始文本 Chunk
|
|||
|
|
RelationChunks JSON `json:"relation_chunks" gorm:"type:json"`
|
|||
|
|
// 间接关系 Chunk ID，用于关联间接关系 Chunk 和原始文本 Chunk
|
|||
|
|
IndirectRelationChunks JSON `json:"indirect_relation_chunks" gorm:"type:json"`
|
|||
|
|
// Metadata 存储 chunk 级别的扩展信息，例如 FAQ 元数据
|
|||
|
|
Metadata JSON `json:"metadata" gorm:"type:json"`
|
|||
|
|
// ContentHash 存储内容的 hash 值，用于快速匹配（主要用于 FAQ）
|
|||
|
|
ContentHash string `json:"content_hash" gorm:"type:varchar(64);index"`
|
|||
|
|
// 图片信息，存储为 JSON
|
|||
|
|
ImageInfo string `json:"image_info" gorm:"type:text"`
|
|||
|
|
// Chunk creation time
|
|||
|
|
CreatedAt time.Time `json:"created_at"`
|
|||
|
|
// Chunk last update time
|
|||
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|||
|
|
// Soft delete marker, supports data recovery
|
|||
|
|
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
|
|||
|
|
// ContextHeader is a Markdown heading breadcrumb prepended when indexing.
|
|||
|
|
// It is persisted so a later content edit can rebuild the same index input.
|
|||
|
|
ContextHeader string `json:"-" gorm:"type:text"`
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ChunkRevision is an immutable snapshot of a superseded chunk revision.
|
|||
|
|
// The current content lives on Chunk; this table stores prior versions.
|
|||
|
|
type ChunkRevision struct {
|
|||
|
|
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
|||
|
|
TenantID uint64 `json:"tenant_id" gorm:"index"`
|
|||
|
|
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"type:varchar(36);index"`
|
|||
|
|
KnowledgeID string `json:"knowledge_id" gorm:"type:varchar(36);index"`
|
|||
|
|
ChunkID string `json:"chunk_id" gorm:"type:varchar(36);uniqueIndex:idx_chunk_revision"`
|
|||
|
|
Revision int `json:"revision" gorm:"uniqueIndex:idx_chunk_revision"`
|
|||
|
|
Content string `json:"content" gorm:"type:text"`
|
|||
|
|
IsEnabled bool `json:"is_enabled"`
|
|||
|
|
EditorID string `json:"editor_id" gorm:"type:varchar(64)"`
|
|||
|
|
EditSource string `json:"edit_source" gorm:"type:varchar(16)"`
|
|||
|
|
EditedAt time.Time `json:"edited_at"`
|
|||
|
|
CreatedAt time.Time `json:"created_at"`
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// EmbeddingContent returns the chunk content with ContextHeader prepended
|
|||
|
|
// when set. Use this where the embedding model needs section context that
|
|||
|
|
// isn't part of the literal Content. Surrounding whitespace on Content is
|
|||
|
|
// trimmed so leading/trailing newlines from boundary slicing don't dilute
|
|||
|
|
// the embedded vector.
|
|||
|
|
func (c *Chunk) EmbeddingContent() string {
|
|||
|
|
if c == nil {
|
|||
|
|
return ""
|
|||
|
|
}
|
|||
|
|
body := strings.TrimSpace(c.Content)
|
|||
|
|
if c.ContextHeader == "" {
|
|||
|
|
return body
|
|||
|
|
}
|
|||
|
|
return c.ContextHeader + "\n\n" + body
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// AssignChunkSeqIDs assigns sequential SeqIDs to a batch of chunks that have SeqID == 0.
|
|||
|
|
// Must be called before CreateInBatches for SQLite compatibility.
|
|||
|
|
func AssignChunkSeqIDs(tx *gorm.DB, chunks []*Chunk) error {
|
|||
|
|
needAssign := false
|
|||
|
|
for _, c := range chunks {
|
|||
|
|
if c.SeqID == 0 {
|
|||
|
|
needAssign = true
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if !needAssign {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var maxSeqID *int64
|
|||
|
|
if err := tx.Unscoped().Model(&Chunk{}).Select("MAX(seq_id)").Scan(&maxSeqID).Error; err != nil {
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
next := int64(1)
|
|||
|
|
if maxSeqID != nil {
|
|||
|
|
next = *maxSeqID + 1
|
|||
|
|
}
|
|||
|
|
for _, c := range chunks {
|
|||
|
|
if c.SeqID == 0 {
|
|||
|
|
c.SeqID = next
|
|||
|
|
next++
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return nil
|
|||
|
|
}
|