// Package types defines data structures and types used throughout the system // These types are shared across different service modules to ensure data consistency package types import ( "strings" "time" "gorm.io/gorm" ) // ChunkType 定义了不同类型的 Chunk type ChunkType = string const ( // ChunkTypeText 表示普通的文本 Chunk ChunkTypeText ChunkType = "text" // ChunkTypeParentText 表示父子分块策略中的父文本 Chunk(仅用于上下文,不参与向量索引) ChunkTypeParentText ChunkType = "parent_text" // ChunkTypeImageOCR 表示图片 OCR 文本的 Chunk ChunkTypeImageOCR ChunkType = "image_ocr" // ChunkTypeImageCaption 表示图片描述的 Chunk ChunkTypeImageCaption ChunkType = "image_caption" // ChunkTypeSummary 表示摘要类型的 Chunk ChunkTypeSummary = "summary" // ChunkTypeEntity 表示实体类型的 Chunk ChunkTypeEntity ChunkType = "entity" // ChunkTypeRelationship 表示关系类型的 Chunk ChunkTypeRelationship ChunkType = "relationship" // ChunkTypeFAQ 表示 FAQ 条目 Chunk ChunkTypeFAQ ChunkType = "faq" // ChunkTypeWebSearch 表示 Web 搜索结果的 Chunk ChunkTypeWebSearch ChunkType = "web_search" // ChunkTypeTableSummary 表示数据表摘要的 Chunk ChunkTypeTableSummary ChunkType = "table_summary" // ChunkTypeTableColumn 表示数据表列描述的 Chunk ChunkTypeTableColumn ChunkType = "table_column" // ChunkTypeWikiPage 表示 Wiki 页面同步的 Chunk,用于将 wiki 页面接入现有检索管线 ChunkTypeWikiPage ChunkType = "wiki_page" ) // ChunkStatus 定义了不同状态的 Chunk type ChunkStatus int const ( ChunkStatusDefault ChunkStatus = 0 // ChunkStatusStored 表示已存储的 Chunk ChunkStatusStored ChunkStatus = 1 // ChunkStatusIndexed 表示已索引的 Chunk ChunkStatusIndexed ChunkStatus = 2 ) // ChunkFlags 定义 Chunk 的标志位,用于管理多个布尔状态 type ChunkFlags int const ( // ChunkFlagRecommended 表示可推荐状态(1 << 0 = 1) // 当设置此标志时,该 Chunk 可以被推荐给用户 ChunkFlagRecommended ChunkFlags = 1 << 0 // 未来可扩展更多标志位: // ChunkFlagPinned ChunkFlags = 1 << 1 // 置顶 // ChunkFlagHot ChunkFlags = 1 << 2 // 热门 ) // HasFlag 检查是否设置了指定标志 func (f ChunkFlags) HasFlag(flag ChunkFlags) bool { return f&flag != 0 } // SetFlag 设置指定标志 func (f ChunkFlags) SetFlag(flag ChunkFlags) ChunkFlags { return f | flag } // ClearFlag 清除指定标志 func (f ChunkFlags) ClearFlag(flag ChunkFlags) ChunkFlags { return f &^ flag } // ToggleFlag 切换指定标志 func (f ChunkFlags) ToggleFlag(flag ChunkFlags) ChunkFlags { return f ^ flag } // ImageInfo 表示与 Chunk 关联的图片信息 type ImageInfo struct { // 图片URL(COS) URL string `json:"url" gorm:"type:text"` // 原始图片URL OriginalURL string `json:"original_url" gorm:"type:text"` // 图片在文本中的开始位置 StartPos int `json:"start_pos"` // 图片在文本中的结束位置 EndPos int `json:"end_pos"` // 图片描述 Caption string `json:"caption"` // 图片OCR文本 OCRText string `json:"ocr_text"` } // VideoInfo 表示与 Chunk 关联的视频信息 type VideoInfo struct { // 视频URL URL string `json:"url" gorm:"type:text"` } // Chunk represents a document chunk // Chunks are meaningful text segments extracted from original documents // and are the basic units of knowledge base retrieval // Each chunk contains a portion of the original content // and maintains its positional relationship with the original text // Chunks can be independently embedded as vectors and retrieved, supporting precise content localization type Chunk struct { // Unique identifier of the chunk, using UUID format ID string `json:"id" gorm:"type:varchar(36);primaryKey"` // SeqID is an auto-increment integer ID for external API usage (FAQ entries) SeqID int64 `json:"seq_id" gorm:"type:bigint;uniqueIndex;autoIncrement"` // Tenant ID, used for multi-tenant isolation TenantID uint64 `json:"tenant_id"` // ID of the parent knowledge, associated with the Knowledge model KnowledgeID string `json:"knowledge_id"` // ID of the knowledge base, for quick location KnowledgeBaseID string `json:"knowledge_base_id"` // Optional tag ID for categorization within a knowledge base (used for FAQ) TagID string `json:"tag_id" gorm:"type:varchar(36);index"` // Actual text content of the chunk Content string `json:"content"` // SourceContent is the immutable parser output. Legacy rows are lazily // backfilled from Content on the first manual edit. SourceContent string `json:"-"` // ContentRevision is incremented for every user edit or rollback. ContentRevision int `json:"content_revision" gorm:"not null;default:0"` // IndexStatus reports whether the current content is reflected in the // retrieval stores: ready | processing | failed. IndexStatus string `json:"index_status" gorm:"type:varchar(16);not null;default:'ready'"` // LastEditorID records the actor that produced the current revision. LastEditorID string `json:"last_editor_id" gorm:"type:varchar(64);not null;default:''"` // Index position of the chunk in the original document ChunkIndex int `json:"chunk_index"` // Whether the chunk is enabled, can be used to temporarily disable certain chunks IsEnabled bool `json:"is_enabled" gorm:"default:true"` // Flags 存储多个布尔状态的位标志(如推荐状态等) // 默认值为 ChunkFlagRecommended (1),表示默认可推荐 Flags ChunkFlags `json:"flags" gorm:"default:1"` // Status of the chunk Status int `json:"status" gorm:"default:0"` // Starting character position in the original text StartAt int `json:"start_at"` // Ending character position in the original text EndAt int `json:"end_at"` // Previous chunk ID PreChunkID string `json:"pre_chunk_id"` // Next chunk ID NextChunkID string `json:"next_chunk_id"` // Chunk 类型,用于区分不同类型的 Chunk ChunkType ChunkType `json:"chunk_type" gorm:"type:varchar(20);default:'text'"` // 父 Chunk ID,用于关联图片 Chunk 和原始文本 Chunk ParentChunkID string `json:"parent_chunk_id" gorm:"type:varchar(36);index"` // 关系 Chunk ID,用于关联关系 Chunk 和原始文本 Chunk RelationChunks JSON `json:"relation_chunks" gorm:"type:json"` // 间接关系 Chunk ID,用于关联间接关系 Chunk 和原始文本 Chunk IndirectRelationChunks JSON `json:"indirect_relation_chunks" gorm:"type:json"` // Metadata 存储 chunk 级别的扩展信息,例如 FAQ 元数据 Metadata JSON `json:"metadata" gorm:"type:json"` // ContentHash 存储内容的 hash 值,用于快速匹配(主要用于 FAQ) ContentHash string `json:"content_hash" gorm:"type:varchar(64);index"` // 图片信息,存储为 JSON ImageInfo string `json:"image_info" gorm:"type:text"` // Chunk creation time CreatedAt time.Time `json:"created_at"` // Chunk last update time UpdatedAt time.Time `json:"updated_at"` // Soft delete marker, supports data recovery DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"` // ContextHeader is a Markdown heading breadcrumb prepended when indexing. // It is persisted so a later content edit can rebuild the same index input. ContextHeader string `json:"-" gorm:"type:text"` } // ChunkRevision is an immutable snapshot of a superseded chunk revision. // The current content lives on Chunk; this table stores prior versions. type ChunkRevision struct { ID string `json:"id" gorm:"type:varchar(36);primaryKey"` TenantID uint64 `json:"tenant_id" gorm:"index"` KnowledgeBaseID string `json:"knowledge_base_id" gorm:"type:varchar(36);index"` KnowledgeID string `json:"knowledge_id" gorm:"type:varchar(36);index"` ChunkID string `json:"chunk_id" gorm:"type:varchar(36);uniqueIndex:idx_chunk_revision"` Revision int `json:"revision" gorm:"uniqueIndex:idx_chunk_revision"` Content string `json:"content" gorm:"type:text"` IsEnabled bool `json:"is_enabled"` EditorID string `json:"editor_id" gorm:"type:varchar(64)"` EditSource string `json:"edit_source" gorm:"type:varchar(16)"` EditedAt time.Time `json:"edited_at"` CreatedAt time.Time `json:"created_at"` } // EmbeddingContent returns the chunk content with ContextHeader prepended // when set. Use this where the embedding model needs section context that // isn't part of the literal Content. Surrounding whitespace on Content is // trimmed so leading/trailing newlines from boundary slicing don't dilute // the embedded vector. func (c *Chunk) EmbeddingContent() string { if c == nil { return "" } body := strings.TrimSpace(c.Content) if c.ContextHeader == "" { return body } return c.ContextHeader + "\n\n" + body } // AssignChunkSeqIDs assigns sequential SeqIDs to a batch of chunks that have SeqID == 0. // Must be called before CreateInBatches for SQLite compatibility. func AssignChunkSeqIDs(tx *gorm.DB, chunks []*Chunk) error { needAssign := false for _, c := range chunks { if c.SeqID == 0 { needAssign = true break } } if !needAssign { return nil } var maxSeqID *int64 if err := tx.Unscoped().Model(&Chunk{}).Select("MAX(seq_id)").Scan(&maxSeqID).Error; err != nil { return err } next := int64(1) if maxSeqID != nil { next = *maxSeqID + 1 } for _, c := range chunks { if c.SeqID == 0 { c.SeqID = next next++ } } return nil }