xinmi/internal/types/knowledge.go

package types

import (
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/google/uuid"
	"gorm.io/gorm"
)

const (
	// KnowledgeTypeManual represents the manual knowledge type
	KnowledgeTypeManual = "manual"
	// KnowledgeTypeFAQ represents the FAQ knowledge type
	KnowledgeTypeFAQ = "faq"
)

// Channel constants identify through which channel a knowledge entry was ingested.
// Aligned with Message.Channel values ("web", "api", "im") but allows finer granularity.
const (
	ChannelWeb              = "web"               // Web UI (default)
	ChannelAPI              = "api"               // External API call
	ChannelBrowserExtension = "browser_extension" // Browser extension / plugin
	ChannelWechat           = "wechat"            // WeChat
	ChannelWecom            = "wecom"             // WeCom (企业微信)
	ChannelFeishu           = "feishu"            // Feishu / Lark
	ChannelFeishuDrive      = "feishu_drive"      // Feishu Drive (云盘)
	ChannelLarkDrive        = "lark_drive"        // Lark Drive (international)
	ChannelDingtalk         = "dingtalk"          // DingTalk
	ChannelSlack            = "slack"             // Slack
	ChannelIM               = "im"                // Generic IM channel
	ChannelNotion           = "notion"            // Notion
	ChannelYuque            = "yuque"             // Yuque (语雀)
	ChannelRSS              = "rss"               // RSS / Atom feed
	ChannelIMA              = "ima"               // Tencent IMA (ima.qq.com)
)

// Knowledge parse status constants
const (
	// ParseStatusPending indicates the knowledge is waiting to be processed
	ParseStatusPending = "pending"
	// ParseStatusProcessing indicates the knowledge is being processed
	// (DocReader / chunking / embedding stage).
	ParseStatusProcessing = "processing"
	// ParseStatusFinalizing indicates the primary parse has finished but
	// enrichment subtasks (summary, question generation, graph extract)
	// are still in flight. The user-facing intuition behind this state is
	// "the document is queryable for vector search but is still spending
	// resources" — cancel-parse can interrupt enrichment from here.
	// pending_subtasks_count holds the outstanding subtask count; the
	// last subtask to finish atomically promotes the row to completed.
	ParseStatusFinalizing = "finalizing"
	// ParseStatusCompleted indicates the knowledge has been processed
	// successfully AND every enrichment subtask has reached a terminal
	// state. No further resources will be spent on the document until
	// the user explicitly re-parses it.
	ParseStatusCompleted = "completed"
	// ParseStatusFailed indicates the knowledge processing failed
	ParseStatusFailed = "failed"
	// ParseStatusDeleting indicates the knowledge is being deleted (used to prevent async task conflicts)
	ParseStatusDeleting = "deleting"
	// ParseStatusCancelled indicates parsing was cancelled by the user.
	// Same short-circuit semantics as ParseStatusDeleting for in-flight and
	// queued downstream tasks, but the knowledge row and any already-written
	// chunks/index are kept so the user can re-trigger parsing via reparse.
	ParseStatusCancelled = "cancelled"
)

// Summary status constants for async summary generation
const (
	// SummaryStatusNone indicates no summary task is needed
	SummaryStatusNone = "none"
	// SummaryStatusPending indicates the summary task is waiting to be processed
	SummaryStatusPending = "pending"
	// SummaryStatusProcessing indicates the summary is being generated
	SummaryStatusProcessing = "processing"
	// SummaryStatusCompleted indicates the summary has been generated successfully
	SummaryStatusCompleted = "completed"
	// SummaryStatusFailed indicates the summary generation failed
	SummaryStatusFailed = "failed"
)

// ManualKnowledgeFormat represents the format of the manual knowledge
const (
	ManualKnowledgeFormatMarkdown = "markdown"
	ManualKnowledgeStatusDraft    = "draft"
	ManualKnowledgeStatusPublish  = "publish"
)

// KnowledgeListFilter aggregates optional filters for listing knowledge entries
// under a knowledge base. Empty / zero fields mean "no filter on that dimension".
type KnowledgeListFilter struct {
	// TagIDs filters by multiple tags (OR semantics: match any of the given tags).
	TagIDs []string
	// Keyword performs a LIKE match on file_name / title when non-empty.
	Keyword string
	// FileType filters by file_type, or by type for the special values "manual" / "url".
	FileType string
	// ParseStatus filters by parse_status when non-empty (e.g. pending, processing, completed, failed).
	ParseStatus string
	// Source filters by ingestion channel when non-empty (web, api, feishu, notion, wechat, ...).
	// The special values "manual" and "url" are routed to the `type` column to match
	// FileType semantics, so callers can filter "manually created" / "URL imported" entries.
	Source string
	// UpdatedFrom, when non-zero, keeps rows with updated_at >= UpdatedFrom.
	UpdatedFrom time.Time
	// UpdatedTo, when non-zero, keeps rows with updated_at <= UpdatedTo.
	UpdatedTo time.Time
	// FolderPath is the folder navigated to in the sidebar tree. It is only
	// applied when FolderScope is not FolderScopeAny, so the empty string can
	// unambiguously mean "the knowledge base root".
	FolderPath string
	// FolderScope selects whether FolderPath matches exactly or includes
	// descendant folders. FolderScopeAny (the default) ignores folders.
	FolderScope KnowledgeFolderScope
}

// Knowledge represents a knowledge entity in the system.
// It contains metadata about the knowledge source, its processing status,
// and references to the physical file if applicable.
type Knowledge struct {
	// Unique identifier of the knowledge
	ID string `json:"id"                 gorm:"type:varchar(36);primaryKey"`
	// Workspace ID
	TenantID uint64 `json:"tenant_id"`
	// ID of the knowledge base
	KnowledgeBaseID string `json:"knowledge_base_id"`
	// Tags holds the tags associated with this knowledge (populated on query, not persisted directly).
	Tags []*KnowledgeTag `json:"tags"               gorm:"-"`
	// Type of the knowledge
	Type string `json:"type"`
	// Title of the knowledge
	Title string `json:"title"`
	// Description of the knowledge
	Description string `json:"description"`
	// DescriptionSpecified distinguishes an explicitly supplied empty description
	// from an omitted field in partial update requests.
	DescriptionSpecified bool `json:"-" gorm:"-"`
	// Source of the knowledge (e.g. URL address for url type, "manual" for manual type)
	Source string `json:"source"             gorm:"type:varchar(2048)"`
	// Channel indicates through which channel the knowledge was ingested (web, api, browser_extension, wechat, etc.)
	Channel string `json:"channel"            gorm:"type:varchar(50);default:'web'"`
	// Parse status of the knowledge
	ParseStatus string `json:"parse_status"`
	// PendingSubtasksCount is the outstanding enrichment subtask count
	// (summary + question + graph chunks). Only meaningful while
	// ParseStatus == "finalizing"; defaults to 0 in any terminal state.
	PendingSubtasksCount int `json:"pending_subtasks_count" gorm:"type:int;not null;default:0"`
	// Summary status for async summary generation
	SummaryStatus string `json:"summary_status"     gorm:"type:varchar(32);default:none"`
	// Enable status of the knowledge
	EnableStatus string `json:"enable_status"`
	// ID of the embedding model
	EmbeddingModelID string `json:"embedding_model_id"`
	// File name of the knowledge
	FileName string `json:"file_name"`
	// FolderPath is the canonical relative directory this entry belongs to
	// inside the knowledge base, e.g. "docs/spec" for a folder upload of
	// "docs/spec/design.md". Empty means the knowledge base root. It is a
	// display/navigation concern only: it never affects where the file is
	// physically stored (see FilePath).
	FolderPath string `json:"folder_path"        gorm:"type:varchar(1024);not null;default:''"`
	// File type of the knowledge
	FileType string `json:"file_type"`
	// File size of the knowledge
	FileSize int64 `json:"file_size"`
	// File hash of the knowledge
	FileHash string `json:"file_hash"`
	// File path of the knowledge
	FilePath string `json:"file_path"`
	// Storage size of the knowledge
	StorageSize int64 `json:"storage_size"`
	// Metadata of the knowledge
	Metadata JSON `json:"metadata"           gorm:"type:json"`
	// CustomMetadata is user-authored descriptive metadata. It is deliberately
	// separate from Metadata, which contains internal ingestion state and IDs.
	CustomMetadata JSON `json:"custom_metadata" gorm:"type:json;not null"`
	// Last FAQ import result (for FAQ type knowledge only)
	LastFAQImportResult JSON `json:"last_faq_import_result" gorm:"type:json"`
	// Creation time of the knowledge
	CreatedAt time.Time `json:"created_at"`
	// Last updated time of the knowledge
	UpdatedAt time.Time `json:"updated_at"`
	// Processed time of the knowledge
	ProcessedAt *time.Time `json:"processed_at"`
	// Error message of the knowledge
	ErrorMessage string `json:"error_message"`
	// Deletion time of the knowledge
	DeletedAt gorm.DeletedAt `json:"deleted_at"         gorm:"index"`
	// Knowledge base name (not stored in database, populated on query)
	KnowledgeBaseName string `json:"knowledge_base_name" gorm:"-"`
}

// CustomMetadataText returns stable human-readable metadata for summaries and
// document-scoped model context. Internal ingestion metadata is intentionally
// excluded.
func (k *Knowledge) CustomMetadataText() string {
	if k == nil || len(k.CustomMetadata) == 0 {
		return ""
	}
	var values map[string]interface{}
	if err := json.Unmarshal(k.CustomMetadata, &values); err != nil || len(values) == 0 {
		return ""
	}
	keys := make([]string, 0, len(values))
	for key := range values {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	lines := make([]string, 0, len(keys))
	for _, key := range keys {
		if values[key] == nil {
			continue
		}
		value := strings.TrimSpace(fmt.Sprint(values[key]))
		if strings.TrimSpace(key) != "" && value != "" {
			lines = append(lines, fmt.Sprintf("%s: %s", strings.TrimSpace(key), value))
		}
	}
	return strings.Join(lines, "\n")
}

// GetMetadata returns the metadata as a map[string]string.
func (k *Knowledge) GetMetadata() map[string]string {
	metadata := make(map[string]string)
	if len(k.Metadata) == 0 {
		return metadata
	}
	metadataMap, err := k.Metadata.Map()
	if err != nil {
		return nil
	}
	for k, v := range metadataMap {
		metadata[k] = fmt.Sprintf("%v", v)
	}
	return metadata
}

// BeforeCreate initializes required defaults for new Knowledge entities.
func (k *Knowledge) BeforeCreate(tx *gorm.DB) (err error) {
	if k.ID == "" {
		k.ID = uuid.New().String()
	}
	// JSON.Value returns SQL NULL for an empty value. PostgreSQL's column
	// default is not applied when GORM explicitly inserts that NULL, so keep the
	// application-side representation aligned with the NOT NULL database
	// invariant for every knowledge creation path.
	if len(k.CustomMetadata) == 0 {
		k.CustomMetadata = JSON(`{}`)
	}
	return nil
}

// ManualKnowledgeMetadata stores metadata for manual Markdown knowledge content.
type ManualKnowledgeMetadata struct {
	Content   string `json:"content"`
	Format    string `json:"format"`
	Status    string `json:"status"`
	Version   int    `json:"version"`
	UpdatedAt string `json:"updated_at"`
}

// ManualKnowledgePayload represents the payload for manual knowledge operations.
type ManualKnowledgePayload struct {
	Title         string                     `json:"title"`
	Content       string                     `json:"content"`
	Status        string                     `json:"status"`
	TagIDs        []string                   `json:"tag_ids"`
	Channel       string                     `json:"channel"`
	ProcessConfig *KnowledgeProcessOverrides `json:"process_config,omitempty"`
}

// KnowledgeSearchScope defines a (tenant_id, knowledge_base_id) scope for knowledge search (e.g. own KBs + shared KBs).
type KnowledgeSearchScope struct {
	TenantID uint64
	KBID     string
}

// NewManualKnowledgeMetadata creates a new ManualKnowledgeMetadata instance.
func NewManualKnowledgeMetadata(content, status string, version int) *ManualKnowledgeMetadata {
	if version <= 0 {
		version = 1
	}
	return &ManualKnowledgeMetadata{
		Content:   content,
		Format:    ManualKnowledgeFormatMarkdown,
		Status:    status,
		Version:   version,
		UpdatedAt: time.Now().UTC().Format(time.RFC3339),
	}
}

// ToJSON converts the metadata to JSON type.
func (m *ManualKnowledgeMetadata) ToJSON() (JSON, error) {
	if m == nil {
		return nil, nil
	}
	if m.Format == "" {
		m.Format = ManualKnowledgeFormatMarkdown
	}
	if m.Status == "" {
		m.Status = ManualKnowledgeStatusDraft
	}
	if m.Version <= 0 {
		m.Version = 1
	}
	if m.UpdatedAt == "" {
		m.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
	}
	bytes, err := json.Marshal(m)
	if err != nil {
		return nil, err
	}
	return JSON(bytes), nil
}

// ManualMetadata parses and returns manual knowledge metadata.
func (k *Knowledge) ManualMetadata() (*ManualKnowledgeMetadata, error) {
	if len(k.Metadata) == 0 {
		return nil, nil
	}
	var metadata ManualKnowledgeMetadata
	if err := json.Unmarshal(k.Metadata, &metadata); err != nil {
		return nil, err
	}
	if metadata.Format == "" {
		metadata.Format = ManualKnowledgeFormatMarkdown
	}
	if metadata.Version <= 0 {
		metadata.Version = 1
	}
	return &metadata, nil
}

// SetManualMetadata sets manual knowledge metadata onto the knowledge instance.
func (k *Knowledge) SetManualMetadata(meta *ManualKnowledgeMetadata) error {
	if meta == nil {
		k.Metadata = nil
		return nil
	}
	jsonValue, err := meta.ToJSON()
	if err != nil {
		return err
	}
	k.Metadata = jsonValue
	return nil
}

// SetLastFAQImportResult sets FAQ import result to the dedicated field.
func (k *Knowledge) SetLastFAQImportResult(result *FAQImportResult) error {
	if result == nil {
		k.LastFAQImportResult = nil
		return nil
	}
	jsonValue, err := result.ToJSON()
	if err != nil {
		return err
	}
	k.LastFAQImportResult = jsonValue
	return nil
}

// GetLastFAQImportResult parses and returns FAQ import result from the dedicated field.
func (k *Knowledge) GetLastFAQImportResult() (*FAQImportResult, error) {
	if len(k.LastFAQImportResult) == 0 {
		return nil, nil
	}
	var result FAQImportResult
	if err := json.Unmarshal(k.LastFAQImportResult, &result); err != nil {
		return nil, err
	}
	return &result, nil
}

// IsManual returns true if the knowledge item is manual Markdown knowledge.
func (k *Knowledge) IsManual() bool {
	return k != nil && k.Type == KnowledgeTypeManual
}

// EnsureManualDefaults sets default values for manual knowledge entries.
func (k *Knowledge) EnsureManualDefaults() {
	if k == nil {
		return
	}
	if k.Type == "" {
		k.Type = KnowledgeTypeManual
	}
	if k.FileType == "" {
		k.FileType = KnowledgeTypeManual
	}
	if k.Source == "" {
		k.Source = KnowledgeTypeManual
	}
	if k.Channel == "" {
		k.Channel = ChannelWeb
	}
}

// IsDraft returns whether the payload should be saved as draft.
func (p ManualKnowledgePayload) IsDraft() bool {
	return p.Status == "" || p.Status == ManualKnowledgeStatusDraft
}

const metadataKeyProcessOverrides = "process_overrides"

// ProcessOverrides parses process config overrides from knowledge metadata.
func (k *Knowledge) ProcessOverrides() (*KnowledgeProcessOverrides, error) {
	if k == nil || len(k.Metadata) == 0 {
		return nil, nil
	}
	metadataMap, err := k.Metadata.Map()
	if err != nil {
		return nil, err
	}
	raw, ok := metadataMap[metadataKeyProcessOverrides]
	if !ok || raw == nil {
		return nil, nil
	}
	bytes, err := json.Marshal(raw)
	if err != nil {
		return nil, err
	}
	var overrides KnowledgeProcessOverrides
	if err := json.Unmarshal(bytes, &overrides); err != nil {
		return nil, err
	}
	return &overrides, nil
}

// SetProcessOverrides merges process config overrides into knowledge metadata.
func (k *Knowledge) SetProcessOverrides(o *KnowledgeProcessOverrides) error {
	if k == nil {
		return nil
	}
	metadataMap, err := k.Metadata.Map()
	if err != nil {
		return err
	}
	if o == nil {
		delete(metadataMap, metadataKeyProcessOverrides)
	} else {
		bytes, err := json.Marshal(o)
		if err != nil {
			return err
		}
		var value interface{}
		if err := json.Unmarshal(bytes, &value); err != nil {
			return err
		}
		metadataMap[metadataKeyProcessOverrides] = value
	}
	bytes, err := json.Marshal(metadataMap)
	if err != nil {
		return err
	}
	k.Metadata = JSON(bytes)
	return nil
}

// KnowledgeCheckParams defines parameters used to check if knowledge already exists.
type KnowledgeCheckParams struct {
	// File parameters
	FileName string
	// FileType scopes file-hash deduplication; callers checking file uploads should set it.
	FileType string
	FileSize int64
	FileHash string
	// URL parameters
	URL string
	// Text passage parameters
	Passages []string
	// Knowledge type
	Type string
}