WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

migrations/versioned/000000_init.down.sql

@@ -0,0 +1,55 @@
+-- Drop indexes for chunks
+DROP INDEX IF EXISTS idx_chunks_tenant_kg;
+DROP INDEX IF EXISTS idx_chunks_parent_id;
+DROP INDEX IF EXISTS idx_chunks_chunk_type;
+
+-- Drop chunks table
+DROP TABLE IF EXISTS chunks;
+
+-- Drop indexes for messages
+DROP INDEX IF EXISTS idx_messages_session_id;
+
+-- Drop messages table
+DROP TABLE IF EXISTS messages;
+
+-- Drop indexes for sessions
+DROP INDEX IF EXISTS idx_sessions_tenant_id;
+
+-- Drop sessions table
+DROP TABLE IF EXISTS sessions;
+
+-- Drop indexes for knowledges
+DROP INDEX IF EXISTS idx_knowledges_tenant_id;
+DROP INDEX IF EXISTS idx_knowledges_base_id;
+DROP INDEX IF EXISTS idx_knowledges_parse_status;
+DROP INDEX IF EXISTS idx_knowledges_enable_status;
+
+-- Drop knowledges table
+DROP TABLE IF EXISTS knowledges;
+
+-- Drop indexes for knowledge_bases
+DROP INDEX IF EXISTS idx_knowledge_bases_tenant_id;
+
+-- Drop knowledge_bases table
+DROP TABLE IF EXISTS knowledge_bases;
+
+-- Drop indexes for models
+DROP INDEX IF EXISTS idx_models_type;
+DROP INDEX IF EXISTS idx_models_source;
+
+-- Drop models table
+DROP TABLE IF EXISTS models;
+
+-- Drop indexes for tenants
+DROP INDEX IF EXISTS idx_tenants_api_key;
+DROP INDEX IF EXISTS idx_tenants_status;
+
+-- Drop tenants table
+DROP TABLE IF EXISTS tenants;
+
+-- Note: Extensions are not dropped as they may be used by other databases/schemas
+-- If you want to drop extensions, uncomment the following lines:
+-- DROP EXTENSION IF EXISTS pg_search;
+-- DROP EXTENSION IF EXISTS pg_trgm;
+-- DROP EXTENSION IF EXISTS vector;
+-- DROP EXTENSION IF EXISTS "uuid-ossp";

migrations/versioned/000000_init.up.sql

@@ -0,0 +1,205 @@
+-- Migration: 000000_init
+-- Description: Initialize database schema
+DO $$ BEGIN RAISE NOTICE '[Migration 000000] Starting initial database setup...'; END $$;
+
+-- Create extensions
+DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating extensions...'; END $$;
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+-- Create tenant table
+DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: tenants'; END $$;
+CREATE TABLE IF NOT EXISTS tenants (
+    id SERIAL PRIMARY KEY,
+    name VARCHAR(255) NOT NULL,
+    description TEXT,
+    api_key VARCHAR(64) NOT NULL,
+    retriever_engines JSONB NOT NULL DEFAULT '[]',
+    status VARCHAR(50) DEFAULT 'active',
+    business VARCHAR(255) NOT NULL,
+    storage_quota BIGINT NOT NULL DEFAULT 10737418240, -- 默认10GB配额(Bytes)
+    storage_used BIGINT NOT NULL DEFAULT 0, -- 已使用的存储空间(Bytes)
+    agent_config JSONB DEFAULT NULL,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP WITH TIME ZONE
+);
+
+-- Set the starting value for tenants id sequence (only if current value is less than 10000)
+DO $$
+DECLARE
+    current_val BIGINT;
+BEGIN
+    SELECT last_value INTO current_val FROM tenants_id_seq;
+    IF current_val < 10000 THEN
+        ALTER SEQUENCE tenants_id_seq RESTART WITH 10000;
+        RAISE NOTICE '[Migration 000000] Set tenants_id_seq to start at 10000';
+    ELSE
+        RAISE NOTICE '[Migration 000000] tenants_id_seq already at % (>= 10000), skipping', current_val;
+    END IF;
+EXCEPTION
+    WHEN undefined_table THEN
+        -- Sequence doesn't exist yet, will be created with table
+        RAISE NOTICE '[Migration 000000] tenants_id_seq not found, will be created with table';
+END $$;
+
+-- Add indexes
+CREATE INDEX IF NOT EXISTS idx_tenants_api_key ON tenants(api_key);
+CREATE INDEX IF NOT EXISTS idx_tenants_status ON tenants(status);
+
+-- Create model table
+DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: models'; END $$;
+CREATE TABLE IF NOT EXISTS models (
+    id VARCHAR(64) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    tenant_id INTEGER NOT NULL,
+    name VARCHAR(255) NOT NULL,
+    display_name VARCHAR(255) NOT NULL DEFAULT '',
+    type VARCHAR(50) NOT NULL,
+    source VARCHAR(50) NOT NULL,
+    description TEXT,
+    parameters JSONB NOT NULL,
+    is_default BOOLEAN NOT NULL DEFAULT false,
+    status VARCHAR(50) NOT NULL DEFAULT 'active',
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP WITH TIME ZONE
+);  
+
+-- Add indexes for models
+CREATE INDEX IF NOT EXISTS idx_models_type ON models(type);
+CREATE INDEX IF NOT EXISTS idx_models_source ON models(source);
+
+-- Create knowledge_base table
+DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: knowledge_bases'; END $$;
+CREATE TABLE IF NOT EXISTS knowledge_bases (
+    id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    name VARCHAR(255) NOT NULL,
+    description TEXT,
+    tenant_id INTEGER NOT NULL,
+    chunking_config JSONB NOT NULL DEFAULT '{"chunk_size": 512, "chunk_overlap": 50, "split_markers": ["\n\n", "\n", "。"], "keep_separator": true}',
+    image_processing_config JSONB NOT NULL DEFAULT '{"enable_multimodal": false, "model_id": ""}',
+    embedding_model_id VARCHAR(64) NOT NULL,
+    summary_model_id VARCHAR(64) NOT NULL,
+    rerank_model_id VARCHAR(64) NOT NULL,
+    cos_config JSONB NOT NULL DEFAULT '{}',
+    vlm_config JSONB NOT NULL DEFAULT '{}',
+    extract_config JSONB NULL DEFAULT NULL,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP WITH TIME ZONE
+);
+
+-- Add indexes for knowledge_bases
+CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_id ON knowledge_bases(tenant_id);
+
+-- Create knowledge table
+DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: knowledges'; END $$;
+CREATE TABLE IF NOT EXISTS knowledges (
+    id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    tenant_id INTEGER NOT NULL,
+    knowledge_base_id VARCHAR(36) NOT NULL,
+    type VARCHAR(50) NOT NULL,
+    title VARCHAR(255) NOT NULL,
+    description TEXT,
+    source VARCHAR(2048) NOT NULL,
+    parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed',
+    enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled',
+    embedding_model_id VARCHAR(64),
+    file_name VARCHAR(255),
+    file_type VARCHAR(50),
+    file_size BIGINT,
+    file_path TEXT,
+    file_hash VARCHAR(64),
+    storage_size BIGINT NOT NULL DEFAULT 0, -- 存储大小(Byte)
+    metadata JSONB,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    processed_at TIMESTAMP WITH TIME ZONE,
+    error_message TEXT,
+    deleted_at TIMESTAMP WITH TIME ZONE
+);
+
+-- Add indexes for knowledge
+CREATE INDEX IF NOT EXISTS idx_knowledges_tenant_id ON knowledges(tenant_id);
+CREATE INDEX IF NOT EXISTS idx_knowledges_base_id ON knowledges(knowledge_base_id);
+CREATE INDEX IF NOT EXISTS idx_knowledges_parse_status ON knowledges(parse_status);
+CREATE INDEX IF NOT EXISTS idx_knowledges_enable_status ON knowledges(enable_status);
+
+-- Create session table
+DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: sessions'; END $$;
+CREATE TABLE IF NOT EXISTS sessions (
+    id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    tenant_id INTEGER NOT NULL,
+    title VARCHAR(255),
+    description TEXT,
+    knowledge_base_id VARCHAR(36),
+    max_rounds INTEGER NOT NULL DEFAULT 5,
+    enable_rewrite BOOLEAN NOT NULL DEFAULT true,
+    fallback_strategy VARCHAR(255) NOT NULL DEFAULT 'fixed',
+    fallback_response TEXT NOT NULL DEFAULT '很抱歉，我暂时无法回答这个问题。',
+    keyword_threshold FLOAT NOT NULL DEFAULT 0.5,
+    vector_threshold FLOAT NOT NULL DEFAULT 0.5,
+    rerank_model_id VARCHAR(64),
+    embedding_top_k INTEGER NOT NULL DEFAULT 10,
+    rerank_top_k INTEGER NOT NULL DEFAULT 10,
+    rerank_threshold FLOAT NOT NULL DEFAULT 0.65,
+    summary_model_id VARCHAR(64),
+    summary_parameters JSONB NOT NULL DEFAULT '{}',
+    agent_config JSONB DEFAULT NULL,
+    context_config JSONB DEFAULT NULL,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP WITH TIME ZONE
+);
+
+-- Create Index for sessions
+CREATE INDEX IF NOT EXISTS idx_sessions_tenant_id ON sessions(tenant_id);
+
+
+-- Create message table
+DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: messages'; END $$;
+CREATE TABLE IF NOT EXISTS messages (
+    id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    request_id VARCHAR(36) NOT NULL,
+    session_id VARCHAR(36) NOT NULL,
+    role VARCHAR(50) NOT NULL,
+    content TEXT NOT NULL,
+    knowledge_references JSONB NOT NULL DEFAULT '[]',
+    agent_steps JSONB DEFAULT NULL,
+    is_completed BOOLEAN NOT NULL DEFAULT false,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP WITH TIME ZONE
+);
+
+-- Create Index for messages
+CREATE INDEX IF NOT EXISTS idx_messages_session_id ON messages(session_id); 
+
+-- Create chunks table
+DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: chunks'; END $$;
+CREATE TABLE IF NOT EXISTS chunks (
+    id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    tenant_id INTEGER NOT NULL,
+    knowledge_base_id VARCHAR(36) NOT NULL,
+    knowledge_id VARCHAR(36) NOT NULL,
+    content TEXT NOT NULL,
+    chunk_index INTEGER NOT NULL,
+    is_enabled BOOLEAN NOT NULL DEFAULT true,
+    start_at INTEGER NOT NULL,
+    end_at INTEGER NOT NULL,
+    pre_chunk_id VARCHAR(36),
+    next_chunk_id VARCHAR(36),
+    chunk_type VARCHAR(20) NOT NULL DEFAULT 'text',
+    parent_chunk_id VARCHAR(36),
+    image_info TEXT,
+    relation_chunks JSONB,
+    indirect_relation_chunks JSONB,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP WITH TIME ZONE
+);
+
+CREATE INDEX IF NOT EXISTS idx_chunks_tenant_kg ON chunks(tenant_id, knowledge_id);
+CREATE INDEX IF NOT EXISTS idx_chunks_parent_id ON chunks(parent_chunk_id);
+CREATE INDEX IF NOT EXISTS idx_chunks_chunk_type ON chunks(chunk_type);
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000000] Initial database setup completed successfully!'; END $$;

migrations/versioned/000001_agent.down.sql

@@ -0,0 +1,130 @@
+BEGIN;
+
+DROP INDEX IF EXISTS idx_knowledges_summary_status;
+ALTER TABLE knowledges DROP COLUMN IF EXISTS summary_status;
+
+ALTER TABLE knowledge_bases 
+DROP COLUMN IF EXISTS question_generation_config;
+
+ALTER TABLE users
+    DROP COLUMN IF EXISTS can_access_all_tenants;
+
+ALTER TABLE knowledge_bases
+    ADD COLUMN IF NOT EXISTS rerank_model_id VARCHAR(64);
+
+UPDATE models m
+JOIN knowledges k ON m.id = k.id
+SET m.tenant_id = 0
+WHERE m.tenant_id = k.tenant_id;
+
+-- Drop index on content_hash
+DROP INDEX IF EXISTS idx_chunks_content_hash;
+
+-- Drop content_hash column
+ALTER TABLE chunks
+    DROP COLUMN IF EXISTS content_hash;
+
+-- Drop status column
+ALTER TABLE chunks
+    DROP COLUMN IF EXISTS status;
+
+-- Drop indexes and columns referencing tags
+DROP INDEX IF EXISTS idx_chunks_tag;
+ALTER TABLE chunks DROP COLUMN IF EXISTS tag_id;
+
+DROP INDEX IF EXISTS idx_knowledges_tag;
+ALTER TABLE knowledges DROP COLUMN IF EXISTS tag_id;
+
+-- Drop tag table
+DROP INDEX IF EXISTS idx_knowledge_tags_kb_name;
+DROP INDEX IF EXISTS idx_knowledge_tags_kb;
+DROP TABLE IF EXISTS knowledge_tags;
+
+ALTER TABLE chunks
+  DROP COLUMN IF EXISTS metadata;
+
+ALTER TABLE knowledge_bases
+  DROP COLUMN IF EXISTS faq_config,
+  DROP COLUMN IF EXISTS type;
+
+-- Drop index
+DROP INDEX IF EXISTS idx_models_is_builtin;
+
+-- Remove is_builtin column
+ALTER TABLE models 
+DROP COLUMN IF EXISTS is_builtin;
+
+-- Remove check constraint
+ALTER TABLE mcp_services
+DROP CONSTRAINT IF EXISTS chk_mcp_transport_config;
+
+-- Make url column required again
+ALTER TABLE mcp_services 
+ALTER COLUMN url SET NOT NULL;
+
+-- Remove stdio_config and env_vars columns
+ALTER TABLE mcp_services 
+DROP COLUMN IF EXISTS env_vars,
+DROP COLUMN IF EXISTS stdio_config;
+
+-- Remove web_search_config column
+ALTER TABLE tenants 
+DROP COLUMN IF EXISTS web_search_config;
+
+-- Drop trigger
+DROP TRIGGER IF EXISTS trigger_mcp_services_updated_at ON mcp_services;
+
+-- Drop function
+DROP FUNCTION IF EXISTS update_mcp_services_updated_at();
+
+-- Drop indexes
+DROP INDEX IF EXISTS idx_mcp_services_tenant_id;
+DROP INDEX IF EXISTS idx_mcp_services_enabled;
+DROP INDEX IF EXISTS idx_mcp_services_deleted_at;
+
+-- Drop table
+DROP TABLE IF EXISTS mcp_services;
+
+-- This migration performs a data cleanup (soft delete) which is not safely reversible.
+-- Down migration is a no-op.
+
+-- Drop foreign key constraints first
+ALTER TABLE auth_tokens DROP CONSTRAINT IF EXISTS fk_auth_tokens_user;
+ALTER TABLE users DROP CONSTRAINT IF EXISTS fk_users_tenant;
+
+-- Drop tables
+DROP TABLE IF EXISTS auth_tokens;
+DROP TABLE IF EXISTS users;
+
+-- Drop is_temporary column from knowledge_bases
+ALTER TABLE knowledge_bases
+    DROP COLUMN IF EXISTS is_temporary;
+
+-- Drop context_config column from tenants
+ALTER TABLE tenants
+    DROP COLUMN IF EXISTS context_config;
+
+-- Drop conversation_config column from tenants
+ALTER TABLE tenants
+    DROP COLUMN IF EXISTS conversation_config;
+
+-- Drop JSONB indexes if they exist
+DROP INDEX IF EXISTS idx_messages_agent_steps;
+DROP INDEX IF EXISTS idx_sessions_context_config;
+DROP INDEX IF EXISTS idx_sessions_agent_config;
+DROP INDEX IF EXISTS idx_tenants_agent_config;
+
+-- Drop columns if they exist
+ALTER TABLE messages
+    DROP COLUMN IF EXISTS agent_steps;
+
+ALTER TABLE sessions
+    DROP COLUMN IF EXISTS context_config;
+
+ALTER TABLE sessions
+    DROP COLUMN IF EXISTS agent_config;
+
+ALTER TABLE tenants
+    DROP COLUMN IF EXISTS agent_config;
+
+COMMIT;

migrations/versioned/000001_agent.up.sql

@@ -0,0 +1,408 @@
+-- Migration: 000001_agent
+-- Description: Add user authentication, agent config, MCP services and other enhancements
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Starting agent and authentication migration...'; END $$;
+
+-- ============================================================================
+-- Section 1: User Authentication Tables
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Creating table: users'; END $$;
+CREATE TABLE IF NOT EXISTS users (
+    id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    username VARCHAR(100) NOT NULL,
+    email VARCHAR(255) NOT NULL,
+    password_hash VARCHAR(255) NOT NULL,
+    avatar VARCHAR(500),
+    tenant_id INTEGER,
+    is_active BOOLEAN NOT NULL DEFAULT true,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP WITH TIME ZONE
+);
+
+-- Add unique constraints if not exists
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'users_username_key') THEN
+        ALTER TABLE users ADD CONSTRAINT users_username_key UNIQUE (username);
+        RAISE NOTICE '[Migration 000001] Added unique constraint on users.username';
+    END IF;
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'users_email_key') THEN
+        ALTER TABLE users ADD CONSTRAINT users_email_key UNIQUE (email);
+        RAISE NOTICE '[Migration 000001] Added unique constraint on users.email';
+    END IF;
+END $$;
+
+COMMENT ON TABLE users IS 'User accounts in the system';
+COMMENT ON COLUMN users.id IS 'Unique identifier of the user';
+COMMENT ON COLUMN users.username IS 'Username of the user';
+COMMENT ON COLUMN users.email IS 'Email address of the user';
+COMMENT ON COLUMN users.password_hash IS 'Hashed password of the user';
+COMMENT ON COLUMN users.avatar IS 'Avatar URL of the user';
+COMMENT ON COLUMN users.tenant_id IS 'Tenant ID that the user belongs to';
+COMMENT ON COLUMN users.is_active IS 'Whether the user is active';
+
+-- Add indexes for users
+CREATE INDEX IF NOT EXISTS idx_users_username ON users(username);
+CREATE INDEX IF NOT EXISTS idx_users_email ON users(email);
+CREATE INDEX IF NOT EXISTS idx_users_tenant_id ON users(tenant_id);
+CREATE INDEX IF NOT EXISTS idx_users_deleted_at ON users(deleted_at);
+
+-- Add foreign key constraint for tenant_id
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_users_tenant') THEN
+        ALTER TABLE users ADD CONSTRAINT fk_users_tenant
+            FOREIGN KEY (tenant_id) REFERENCES tenants(id) ON DELETE SET NULL;
+        RAISE NOTICE '[Migration 000001] Added foreign key constraint fk_users_tenant';
+    END IF;
+END $$;
+
+-- Add can_access_all_tenants column to users
+ALTER TABLE users ADD COLUMN IF NOT EXISTS can_access_all_tenants BOOLEAN NOT NULL DEFAULT FALSE;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Creating table: auth_tokens'; END $$;
+CREATE TABLE IF NOT EXISTS auth_tokens (
+    id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    user_id VARCHAR(36) NOT NULL,
+    token TEXT NOT NULL,
+    token_type VARCHAR(50) NOT NULL,
+    expires_at TIMESTAMP WITH TIME ZONE NOT NULL,
+    is_revoked BOOLEAN NOT NULL DEFAULT false,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+COMMENT ON TABLE auth_tokens IS 'Authentication tokens for users';
+COMMENT ON COLUMN auth_tokens.id IS 'Unique identifier of the token';
+COMMENT ON COLUMN auth_tokens.user_id IS 'User ID that owns this token';
+COMMENT ON COLUMN auth_tokens.token IS 'Token value (JWT or other format)';
+COMMENT ON COLUMN auth_tokens.token_type IS 'Token type (access_token, refresh_token)';
+COMMENT ON COLUMN auth_tokens.expires_at IS 'Token expiration time';
+COMMENT ON COLUMN auth_tokens.is_revoked IS 'Whether the token is revoked';
+
+-- Add indexes for auth_tokens
+CREATE INDEX IF NOT EXISTS idx_auth_tokens_user_id ON auth_tokens(user_id);
+CREATE INDEX IF NOT EXISTS idx_auth_tokens_token ON auth_tokens(token);
+CREATE INDEX IF NOT EXISTS idx_auth_tokens_token_type ON auth_tokens(token_type);
+CREATE INDEX IF NOT EXISTS idx_auth_tokens_expires_at ON auth_tokens(expires_at);
+
+-- Add foreign key constraint for auth_tokens
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_auth_tokens_user') THEN
+        ALTER TABLE auth_tokens ADD CONSTRAINT fk_auth_tokens_user
+            FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;
+        RAISE NOTICE '[Migration 000001] Added foreign key constraint fk_auth_tokens_user';
+    END IF;
+END $$;
+
+-- ============================================================================
+-- Section 2: Tenant Configuration Enhancements
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding tenant configuration columns...'; END $$;
+
+-- Add context_config column to tenants
+ALTER TABLE tenants ADD COLUMN IF NOT EXISTS context_config JSONB;
+COMMENT ON COLUMN tenants.context_config IS 'Global Context configuration for this tenant (default for all sessions)';
+
+-- Add conversation_config column to tenants
+ALTER TABLE tenants ADD COLUMN IF NOT EXISTS conversation_config JSONB;
+COMMENT ON COLUMN tenants.conversation_config IS 'Global Conversation configuration for this tenant (default for normal mode sessions)';
+
+-- Add web_search_config column to tenants
+ALTER TABLE tenants ADD COLUMN IF NOT EXISTS web_search_config JSONB DEFAULT NULL;
+COMMENT ON COLUMN tenants.web_search_config IS 'Web search configuration for the tenant';
+
+-- Ensure agent_config exists and is JSONB type
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM information_schema.columns 
+        WHERE table_name = 'tenants' AND column_name = 'agent_config'
+    ) THEN
+        ALTER TABLE tenants ADD COLUMN agent_config JSONB DEFAULT NULL;
+        RAISE NOTICE '[Migration 000001] Added agent_config column to tenants table';
+    ELSE
+        -- If field exists but type is JSON, convert to JSONB
+        IF EXISTS (
+            SELECT 1 FROM information_schema.columns 
+            WHERE table_name = 'tenants' AND column_name = 'agent_config' AND data_type = 'json'
+        ) THEN
+            ALTER TABLE tenants ALTER COLUMN agent_config TYPE JSONB USING agent_config::jsonb;
+            RAISE NOTICE '[Migration 000001] Converted tenants.agent_config from JSON to JSONB';
+        END IF;
+    END IF;
+END $$;
+COMMENT ON COLUMN tenants.agent_config IS 'Tenant-level agent configuration in JSON format';
+
+-- ============================================================================
+-- Section 3: Session Configuration Enhancements
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding session configuration columns...'; END $$;
+
+-- Add agent_config column to sessions
+ALTER TABLE sessions ADD COLUMN IF NOT EXISTS agent_config JSONB DEFAULT NULL;
+COMMENT ON COLUMN sessions.agent_config IS 'Session-level agent configuration in JSON format';
+
+-- Add context_config column to sessions
+ALTER TABLE sessions ADD COLUMN IF NOT EXISTS context_config JSONB DEFAULT NULL;
+COMMENT ON COLUMN sessions.context_config IS 'LLM context management configuration (separate from message storage)';
+
+-- ============================================================================
+-- Section 4: Messages Enhancements
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding messages enhancements...'; END $$;
+
+-- Add agent_steps column to messages
+ALTER TABLE messages ADD COLUMN IF NOT EXISTS agent_steps JSONB DEFAULT NULL;
+COMMENT ON COLUMN messages.agent_steps IS 'Agent execution steps (reasoning process and tool calls)';
+
+-- ============================================================================
+-- Section 5: Knowledge Base Enhancements
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding knowledge base enhancements...'; END $$;
+
+-- Add is_temporary column to knowledge_bases
+ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS is_temporary BOOLEAN NOT NULL DEFAULT false;
+COMMENT ON COLUMN knowledge_bases.is_temporary IS 'Whether this knowledge base is temporary (ephemeral) and should be hidden from UI';
+
+-- Add type and faq_config columns
+ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS type VARCHAR(32) NOT NULL DEFAULT 'document';
+ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS faq_config JSONB;
+
+-- Add question_generation_config column
+ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS question_generation_config JSONB NULL;
+
+-- Update existing rows with default type
+UPDATE knowledge_bases SET type = 'document' WHERE type IS NULL OR type = '';
+
+-- Drop rerank_model_id column if exists (moved to session level)
+ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS rerank_model_id;
+
+-- ============================================================================
+-- Section 6: Knowledges Enhancements
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding knowledges enhancements...'; END $$;
+
+-- Add tag_id column
+ALTER TABLE knowledges ADD COLUMN IF NOT EXISTS tag_id VARCHAR(36);
+CREATE INDEX IF NOT EXISTS idx_knowledges_tag ON knowledges(tag_id);
+
+-- Add summary_status column
+ALTER TABLE knowledges ADD COLUMN IF NOT EXISTS summary_status VARCHAR(32) DEFAULT 'none';
+CREATE INDEX IF NOT EXISTS idx_knowledges_summary_status ON knowledges(summary_status);
+
+-- ============================================================================
+-- Section 7: Chunks Enhancements
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding chunks enhancements...'; END $$;
+
+-- Add metadata column
+ALTER TABLE chunks ADD COLUMN IF NOT EXISTS metadata JSONB;
+
+-- Add tag_id column
+ALTER TABLE chunks ADD COLUMN IF NOT EXISTS tag_id VARCHAR(36);
+CREATE INDEX IF NOT EXISTS idx_chunks_tag ON chunks(tag_id);
+
+-- Add status field to track chunk processing state
+ALTER TABLE chunks ADD COLUMN IF NOT EXISTS status INT NOT NULL DEFAULT 0;
+
+-- Add content_hash field for quick content matching
+ALTER TABLE chunks ADD COLUMN IF NOT EXISTS content_hash VARCHAR(64);
+CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);
+
+-- ============================================================================
+-- Section 8: Embeddings Enhancements
+-- ============================================================================
+
+-- move embeddings to 000002 migrations
+
+-- ============================================================================
+-- Section 9: Models Enhancements
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding models enhancements...'; END $$;
+
+-- Add is_builtin column
+ALTER TABLE models ADD COLUMN IF NOT EXISTS is_builtin BOOLEAN NOT NULL DEFAULT false;
+CREATE INDEX IF NOT EXISTS idx_models_is_builtin ON models(is_builtin);
+
+-- ============================================================================
+-- Section 10: Knowledge Tags Table
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Creating table: knowledge_tags'; END $$;
+CREATE TABLE IF NOT EXISTS knowledge_tags (
+    id VARCHAR(36) PRIMARY KEY,
+    tenant_id INTEGER NOT NULL,
+    knowledge_base_id VARCHAR(36) NOT NULL,
+    name VARCHAR(128) NOT NULL,
+    color VARCHAR(32),
+    sort_order INTEGER NOT NULL DEFAULT 0,
+    created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMPTZ
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_knowledge_tags_kb_name ON knowledge_tags(tenant_id, knowledge_base_id, name);
+CREATE INDEX IF NOT EXISTS idx_knowledge_tags_kb ON knowledge_tags(tenant_id, knowledge_base_id);
+
+-- ============================================================================
+-- Section 11: MCP Services Table
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Creating table: mcp_services'; END $$;
+CREATE TABLE IF NOT EXISTS mcp_services (
+    id VARCHAR(36) PRIMARY KEY,
+    tenant_id INTEGER NOT NULL,
+    name VARCHAR(255) NOT NULL,
+    description TEXT,
+    enabled BOOLEAN DEFAULT true,
+    transport_type VARCHAR(50) NOT NULL,
+    url VARCHAR(512),
+    headers JSONB,
+    auth_config JSONB,
+    advanced_config JSONB,
+    stdio_config JSONB,
+    env_vars JSONB,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS idx_mcp_services_tenant_id ON mcp_services(tenant_id);
+CREATE INDEX IF NOT EXISTS idx_mcp_services_enabled ON mcp_services(enabled);
+CREATE INDEX IF NOT EXISTS idx_mcp_services_deleted_at ON mcp_services(deleted_at);
+
+COMMENT ON TABLE mcp_services IS 'MCP service configurations';
+
+-- Create or replace trigger function for updated_at
+CREATE OR REPLACE FUNCTION update_mcp_services_updated_at()
+RETURNS TRIGGER AS $$
+BEGIN
+    NEW.updated_at = CURRENT_TIMESTAMP;
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Create trigger if not exists
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname = 'trigger_mcp_services_updated_at') THEN
+        CREATE TRIGGER trigger_mcp_services_updated_at
+            BEFORE UPDATE ON mcp_services
+            FOR EACH ROW
+            EXECUTE FUNCTION update_mcp_services_updated_at();
+        RAISE NOTICE '[Migration 000001] Created trigger trigger_mcp_services_updated_at';
+    END IF;
+END $$;
+
+-- ============================================================================
+-- Section 12: GIN Indexes for JSONB Fields
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Creating GIN indexes for JSONB fields...'; END $$;
+
+DO $$
+BEGIN
+    -- For tenants.agent_config
+    IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'idx_tenants_agent_config') THEN
+        IF EXISTS (
+            SELECT 1 FROM information_schema.columns 
+            WHERE table_name = 'tenants' AND column_name = 'agent_config' AND data_type = 'jsonb'
+        ) THEN
+            CREATE INDEX idx_tenants_agent_config ON tenants USING GIN (agent_config);
+            RAISE NOTICE '[Migration 000001] Created index idx_tenants_agent_config';
+        END IF;
+    END IF;
+    
+    -- For sessions.agent_config
+    IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'idx_sessions_agent_config') THEN
+        CREATE INDEX idx_sessions_agent_config ON sessions USING GIN (agent_config);
+        RAISE NOTICE '[Migration 000001] Created index idx_sessions_agent_config';
+    END IF;
+    
+    -- For sessions.context_config
+    IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'idx_sessions_context_config') THEN
+        CREATE INDEX idx_sessions_context_config ON sessions USING GIN (context_config);
+        RAISE NOTICE '[Migration 000001] Created index idx_sessions_context_config';
+    END IF;
+    
+    -- For messages.agent_steps
+    IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'idx_messages_agent_steps') THEN
+        CREATE INDEX idx_messages_agent_steps ON messages USING GIN (agent_steps);
+        RAISE NOTICE '[Migration 000001] Created index idx_messages_agent_steps';
+    END IF;
+END $$;
+
+-- ============================================================================
+-- Section 13: Data Migrations
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Running data migrations...'; END $$;
+
+-- Clean up unreferenced models (soft delete)
+DO $$
+DECLARE
+    affected_rows INTEGER;
+BEGIN
+    WITH referenced_models AS (
+        SELECT embedding_model_id AS model_id FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != ''
+        UNION
+        SELECT summary_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != ''
+        UNION
+        SELECT vlm_config ->> 'model_id'
+        FROM knowledge_bases
+        WHERE deleted_at IS NULL
+          AND vlm_config ->> 'model_id' IS NOT NULL
+          AND vlm_config ->> 'model_id' != ''
+        UNION
+        SELECT embedding_model_id FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != ''
+    )
+    UPDATE models m
+    SET deleted_at = CURRENT_TIMESTAMP
+    WHERE m.deleted_at IS NULL
+      AND m.is_default = FALSE
+      AND m.tenant_id != 0
+      AND m.id NOT IN (SELECT model_id FROM referenced_models WHERE model_id IS NOT NULL);
+    
+    GET DIAGNOSTICS affected_rows = ROW_COUNT;
+    IF affected_rows > 0 THEN
+        RAISE NOTICE '[Migration 000001] Soft deleted % unreferenced models', affected_rows;
+    END IF;
+END $$;
+
+-- Update models tenant_id from knowledge_bases references
+DO $$
+DECLARE
+    affected_rows INTEGER;
+BEGIN
+    WITH tenant_source AS (
+        SELECT kb.embedding_model_id AS model_id, kb.tenant_id
+        FROM knowledge_bases kb
+        WHERE kb.tenant_id IS NOT NULL AND kb.embedding_model_id IS NOT NULL AND kb.embedding_model_id <> ''
+        UNION
+        SELECT kb.summary_model_id AS model_id, kb.tenant_id
+        FROM knowledge_bases kb
+        WHERE kb.tenant_id IS NOT NULL AND kb.summary_model_id IS NOT NULL AND kb.summary_model_id <> ''
+    )
+    UPDATE models m
+    SET tenant_id = ts.tenant_id
+    FROM tenant_source ts
+    WHERE m.id = ts.model_id AND m.tenant_id = 0;
+    
+    GET DIAGNOSTICS affected_rows = ROW_COUNT;
+    IF affected_rows > 0 THEN
+        RAISE NOTICE '[Migration 000001] Updated tenant_id for % models', affected_rows;
+    END IF;
+END $$;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000001] Migration completed successfully!'; END $$;

migrations/versioned/000002_embeddings.down.sql

@@ -0,0 +1,14 @@
+-- Drop indexes for embeddings
+DROP INDEX IF EXISTS idx_embeddings_knowledge_base_id;
+
+-- Drop index
+DROP INDEX IF EXISTS idx_embeddings_is_enabled;
+
+DROP INDEX IF EXISTS embeddings_unique_source;
+DROP INDEX IF EXISTS embeddings_search_idx;
+DROP INDEX IF EXISTS embeddings_embedding_idx_3584;
+DROP INDEX IF EXISTS embeddings_embedding_idx_798;
+DROP INDEX IF EXISTS embeddings_embedding_idx;
+
+-- Drop embeddings table
+DROP TABLE IF EXISTS embeddings;

migrations/versioned/000002_embeddings.up.sql

@@ -0,0 +1,94 @@
+-- Migration: embeddings (conditional)
+-- Description: Create embeddings table and indexes (only for postgres retrieve driver)
+
+DO $$
+BEGIN
+    -- Check if we should skip this migration
+    IF current_setting('app.skip_embedding', true) = 'true' THEN
+        RAISE NOTICE 'Skipping migration embeddings (app.skip_embedding=true)';
+        RETURN;
+    END IF;
+
+    -- If we reach here, proceed with migration
+    RAISE NOTICE '[Conditional Migration: embeddings] Creating embeddings table...';
+
+    -- Create required extensions
+    CREATE EXTENSION IF NOT EXISTS vector;
+    CREATE EXTENSION IF NOT EXISTS pg_trgm;
+    CREATE EXTENSION IF NOT EXISTS pg_search;
+
+    -- Create embeddings table
+    RAISE NOTICE '[Conditional Migration: embeddings] Creating indexes for embeddings (this may take a while)...';
+    
+    CREATE TABLE IF NOT EXISTS embeddings (
+        id SERIAL PRIMARY KEY,
+        created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+        updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+
+        source_id VARCHAR(64) NOT NULL,
+        source_type INTEGER NOT NULL,
+        chunk_id VARCHAR(64),
+        knowledge_id VARCHAR(64),
+        knowledge_base_id VARCHAR(64),
+        content TEXT,
+        dimension INTEGER NOT NULL,
+        embedding halfvec
+    );
+
+    CREATE UNIQUE INDEX IF NOT EXISTS embeddings_unique_source ON embeddings(source_id, source_type);
+
+    -- Create BM25 search index (check if exists first)
+    IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'embeddings_search_idx') THEN
+        CREATE INDEX embeddings_search_idx ON embeddings
+        USING bm25 (id, knowledge_base_id, content, knowledge_id, chunk_id)
+        WITH (
+            key_field = 'id',
+            text_fields = '{
+                "content": {
+                  "tokenizer": {"type": "chinese_lindera"}
+                }
+            }'
+        );
+        RAISE NOTICE '[Conditional Migration: embeddings] Created BM25 index embeddings_search_idx';
+    ELSE
+        RAISE NOTICE '[Conditional Migration: embeddings] BM25 index embeddings_search_idx already exists';
+    END IF;
+
+    -- Create HNSW indexes for vector search (check if exists first)
+    IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'embeddings_embedding_idx' OR indexname LIKE 'embeddings_embedding%3584%') THEN
+        CREATE INDEX embeddings_embedding_idx_3584 ON embeddings 
+        USING hnsw ((embedding::halfvec(3584)) halfvec_cosine_ops) 
+        WITH (m = 16, ef_construction = 64) 
+        WHERE (dimension = 3584);
+        RAISE NOTICE '[Conditional Migration: embeddings] Created HNSW index for dimension 3584';
+    ELSE
+        RAISE NOTICE '[Conditional Migration: embeddings] HNSW index for dimension 3584 already exists';
+    END IF;
+
+    IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'embeddings_embedding_idx_798' OR indexname LIKE 'embeddings_embedding%798%') THEN
+        CREATE INDEX embeddings_embedding_idx_798 ON embeddings 
+        USING hnsw ((embedding::halfvec(798)) halfvec_cosine_ops) 
+        WITH (m = 16, ef_construction = 64) 
+        WHERE (dimension = 798);
+        RAISE NOTICE '[Conditional Migration: embeddings] Created HNSW index for dimension 798';
+    ELSE
+        RAISE NOTICE '[Conditional Migration: embeddings] HNSW index for dimension 798 already exists';
+    END IF;
+
+    RAISE NOTICE '[Migration 000002] Adding embeddings enhancements...';
+
+    -- Add is_enabled column
+    ALTER TABLE embeddings ADD COLUMN IF NOT EXISTS is_enabled BOOLEAN DEFAULT TRUE;
+    CREATE INDEX IF NOT EXISTS idx_embeddings_is_enabled ON embeddings(is_enabled);
+
+    -- Add index for knowledge_base_id
+    CREATE INDEX IF NOT EXISTS idx_embeddings_knowledge_base_id ON embeddings(knowledge_base_id);
+
+    -- Reindex BM25 search index (idempotent - will rebuild if exists)
+    IF EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'embeddings_search_idx') THEN
+        REINDEX INDEX embeddings_search_idx;
+        RAISE NOTICE '[Migration 000002] Reindexed embeddings_search_idx';
+    END IF;
+
+    RAISE NOTICE '[Conditional Migration: embeddings] Embeddings table setup completed successfully!';
+END $$;

migrations/versioned/000003_chunk_flags.down.sql

@@ -0,0 +1,11 @@
+-- Migration: chunk_flags (rollback)
+-- Description: Remove flags column from chunks table
+
+DO $$ 
+BEGIN
+    RAISE NOTICE '[Migration 000003] Removing flags column from chunks table...';
+    
+    ALTER TABLE chunks DROP COLUMN IF EXISTS flags;
+    
+    RAISE NOTICE '[Migration 000003] Flags column removed successfully';
+END $$;

migrations/versioned/000003_chunk_flags.up.sql

@@ -0,0 +1,13 @@
+-- Migration: chunk_flags
+-- Description: Add flags column to chunks table for managing multiple boolean states
+
+DO $$ 
+BEGIN
+    RAISE NOTICE '[Migration 000003] Adding flags column to chunks table...';
+    
+    -- Add flags column with default value 1 (ChunkFlagRecommended)
+    -- This means all existing chunks will be recommended by default
+    ALTER TABLE chunks ADD COLUMN IF NOT EXISTS flags INTEGER NOT NULL DEFAULT 1;
+    
+    RAISE NOTICE '[Migration 000003] Flags column added successfully';
+END $$;

migrations/versioned/000004_drop_vlm_model_id.down.sql

@@ -0,0 +1,12 @@
+-- Migration: drop_vlm_model_id (rollback)
+-- Description: Re-add vlm_model_id column to knowledge_bases table
+
+DO $$ 
+BEGIN
+    RAISE NOTICE '[Migration 000004 Rollback] Re-adding vlm_model_id column to knowledge_bases table...';
+    
+    -- Re-add vlm_model_id column (nullable to avoid issues)
+    ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS vlm_model_id VARCHAR(64);
+    
+    RAISE NOTICE '[Migration 000004 Rollback] vlm_model_id column re-added successfully';
+END $$;

migrations/versioned/000004_drop_vlm_model_id.up.sql

@@ -0,0 +1,12 @@
+-- Migration: drop_vlm_model_id
+-- Description: Drop vlm_model_id column from knowledge_bases table (field moved to vlm_config JSON)
+
+DO $$ 
+BEGIN
+    RAISE NOTICE '[Migration 000004] Dropping vlm_model_id column from knowledge_bases table...';
+    
+    -- Drop vlm_model_id column if exists (this field was moved to vlm_config JSON)
+    ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS vlm_model_id;
+    
+    RAISE NOTICE '[Migration 000004] vlm_model_id column dropped successfully';
+END $$;

migrations/versioned/000005_mentioned_items.down.sql

@@ -0,0 +1,3 @@
+-- Remove mentioned_items column from messages table
+
+ALTER TABLE messages DROP COLUMN IF EXISTS mentioned_items;

migrations/versioned/000005_mentioned_items.up.sql

@@ -0,0 +1,7 @@
+-- Add mentioned_items column to messages table
+-- This column stores @mentioned knowledge bases and files when user sends a message
+
+ALTER TABLE messages ADD COLUMN IF NOT EXISTS mentioned_items JSONB DEFAULT '[]';
+
+-- Add comment for the column
+COMMENT ON COLUMN messages.mentioned_items IS 'Stores @mentioned knowledge bases and files (id, name, type) when user sends a message';

migrations/versioned/000006_custom_agents.down.sql

@@ -0,0 +1,18 @@
+-- Migration: 000006_custom_agents (rollback)
+-- Description: Remove custom agents table and related changes
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000006 DOWN] Starting custom agents rollback...'; END $$;
+
+-- Remove agent_id column from sessions table
+DO $$ BEGIN RAISE NOTICE '[Migration 000006 DOWN] Removing agent_id column from sessions table'; END $$;
+DROP INDEX IF EXISTS idx_sessions_agent_id;
+ALTER TABLE sessions DROP COLUMN IF EXISTS agent_id;
+
+-- Drop custom_agents table (includes built-in agents created during migration)
+DO $$ BEGIN RAISE NOTICE '[Migration 000006 DOWN] Dropping table: custom_agents'; END $$;
+DROP INDEX IF EXISTS idx_custom_agents_tenant_id;
+DROP INDEX IF EXISTS idx_custom_agents_is_builtin;
+DROP INDEX IF EXISTS idx_custom_agents_deleted_at;
+DROP TABLE IF EXISTS custom_agents;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000006 DOWN] Custom agents rollback completed!'; END $$;

migrations/versioned/000006_custom_agents.up.sql

@@ -0,0 +1,176 @@
+-- Migration: 000006_custom_agents
+-- Description: Add custom agents table for GPTs-like agent configuration and migrate tenant config to built-in agents
+DO $$ BEGIN RAISE NOTICE '[Migration 000006] Starting custom agents setup...'; END $$;
+
+-- Create custom_agents table with composite primary key (id, tenant_id)
+-- This allows the same agent ID to exist for different tenants (e.g., 'builtin-normal' for each tenant)
+DO $$ BEGIN RAISE NOTICE '[Migration 000006] Creating table: custom_agents'; END $$;
+CREATE TABLE IF NOT EXISTS custom_agents (
+    id VARCHAR(36) NOT NULL DEFAULT uuid_generate_v4(),
+    name VARCHAR(255) NOT NULL,
+    description TEXT,
+    avatar VARCHAR(64),
+    is_builtin BOOLEAN NOT NULL DEFAULT false,
+    tenant_id INTEGER NOT NULL,
+    created_by VARCHAR(36),
+    config JSONB NOT NULL DEFAULT '{}',
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP WITH TIME ZONE,
+    PRIMARY KEY (id, tenant_id)
+);
+
+-- Add indexes for custom_agents
+CREATE INDEX IF NOT EXISTS idx_custom_agents_tenant_id ON custom_agents(tenant_id);
+CREATE INDEX IF NOT EXISTS idx_custom_agents_is_builtin ON custom_agents(is_builtin);
+CREATE INDEX IF NOT EXISTS idx_custom_agents_deleted_at ON custom_agents(deleted_at);
+
+-- Add agent_id column to sessions table to track which agent was used
+DO $$ BEGIN RAISE NOTICE '[Migration 000006] Adding agent_id column to sessions table'; END $$;
+ALTER TABLE sessions ADD COLUMN IF NOT EXISTS agent_id VARCHAR(36);
+CREATE INDEX IF NOT EXISTS idx_sessions_agent_id ON sessions(agent_id);
+
+-- Helper function to unify prompt placeholders from Go template format to simple format
+CREATE OR REPLACE FUNCTION unify_prompt_placeholder(input TEXT) RETURNS TEXT AS $$
+DECLARE
+    result TEXT := COALESCE(input, '');
+    replacements TEXT[][] := ARRAY[
+        -- Go template variables -> simple placeholders
+        ['{{.Query}}', '{{query}}'],
+        ['{{.Answer}}', '{{answer}}'],
+        ['{{.CurrentTime}}', '{{current_time}}'],
+        ['{{.CurrentWeek}}', '{{current_week}}'],
+        ['{{.Yesterday}}', '{{yesterday}}'],
+        ['{{.Contexts}}', '{{contexts}}'],
+        -- Go template control structures -> simple placeholders or remove
+        ['{{range .Contexts}}', '{{contexts}}'],
+        -- Remove Go template syntax
+        ['{{if .Contexts}}', ''],
+        ['{{else}}', ''],
+        ['{{.}}', '']
+    ];
+    r TEXT[];
+BEGIN
+    FOREACH r SLICE 1 IN ARRAY replacements LOOP
+        result := REPLACE(result, r[1], r[2]);
+    END LOOP;
+    
+    -- Handle {{range .Conversation}}...{{end}} block specially
+    -- Replace the entire block with just {{conversation}}
+    -- The pattern matches: {{range .Conversation}} followed by any content until {{end}}
+    result := regexp_replace(
+        result,
+        '\{\{range \.Conversation\}\}[\s\S]*?\{\{end\}\}',
+        '{{conversation}}',
+        'g'
+    );
+    
+    -- Clean up any remaining {{end}} tags
+    result := REPLACE(result, '{{end}}', '');
+    
+    RETURN result;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Migrate tenant AgentConfig and ConversationConfig to built-in custom agents
+DO $$ BEGIN RAISE NOTICE '[Migration 000006] Migrating tenant config to built-in agents...'; END $$;
+
+-- Insert builtin-quick-answer agent for tenants with ConversationConfig
+INSERT INTO custom_agents (id, name, description, avatar, is_builtin, tenant_id, config, created_at, updated_at)
+SELECT 
+    'builtin-quick-answer',
+    '快速问答',
+    '基于知识库的 RAG 问答，快速准确地回答问题',
+    '💬',
+    true,
+    t.id,
+    jsonb_build_object(
+        'agent_mode', 'quick-answer',
+        'system_prompt', unify_prompt_placeholder(t.conversation_config->>'prompt'),
+        'context_template', unify_prompt_placeholder(t.conversation_config->>'context_template'),
+        'model_id', COALESCE(t.conversation_config->>'summary_model_id', ''),
+        'rerank_model_id', COALESCE(t.conversation_config->>'rerank_model_id', ''),
+        'temperature', COALESCE((t.conversation_config->>'temperature')::float, 0.7),
+        'max_completion_tokens', COALESCE((t.conversation_config->>'max_completion_tokens')::int, 2048),
+        'max_iterations', 10,
+        'allowed_tools', '[]'::jsonb,
+        'reflection_enabled', false,
+        'kb_selection_mode', 'all',
+        'knowledge_bases', '[]'::jsonb,
+        'web_search_enabled', false,
+        'web_search_max_results', COALESCE((t.web_search_config->>'max_results')::int, 5),
+        'multi_turn_enabled', COALESCE((t.conversation_config->>'multi_turn_enabled')::bool, true),
+        'history_turns', COALESCE((t.conversation_config->>'max_rounds')::int, 5),
+        'embedding_top_k', COALESCE((t.conversation_config->>'embedding_top_k')::int, 10),
+        'keyword_threshold', COALESCE((t.conversation_config->>'keyword_threshold')::float, 0.3),
+        'vector_threshold', COALESCE((t.conversation_config->>'vector_threshold')::float, 0.5),
+        'rerank_top_k', COALESCE((t.conversation_config->>'rerank_top_k')::int, 5),
+        'rerank_threshold', COALESCE((t.conversation_config->>'rerank_threshold')::float, 0.5),
+        'enable_query_expansion', COALESCE((t.conversation_config->>'enable_query_expansion')::bool, true),
+        'enable_rewrite', COALESCE((t.conversation_config->>'enable_rewrite')::bool, true),
+        'rewrite_prompt_system', unify_prompt_placeholder(t.conversation_config->>'rewrite_prompt_system'),
+        'rewrite_prompt_user', unify_prompt_placeholder(t.conversation_config->>'rewrite_prompt_user'),
+        'fallback_strategy', COALESCE(t.conversation_config->>'fallback_strategy', 'model'),
+        'fallback_response', unify_prompt_placeholder(t.conversation_config->>'fallback_response'),
+        'fallback_prompt', unify_prompt_placeholder(t.conversation_config->>'fallback_prompt')
+    ),
+    NOW(),
+    NOW()
+FROM tenants t
+WHERE t.conversation_config IS NOT NULL
+  AND t.deleted_at IS NULL
+ON CONFLICT (id, tenant_id) DO UPDATE SET
+    config = EXCLUDED.config,
+    updated_at = NOW();
+
+-- Insert builtin-smart-reasoning agent for tenants with AgentConfig
+INSERT INTO custom_agents (id, name, description, avatar, is_builtin, tenant_id, config, created_at, updated_at)
+SELECT 
+    'builtin-smart-reasoning',
+    '智能推理',
+    'ReAct 推理框架，支持多步思考和工具调用',
+    '🤖',
+    true,
+    t.id,
+    jsonb_build_object(
+        'agent_mode', 'smart-reasoning',
+        'system_prompt', unify_prompt_placeholder(t.agent_config->>'system_prompt_web_disabled'),
+        'system_prompt_web_enabled', unify_prompt_placeholder(t.agent_config->>'system_prompt_web_enabled'),
+        'context_template', '',
+        'model_id', COALESCE(t.conversation_config->>'summary_model_id', ''),
+        'rerank_model_id', COALESCE(t.conversation_config->>'rerank_model_id', ''),
+        'temperature', COALESCE((t.agent_config->>'temperature')::float, 0.7),
+        'max_completion_tokens', 2048,
+        'max_iterations', COALESCE((t.agent_config->>'max_iterations')::int, 50),
+        'allowed_tools', COALESCE(t.agent_config->'allowed_tools', '["thinking", "todo_write", "knowledge_search", "grep_chunks", "list_knowledge_chunks", "query_knowledge_graph", "get_document_info"]'::jsonb),
+        'reflection_enabled', COALESCE((t.agent_config->>'reflection_enabled')::bool, false),
+        'mcp_selection_mode', 'all',
+        'mcp_services', '[]'::jsonb,
+        'kb_selection_mode', 'all',
+        'knowledge_bases', COALESCE(t.agent_config->'knowledge_bases', '[]'::jsonb),
+        'web_search_enabled', COALESCE((t.agent_config->>'web_search_enabled')::bool, true),
+        'web_search_max_results', COALESCE((t.agent_config->>'web_search_max_results')::int, COALESCE((t.web_search_config->>'max_results')::int, 5)),
+        'multi_turn_enabled', COALESCE((t.agent_config->>'multi_turn_enabled')::bool, true),
+        'history_turns', COALESCE((t.agent_config->>'history_turns')::int, 5),
+        'embedding_top_k', 10,
+        'keyword_threshold', 0.3,
+        'vector_threshold', 0.5,
+        'rerank_top_k', 5,
+        'rerank_threshold', 0.5,
+        'enable_query_expansion', false,
+        'enable_rewrite', false,
+        'rewrite_prompt_system', '',
+        'rewrite_prompt_user', '',
+        'fallback_strategy', 'model',
+        'fallback_response', '',
+        'fallback_prompt', ''
+    ),
+    NOW(),
+    NOW()
+FROM tenants t
+WHERE t.agent_config IS NOT NULL
+  AND t.deleted_at IS NULL
+ON CONFLICT (id, tenant_id) DO UPDATE SET
+    config = EXCLUDED.config,
+    updated_at = NOW();
+

migrations/versioned/000007_embeddings_tag_id.down.sql

@@ -0,0 +1,12 @@
+-- Remove tag_id column from embeddings table
+DO $$
+BEGIN
+    IF EXISTS (
+        SELECT 1 FROM information_schema.columns 
+        WHERE table_name = 'embeddings' AND column_name = 'tag_id'
+    ) THEN
+        DROP INDEX IF EXISTS idx_embeddings_tag_id;
+        ALTER TABLE embeddings DROP COLUMN tag_id;
+        RAISE NOTICE '[Migration 000007 Rollback] Removed tag_id column from embeddings table';
+    END IF;
+END $$;

migrations/versioned/000007_embeddings_tag_id.up.sql

@@ -0,0 +1,20 @@
+-- Add tag_id column to embeddings table for FAQ priority filtering
+DO $$
+BEGIN
+    -- Check if table exists first
+    IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'embeddings') THEN
+        -- Add tag_id column if not exists
+        IF NOT EXISTS (
+            SELECT 1 FROM information_schema.columns 
+            WHERE table_name = 'embeddings' AND column_name = 'tag_id'
+        ) THEN
+            ALTER TABLE embeddings ADD COLUMN tag_id VARCHAR(36);
+            CREATE INDEX IF NOT EXISTS idx_embeddings_tag_id ON embeddings(tag_id);
+            RAISE NOTICE '[Migration 000007] Added tag_id column and index to embeddings table';
+        ELSE
+            RAISE NOTICE '[Migration 000007] tag_id column already exists in embeddings table, skipping';
+        END IF;
+    ELSE
+        RAISE NOTICE '[Migration 000007] embeddings table does not exist, skipping';
+    END IF;
+END $$;

migrations/versioned/000008_migrate_untagged_faq.down.sql

@@ -0,0 +1,60 @@
+-- Rollback: This migration cannot be fully rolled back as we don't know which entries
+-- were originally untagged. We can only clear the tag_id for entries that reference
+-- a "未分类" tag, but this may affect entries that were intentionally tagged.
+
+-- WARNING: This rollback is destructive and should only be used if absolutely necessary.
+-- It will set tag_id to empty string for all chunks, knowledges, and embeddings that reference "未分类" tags.
+
+DO $$
+DECLARE
+    kb_record RECORD;
+    untagged_tag_id VARCHAR(36);
+    updated_chunks INT;
+    updated_knowledges INT;
+BEGIN
+    RAISE NOTICE '[Migration 000008 Rollback] WARNING: This rollback will clear tag_id for all entries referencing "未分类" tags';
+    
+    -- Find all "未分类" tags
+    FOR kb_record IN 
+        SELECT id, tenant_id, knowledge_base_id 
+        FROM knowledge_tags
+        WHERE name = '未分类'
+    LOOP
+        untagged_tag_id := kb_record.id;
+        
+        -- Clear tag_id for chunks referencing this tag (both faq and document types)
+        UPDATE chunks 
+        SET tag_id = '', updated_at = NOW()
+        WHERE tag_id = untagged_tag_id
+        AND chunk_type IN ('faq', 'document');
+        
+        GET DIAGNOSTICS updated_chunks = ROW_COUNT;
+        RAISE NOTICE '[Migration 000008 Rollback] Cleared tag_id for % chunks referencing tag %', 
+            updated_chunks, untagged_tag_id;
+
+        -- Clear tag_id for knowledges referencing this tag
+        UPDATE knowledges 
+        SET tag_id = '', updated_at = NOW()
+        WHERE tag_id = untagged_tag_id;
+        
+        GET DIAGNOSTICS updated_knowledges = ROW_COUNT;
+        RAISE NOTICE '[Migration 000008 Rollback] Cleared tag_id for % knowledges referencing tag %', 
+            updated_knowledges, untagged_tag_id;
+
+        -- Clear tag_id in embeddings if column exists
+        IF EXISTS (
+            SELECT 1 FROM information_schema.columns 
+            WHERE table_name = 'embeddings' AND column_name = 'tag_id'
+        ) THEN
+            UPDATE embeddings 
+            SET tag_id = ''
+            WHERE tag_id = untagged_tag_id;
+        END IF;
+
+        -- Delete the "未分类" tag
+        DELETE FROM knowledge_tags WHERE id = untagged_tag_id;
+        RAISE NOTICE '[Migration 000008 Rollback] Deleted "未分类" tag: %', untagged_tag_id;
+    END LOOP;
+
+    RAISE NOTICE '[Migration 000008 Rollback] Completed rollback';
+END $$;

migrations/versioned/000008_migrate_untagged_faq.up.sql

@@ -0,0 +1,105 @@
+-- Migration: Create "未分类" tag for each knowledge base that has untagged entries
+-- and update chunks, knowledges, and embeddings to reference the new tag
+DO $$
+BEGIN
+    IF current_setting('app.skip_embedding', true) = 'true' THEN
+        RAISE NOTICE 'Skipping pg_search update (app.skip_embedding=true)';
+        RETURN;
+    END IF;
+
+    ALTER EXTENSION pg_search UPDATE;
+END $$;
+
+DO $$
+DECLARE
+    kb_record RECORD;
+    new_tag_id VARCHAR(36);
+    updated_chunks INT;
+    updated_knowledges INT;
+    updated_embeddings INT;
+BEGIN
+    -- Find all knowledge bases that have untagged chunks or knowledges
+    FOR kb_record IN 
+        SELECT DISTINCT tenant_id, knowledge_base_id 
+        FROM (
+            -- Untagged chunks (FAQ or document)
+            SELECT c.tenant_id, c.knowledge_base_id 
+            FROM chunks c
+            WHERE c.chunk_type IN ('faq', 'document')
+            AND (c.tag_id = '' OR c.tag_id IS NULL)
+            UNION
+            -- Untagged knowledges (documents)
+            SELECT k.tenant_id, k.knowledge_base_id 
+            FROM knowledges k
+            WHERE k.deleted_at IS NULL
+            AND (k.tag_id = '' OR k.tag_id IS NULL)
+        ) AS untagged
+    LOOP
+        -- Check if "未分类" tag already exists for this knowledge base
+        SELECT id INTO new_tag_id
+        FROM knowledge_tags
+        WHERE tenant_id = kb_record.tenant_id 
+        AND knowledge_base_id = kb_record.knowledge_base_id 
+        AND name = '未分类'
+        LIMIT 1;
+
+        -- If not exists, create the tag
+        IF new_tag_id IS NULL THEN
+            new_tag_id := gen_random_uuid()::VARCHAR(36);
+            INSERT INTO knowledge_tags (id, tenant_id, knowledge_base_id, name, color, sort_order, created_at, updated_at)
+            VALUES (new_tag_id, kb_record.tenant_id, kb_record.knowledge_base_id, '未分类', '', 0, NOW(), NOW());
+            RAISE NOTICE '[Migration 000008] Created "未分类" tag (id: %) for tenant_id: %, kb_id: %', 
+                new_tag_id, kb_record.tenant_id, kb_record.knowledge_base_id;
+        ELSE
+            RAISE NOTICE '[Migration 000008] "未分类" tag already exists (id: %) for tenant_id: %, kb_id: %', 
+                new_tag_id, kb_record.tenant_id, kb_record.knowledge_base_id;
+        END IF;
+
+        -- Update chunks with empty tag_id (both faq and document types)
+        UPDATE chunks 
+        SET tag_id = new_tag_id, updated_at = NOW()
+        WHERE tenant_id = kb_record.tenant_id 
+        AND knowledge_base_id = kb_record.knowledge_base_id 
+        AND chunk_type IN ('faq', 'document')
+        AND (tag_id = '' OR tag_id IS NULL);
+        
+        GET DIAGNOSTICS updated_chunks = ROW_COUNT;
+        RAISE NOTICE '[Migration 000008] Updated % chunks for tenant_id: %, kb_id: %', 
+            updated_chunks, kb_record.tenant_id, kb_record.knowledge_base_id;
+
+        -- Update knowledges with empty tag_id (documents)
+        UPDATE knowledges 
+        SET tag_id = new_tag_id, updated_at = NOW()
+        WHERE tenant_id = kb_record.tenant_id 
+        AND knowledge_base_id = kb_record.knowledge_base_id 
+        AND deleted_at IS NULL
+        AND (tag_id = '' OR tag_id IS NULL);
+        
+        GET DIAGNOSTICS updated_knowledges = ROW_COUNT;
+        RAISE NOTICE '[Migration 000008] Updated % knowledges for tenant_id: %, kb_id: %', 
+            updated_knowledges, kb_record.tenant_id, kb_record.knowledge_base_id;
+
+        -- Update embeddings with empty tag_id (if embeddings table exists and has tag_id column)
+        IF EXISTS (
+            SELECT 1 FROM information_schema.columns 
+            WHERE table_name = 'embeddings' AND column_name = 'tag_id'
+        ) THEN
+            UPDATE embeddings 
+            SET tag_id = new_tag_id
+            WHERE knowledge_base_id = kb_record.knowledge_base_id 
+            AND (tag_id = '' OR tag_id IS NULL)
+            AND chunk_id IN (
+                SELECT id FROM chunks 
+                WHERE tenant_id = kb_record.tenant_id 
+                AND knowledge_base_id = kb_record.knowledge_base_id 
+                AND chunk_type IN ('faq', 'document')
+            );
+            
+            GET DIAGNOSTICS updated_embeddings = ROW_COUNT;
+            RAISE NOTICE '[Migration 000008] Updated % embeddings for kb_id: %', 
+                updated_embeddings, kb_record.knowledge_base_id;
+        END IF;
+    END LOOP;
+
+    RAISE NOTICE '[Migration 000008] Completed migration of untagged entries';
+END $$;

migrations/versioned/000009_add_last_faq_import_result.down.sql

@@ -0,0 +1,3 @@
+-- Remove last_faq_import_result column from knowledge table
+
+ALTER TABLE knowledges DROP COLUMN IF EXISTS last_faq_import_result;

migrations/versioned/000009_add_last_faq_import_result.up.sql

@@ -0,0 +1,4 @@
+-- Add last_faq_import_result column to knowledge table
+-- This field stores the latest FAQ import result for FAQ type knowledge
+
+ALTER TABLE knowledges ADD COLUMN IF NOT EXISTS last_faq_import_result JSON DEFAULT NULL;

migrations/versioned/000010_add_seq_id.down.sql

@@ -0,0 +1,11 @@
+-- Migration 000010 Down: Remove seq_id from chunks and knowledge_tags tables
+
+-- Remove seq_id from chunks
+DROP INDEX IF EXISTS idx_chunks_seq_id;
+ALTER TABLE chunks DROP COLUMN IF EXISTS seq_id;
+DROP SEQUENCE IF EXISTS chunks_seq_id_seq;
+
+-- Remove seq_id from knowledge_tags
+DROP INDEX IF EXISTS idx_knowledge_tags_seq_id;
+ALTER TABLE knowledge_tags DROP COLUMN IF EXISTS seq_id;
+DROP SEQUENCE IF EXISTS knowledge_tags_seq_id_seq;

migrations/versioned/000010_add_seq_id.up.sql

@@ -0,0 +1,52 @@
+-- Migration 000010: Add seq_id (auto-increment integer ID) to chunks and knowledge_tags tables
+-- This provides integer IDs for FAQ entries and tags for external API usage
+
+-- ============================================================================
+-- Section 1: Add seq_id to chunks table
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000010] Adding seq_id column to chunks table'; END $$;
+
+-- Create sequence for chunks with starting value > 72528124
+CREATE SEQUENCE IF NOT EXISTS chunks_seq_id_seq START WITH 100000000;
+
+-- Add seq_id column to chunks table
+ALTER TABLE chunks ADD COLUMN IF NOT EXISTS seq_id BIGINT;
+
+-- Set default value using sequence
+ALTER TABLE chunks ALTER COLUMN seq_id SET DEFAULT nextval('chunks_seq_id_seq');
+
+-- Populate existing rows with sequence values
+UPDATE chunks SET seq_id = nextval('chunks_seq_id_seq') WHERE seq_id IS NULL;
+
+-- Make seq_id NOT NULL after populating
+ALTER TABLE chunks ALTER COLUMN seq_id SET NOT NULL;
+
+-- Create unique index on seq_id
+CREATE UNIQUE INDEX IF NOT EXISTS idx_chunks_seq_id ON chunks(seq_id);
+
+-- ============================================================================
+-- Section 2: Add seq_id to knowledge_tags table
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000010] Adding seq_id column to knowledge_tags table'; END $$;
+
+-- Create sequence for knowledge_tags with starting value > 2924026
+CREATE SEQUENCE IF NOT EXISTS knowledge_tags_seq_id_seq START WITH 10000000;
+
+-- Add seq_id column to knowledge_tags table
+ALTER TABLE knowledge_tags ADD COLUMN IF NOT EXISTS seq_id BIGINT;
+
+-- Set default value using sequence
+ALTER TABLE knowledge_tags ALTER COLUMN seq_id SET DEFAULT nextval('knowledge_tags_seq_id_seq');
+
+-- Populate existing rows with sequence values
+UPDATE knowledge_tags SET seq_id = nextval('knowledge_tags_seq_id_seq') WHERE seq_id IS NULL;
+
+-- Make seq_id NOT NULL after populating
+ALTER TABLE knowledge_tags ALTER COLUMN seq_id SET NOT NULL;
+
+-- Create unique index on seq_id
+CREATE UNIQUE INDEX IF NOT EXISTS idx_knowledge_tags_seq_id ON knowledge_tags(seq_id);
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000010] seq_id columns added successfully!'; END $$;

migrations/versioned/000011_pg_search_update.down.sql

@@ -0,0 +1,2 @@
+-- Migration 000011 Down: pg_search extension update cannot be rolled back
+-- No-op: ALTER EXTENSION UPDATE has no reverse operation

migrations/versioned/000011_pg_search_update.up.sql

@@ -0,0 +1,12 @@
+-- Migration 000011: Update pg_search extension to latest version
+-- Equivalent to: psql -c 'ALTER EXTENSION pg_search UPDATE;'
+
+DO $$
+BEGIN
+    IF current_setting('app.skip_embedding', true) = 'true' THEN
+        RAISE NOTICE 'Skipping pg_search update (app.skip_embedding=true)';
+        RETURN;
+    END IF;
+
+    ALTER EXTENSION pg_search UPDATE;
+END $$;

migrations/versioned/000012_organizations.down.sql

@@ -0,0 +1,15 @@
+-- Migration: 000012_organizations (down, merged 000012, 000013, 000014)
+DO $$ BEGIN RAISE NOTICE '[Migration 000012] Rolling back organization and agent_share tables...'; END $$;
+
+-- Rollback 000014: tenant_disabled_shared_agents first (no FK to organizations)
+DROP INDEX IF EXISTS idx_tenant_disabled_shared_agents_tenant_id;
+DROP TABLE IF EXISTS tenant_disabled_shared_agents;
+
+-- Rollback 000012/000013: organization-related tables (order matters for FK)
+DROP TABLE IF EXISTS agent_shares;
+DROP TABLE IF EXISTS organization_join_requests;
+DROP TABLE IF EXISTS kb_shares;
+DROP TABLE IF EXISTS organization_members;
+DROP TABLE IF EXISTS organizations;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000012] Rollback completed successfully!'; END $$;

migrations/versioned/000012_organizations.up.sql

@@ -0,0 +1,146 @@
+-- Migration: 000012_organizations (merged 000012, 000013, 000014)
+-- Description: Organization tables, kb_shares, join requests, agent_shares, tenant_disabled_shared_agents
+DO $$ BEGIN RAISE NOTICE '[Migration 000012] Starting organization tables setup...'; END $$;
+
+-- Create organizations table
+CREATE TABLE IF NOT EXISTS organizations (
+    id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    name VARCHAR(255) NOT NULL,
+    description TEXT,
+    owner_id VARCHAR(36) NOT NULL,
+    invite_code VARCHAR(32),
+    require_approval BOOLEAN DEFAULT FALSE,
+    invite_code_expires_at TIMESTAMP WITH TIME ZONE,
+    invite_code_validity_days SMALLINT NOT NULL DEFAULT 7,
+    avatar VARCHAR(512) DEFAULT '',
+    searchable BOOLEAN NOT NULL DEFAULT FALSE,
+    member_limit INTEGER NOT NULL DEFAULT 50,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP WITH TIME ZONE
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_organizations_invite_code ON organizations(invite_code) WHERE invite_code IS NOT NULL AND deleted_at IS NULL;
+CREATE INDEX IF NOT EXISTS idx_organizations_owner_id ON organizations(owner_id);
+CREATE INDEX IF NOT EXISTS idx_organizations_deleted_at ON organizations(deleted_at);
+
+COMMENT ON TABLE organizations IS 'Organizations for cross-tenant collaboration';
+COMMENT ON COLUMN organizations.owner_id IS 'User ID of the organization owner';
+COMMENT ON COLUMN organizations.invite_code IS 'Unique invitation code for joining the organization';
+COMMENT ON COLUMN organizations.require_approval IS 'Whether joining this organization requires admin approval';
+COMMENT ON COLUMN organizations.invite_code_expires_at IS 'When the current invite code expires; NULL means no expiry (legacy)';
+COMMENT ON COLUMN organizations.invite_code_validity_days IS 'Invite link validity in days: 0=never expire, 1/7/30 days';
+COMMENT ON COLUMN organizations.searchable IS 'When true, space appears in search and can be joined by org ID';
+COMMENT ON COLUMN organizations.member_limit IS 'Max members allowed; 0 means no limit';
+
+-- Create organization_members table
+CREATE TABLE IF NOT EXISTS organization_members (
+    id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    organization_id VARCHAR(36) NOT NULL REFERENCES organizations(id) ON DELETE CASCADE,
+    user_id VARCHAR(36) NOT NULL,
+    tenant_id INTEGER NOT NULL,
+    role VARCHAR(32) NOT NULL DEFAULT 'viewer',
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_org_members_org_user ON organization_members(organization_id, user_id);
+CREATE INDEX IF NOT EXISTS idx_org_members_user_id ON organization_members(user_id);
+CREATE INDEX IF NOT EXISTS idx_org_members_tenant_id ON organization_members(tenant_id);
+CREATE INDEX IF NOT EXISTS idx_org_members_role ON organization_members(role);
+
+COMMENT ON TABLE organization_members IS 'Members of organizations with their roles';
+COMMENT ON COLUMN organization_members.role IS 'Member role: admin, editor, or viewer';
+COMMENT ON COLUMN organization_members.tenant_id IS 'The tenant ID that the member belongs to';
+
+-- Create kb_shares table (knowledge base sharing)
+CREATE TABLE IF NOT EXISTS kb_shares (
+    id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    knowledge_base_id VARCHAR(36) NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
+    organization_id VARCHAR(36) NOT NULL REFERENCES organizations(id) ON DELETE CASCADE,
+    shared_by_user_id VARCHAR(36) NOT NULL,
+    source_tenant_id INTEGER NOT NULL,
+    permission VARCHAR(32) NOT NULL DEFAULT 'viewer',
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP WITH TIME ZONE
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_kb_shares_kb_org ON kb_shares(knowledge_base_id, organization_id) WHERE deleted_at IS NULL;
+CREATE INDEX IF NOT EXISTS idx_kb_shares_kb_id ON kb_shares(knowledge_base_id);
+CREATE INDEX IF NOT EXISTS idx_kb_shares_org_id ON kb_shares(organization_id);
+CREATE INDEX IF NOT EXISTS idx_kb_shares_source_tenant ON kb_shares(source_tenant_id);
+CREATE INDEX IF NOT EXISTS idx_kb_shares_deleted_at ON kb_shares(deleted_at);
+
+COMMENT ON TABLE kb_shares IS 'Knowledge base sharing records to organizations';
+COMMENT ON COLUMN kb_shares.source_tenant_id IS 'Original tenant ID of the knowledge base for cross-tenant embedding model access';
+COMMENT ON COLUMN kb_shares.permission IS 'Access permission level: admin, editor, or viewer';
+
+-- Create organization_join_requests table
+CREATE TABLE IF NOT EXISTS organization_join_requests (
+    id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    organization_id VARCHAR(36) NOT NULL REFERENCES organizations(id) ON DELETE CASCADE,
+    user_id VARCHAR(36) NOT NULL,
+    tenant_id INTEGER NOT NULL,
+    status VARCHAR(32) NOT NULL DEFAULT 'pending',
+    requested_role VARCHAR(32) NOT NULL DEFAULT 'viewer',
+    request_type VARCHAR(32) NOT NULL DEFAULT 'join',
+    prev_role VARCHAR(32),
+    message TEXT,
+    reviewed_by VARCHAR(36),
+    reviewed_at TIMESTAMP WITH TIME ZONE,
+    review_message TEXT,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_org_join_requests_org_user_pending ON organization_join_requests(organization_id, user_id) WHERE status = 'pending';
+CREATE INDEX IF NOT EXISTS idx_org_join_requests_org_id ON organization_join_requests(organization_id);
+CREATE INDEX IF NOT EXISTS idx_org_join_requests_user_id ON organization_join_requests(user_id);
+CREATE INDEX IF NOT EXISTS idx_org_join_requests_status ON organization_join_requests(status);
+CREATE INDEX IF NOT EXISTS idx_org_join_requests_type ON organization_join_requests(request_type);
+
+COMMENT ON TABLE organization_join_requests IS 'Join requests for organizations that require approval';
+COMMENT ON COLUMN organization_join_requests.status IS 'Request status: pending, approved, rejected';
+COMMENT ON COLUMN organization_join_requests.requested_role IS 'Role requested by the applicant: admin, editor, viewer';
+COMMENT ON COLUMN organization_join_requests.request_type IS 'join for new member, upgrade for role upgrade';
+COMMENT ON COLUMN organization_join_requests.message IS 'Optional message from the requester';
+COMMENT ON COLUMN organization_join_requests.reviewed_by IS 'User ID of the admin who reviewed the request';
+COMMENT ON COLUMN organization_join_requests.review_message IS 'Optional message from the reviewer';
+
+-- Agent shares (merged from 000013; model_shares omitted, dropped in 000014)
+CREATE TABLE IF NOT EXISTS agent_shares (
+    id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    agent_id VARCHAR(36) NOT NULL,
+    organization_id VARCHAR(36) NOT NULL REFERENCES organizations(id) ON DELETE CASCADE,
+    shared_by_user_id VARCHAR(36) NOT NULL,
+    source_tenant_id INTEGER NOT NULL,
+    permission VARCHAR(32) NOT NULL DEFAULT 'viewer',
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP WITH TIME ZONE,
+    FOREIGN KEY (agent_id, source_tenant_id) REFERENCES custom_agents(id, tenant_id) ON DELETE CASCADE
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_agent_shares_agent_org ON agent_shares(agent_id, source_tenant_id, organization_id) WHERE deleted_at IS NULL;
+CREATE INDEX IF NOT EXISTS idx_agent_shares_agent_id ON agent_shares(agent_id);
+CREATE INDEX IF NOT EXISTS idx_agent_shares_org_id ON agent_shares(organization_id);
+CREATE INDEX IF NOT EXISTS idx_agent_shares_source_tenant ON agent_shares(source_tenant_id);
+CREATE INDEX IF NOT EXISTS idx_agent_shares_deleted_at ON agent_shares(deleted_at);
+
+COMMENT ON TABLE agent_shares IS 'Custom agent sharing records to organizations';
+COMMENT ON COLUMN agent_shares.source_tenant_id IS 'Original tenant ID of the agent';
+COMMENT ON COLUMN agent_shares.permission IS 'Access permission: viewer or editor';
+
+-- Per-tenant "disabled" list for shared agents (merged from 000014)
+DO $$ BEGIN RAISE NOTICE '[Migration 000012] Creating table: tenant_disabled_shared_agents'; END $$;
+CREATE TABLE IF NOT EXISTS tenant_disabled_shared_agents (
+    tenant_id BIGINT NOT NULL,
+    agent_id VARCHAR(36) NOT NULL,
+    source_tenant_id BIGINT NOT NULL,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY (tenant_id, agent_id, source_tenant_id)
+);
+CREATE INDEX IF NOT EXISTS idx_tenant_disabled_shared_agents_tenant_id ON tenant_disabled_shared_agents(tenant_id);
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000012] Organization tables and tenant_disabled_shared_agents setup completed successfully!'; END $$;

migrations/versioned/000013_engine_configs.down.sql

@@ -0,0 +1,3 @@
+-- Rollback engine config columns
+ALTER TABLE tenants DROP COLUMN IF EXISTS storage_engine_config;
+ALTER TABLE tenants DROP COLUMN IF EXISTS parser_engine_config;

migrations/versioned/000013_engine_configs.up.sql

@@ -0,0 +1,8 @@
+-- Description: Add parser_engine_config and storage_engine_config to tenants for UI-configured overrides.
+DO $$ BEGIN RAISE NOTICE '[Migration 000013] Adding engine config columns to tenants'; END $$;
+
+ALTER TABLE tenants ADD COLUMN IF NOT EXISTS parser_engine_config JSONB DEFAULT NULL;
+COMMENT ON COLUMN tenants.parser_engine_config IS 'Parser engine overrides (mineru_endpoint, mineru_api_key, etc.); takes precedence over env when parsing';
+
+ALTER TABLE tenants ADD COLUMN IF NOT EXISTS storage_engine_config JSONB DEFAULT NULL;
+COMMENT ON COLUMN tenants.storage_engine_config IS 'Storage engine parameters for Local, MinIO, COS; used for document/file storage and docreader';

migrations/versioned/000014_storage_provider_config.down.sql

@@ -0,0 +1,16 @@
+-- Rollback: copy provider back to cos_config if needed, then drop new column
+UPDATE knowledge_bases
+SET cos_config = jsonb_set(
+    COALESCE(cos_config, '{}'::jsonb),
+    '{provider}',
+    to_jsonb(storage_provider_config->>'provider')
+)
+WHERE storage_provider_config IS NOT NULL
+  AND storage_provider_config->>'provider' IS NOT NULL
+  AND storage_provider_config->>'provider' != ''
+  AND storage_provider_config->>'provider' != '__pending_env__'
+  AND (cos_config IS NULL
+       OR cos_config->>'provider' IS NULL
+       OR cos_config->>'provider' = '');
+
+ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS storage_provider_config;

migrations/versioned/000014_storage_provider_config.up.sql

@@ -0,0 +1,31 @@
+-- Description: Add storage_provider_config column to knowledge_bases, migrating from legacy cos_config.
+-- This separates the storage provider selection (KB-level) from storage credentials (tenant-level).
+DO $$ BEGIN RAISE NOTICE '[Migration 000014] Adding storage_provider_config to knowledge_bases'; END $$;
+
+-- Step 1: Add new column
+ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS storage_provider_config JSONB DEFAULT NULL;
+COMMENT ON COLUMN knowledge_bases.storage_provider_config IS 'Storage provider config for this KB. Only stores provider name; credentials come from tenant StorageEngineConfig.';
+
+-- Step 2: Migrate existing provider from legacy cos_config
+UPDATE knowledge_bases
+SET storage_provider_config = jsonb_build_object('provider', cos_config->>'provider')
+WHERE cos_config IS NOT NULL
+  AND cos_config->>'provider' IS NOT NULL
+  AND cos_config->>'provider' != ''
+  AND (storage_provider_config IS NULL
+       OR storage_provider_config->>'provider' IS NULL
+       OR storage_provider_config->>'provider' = '');
+
+-- Step 3: For KBs that have documents but no provider set, mark them with the
+-- sentinel value so the application can fill in the actual STORAGE_TYPE on startup.
+-- We use 'pending_migration' as a marker; the app replaces it with the real env value.
+UPDATE knowledge_bases kb
+SET storage_provider_config = '{"provider": "__pending_env__"}'
+WHERE (kb.storage_provider_config IS NULL
+       OR kb.storage_provider_config->>'provider' IS NULL
+       OR kb.storage_provider_config->>'provider' = '')
+  AND EXISTS (
+    SELECT 1 FROM knowledges k
+    WHERE k.knowledge_base_id = kb.id
+      AND k.deleted_at IS NULL
+  );

migrations/versioned/000015_add_is_fallback.down.sql

@@ -0,0 +1,2 @@
+-- Remove is_fallback column from messages table
+ALTER TABLE messages DROP COLUMN IF EXISTS is_fallback;

migrations/versioned/000015_add_is_fallback.up.sql

@@ -0,0 +1,3 @@
+-- Add is_fallback column to messages table
+-- Tracks whether a response was generated using fallback logic (no knowledge base match found)
+ALTER TABLE messages ADD COLUMN IF NOT EXISTS is_fallback BOOLEAN DEFAULT FALSE;

migrations/versioned/000016_add_kb_pinned.down.sql

@@ -0,0 +1,3 @@
+-- Remove pin (置顶) support from knowledge bases
+ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS pinned_at;
+ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS is_pinned;

migrations/versioned/000016_add_kb_pinned.up.sql

@@ -0,0 +1,3 @@
+-- Add pin (置顶) support for knowledge bases
+ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS is_pinned BOOLEAN NOT NULL DEFAULT false;
+ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS pinned_at TIMESTAMP WITH TIME ZONE NULL;

migrations/versioned/000017_mcp_builtin.down.sql

@@ -0,0 +1,14 @@
+-- ============================================================================
+-- Migration 000017 DOWN: Remove is_builtin from mcp_services
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000017 DOWN] Removing is_builtin column from mcp_services...'; END $$;
+
+-- Drop index
+DROP INDEX IF EXISTS idx_mcp_services_is_builtin;
+
+-- Remove is_builtin column
+ALTER TABLE mcp_services
+DROP COLUMN IF EXISTS is_builtin;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000017 DOWN] is_builtin column removed from mcp_services'; END $$;

migrations/versioned/000017_mcp_builtin.up.sql

@@ -0,0 +1,11 @@
+-- ============================================================================
+-- Migration 000017: Add is_builtin support for MCP services
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000017] Adding is_builtin column to mcp_services...'; END $$;
+
+-- Add is_builtin column to mcp_services
+ALTER TABLE mcp_services ADD COLUMN IF NOT EXISTS is_builtin BOOLEAN NOT NULL DEFAULT false;
+CREATE INDEX IF NOT EXISTS idx_mcp_services_is_builtin ON mcp_services(is_builtin);
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000017] is_builtin column added to mcp_services'; END $$;

migrations/versioned/000018_extend_tenant_api_key.down.sql

@@ -0,0 +1,2 @@
+-- Migration 000018 DOWN: Revert tenant api_key column to varchar(64)
+ALTER TABLE tenants ALTER COLUMN api_key TYPE varchar(64);

migrations/versioned/000018_extend_tenant_api_key.up.sql

@@ -0,0 +1,2 @@
+-- Migration 000018: Extend tenant api_key column to support encrypted values
+ALTER TABLE tenants ALTER COLUMN api_key TYPE varchar(256);

migrations/versioned/000019_add_agent_duration_ms.down.sql

@@ -0,0 +1,2 @@
+-- Remove agent_duration_ms column from messages table
+ALTER TABLE messages DROP COLUMN IF EXISTS agent_duration_ms;

migrations/versioned/000019_add_agent_duration_ms.up.sql

@@ -0,0 +1,3 @@
+-- Add agent_duration_ms column to messages table
+-- Stores the total agent execution duration in milliseconds (from query start to answer start)
+ALTER TABLE messages ADD COLUMN IF NOT EXISTS agent_duration_ms BIGINT DEFAULT 0;

migrations/versioned/000020_add_message_knowledge_id.down.sql

@@ -0,0 +1,9 @@
+-- Remove retrieval_config column from tenants table
+ALTER TABLE tenants DROP COLUMN IF EXISTS retrieval_config;
+
+-- Remove chat_history_config column from tenants table
+ALTER TABLE tenants DROP COLUMN IF EXISTS chat_history_config;
+
+-- Remove knowledge_id column from messages table
+DROP INDEX IF EXISTS idx_messages_knowledge_id;
+ALTER TABLE messages DROP COLUMN IF EXISTS knowledge_id;

migrations/versioned/000020_add_message_knowledge_id.up.sql

@@ -0,0 +1,9 @@
+-- Add knowledge_id column to messages table for linking messages to chat history knowledge base entries
+ALTER TABLE messages ADD COLUMN IF NOT EXISTS knowledge_id VARCHAR(36);
+CREATE INDEX IF NOT EXISTS idx_messages_knowledge_id ON messages(knowledge_id);
+
+-- Add chat_history_config JSONB column to tenants table
+ALTER TABLE tenants ADD COLUMN IF NOT EXISTS chat_history_config JSONB;
+
+-- Add retrieval_config JSONB column to tenants table
+ALTER TABLE tenants ADD COLUMN IF NOT EXISTS retrieval_config JSONB;

migrations/versioned/000021_im_channel.down.sql

@@ -0,0 +1,3 @@
+ALTER TABLE im_channel_sessions DROP COLUMN IF EXISTS im_channel_id;
+DROP TABLE IF EXISTS im_channels;
+DROP TABLE IF EXISTS im_channel_sessions;

migrations/versioned/000021_im_channel.up.sql

@@ -0,0 +1,77 @@
+-- Migration: 000021_im_channel_sessions
+-- Description: Create IM channel session mapping and IM channel configuration tables
+DO $$ BEGIN RAISE NOTICE '[Migration 000021] Creating IM channel integration tables'; END $$;
+
+CREATE TABLE IF NOT EXISTS im_channel_sessions (
+    id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    platform VARCHAR(20) NOT NULL,
+    user_id VARCHAR(128) NOT NULL,
+    chat_id VARCHAR(128) NOT NULL DEFAULT '',
+    session_id VARCHAR(36) NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
+    tenant_id BIGINT NOT NULL,
+    agent_id VARCHAR(36) DEFAULT '',
+    status VARCHAR(20) NOT NULL DEFAULT 'active',
+    metadata JSONB DEFAULT '{}',
+    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP WITH TIME ZONE
+);
+
+-- Partial unique index: only enforce uniqueness for non-deleted rows
+CREATE UNIQUE INDEX IF NOT EXISTS idx_channel_lookup
+    ON im_channel_sessions (platform, user_id, chat_id, tenant_id)
+    WHERE deleted_at IS NULL;
+
+-- Index for tenant-based queries
+CREATE INDEX IF NOT EXISTS idx_im_channel_tenant ON im_channel_sessions (tenant_id);
+
+-- Index for session-based queries
+CREATE INDEX IF NOT EXISTS idx_im_channel_session ON im_channel_sessions (session_id);
+
+-- Partial index for soft deletes (only index deleted rows)
+CREATE INDEX IF NOT EXISTS idx_im_channel_deleted ON im_channel_sessions (deleted_at) WHERE deleted_at IS NOT NULL;
+
+COMMENT ON TABLE im_channel_sessions IS 'Maps IM platform channels to WeKnora conversation sessions';
+COMMENT ON COLUMN im_channel_sessions.platform IS 'IM platform identifier: wecom, feishu, etc.';
+COMMENT ON COLUMN im_channel_sessions.user_id IS 'Platform-specific user identifier';
+COMMENT ON COLUMN im_channel_sessions.chat_id IS 'Platform-specific chat/group identifier, empty for direct messages';
+COMMENT ON COLUMN im_channel_sessions.session_id IS 'Associated WeKnora session ID';
+COMMENT ON COLUMN im_channel_sessions.tenant_id IS 'Tenant that owns this channel mapping';
+COMMENT ON COLUMN im_channel_sessions.agent_id IS 'Custom agent ID used for this channel, empty for default';
+COMMENT ON COLUMN im_channel_sessions.status IS 'Channel status: active, paused, expired';
+COMMENT ON COLUMN im_channel_sessions.metadata IS 'Platform-specific extra data (JSON)';
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000021] Creating table: im_channels'; END $$;
+
+CREATE TABLE IF NOT EXISTS im_channels (
+    id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    tenant_id BIGINT NOT NULL,
+    agent_id VARCHAR(36) NOT NULL,
+    platform VARCHAR(20) NOT NULL,
+    name VARCHAR(255) NOT NULL DEFAULT '',
+    enabled BOOLEAN NOT NULL DEFAULT true,
+    mode VARCHAR(20) NOT NULL DEFAULT 'websocket',
+    output_mode VARCHAR(20) NOT NULL DEFAULT 'stream',
+    credentials JSONB NOT NULL DEFAULT '{}',
+    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP WITH TIME ZONE
+);
+
+CREATE INDEX IF NOT EXISTS idx_im_channels_tenant ON im_channels (tenant_id);
+CREATE INDEX IF NOT EXISTS idx_im_channels_agent ON im_channels (agent_id);
+CREATE INDEX IF NOT EXISTS idx_im_channels_deleted ON im_channels (deleted_at) WHERE deleted_at IS NOT NULL;
+
+COMMENT ON TABLE im_channels IS 'IM platform channel configurations bound to agents';
+COMMENT ON COLUMN im_channels.agent_id IS 'Agent ID this channel is bound to';
+COMMENT ON COLUMN im_channels.platform IS 'IM platform: wecom, feishu';
+COMMENT ON COLUMN im_channels.name IS 'User-defined channel name for identification';
+COMMENT ON COLUMN im_channels.mode IS 'Connection mode: webhook or websocket';
+COMMENT ON COLUMN im_channels.output_mode IS 'Output mode: stream (real-time) or full (wait for complete answer)';
+COMMENT ON COLUMN im_channels.credentials IS 'Platform credentials (JSONB): WeCom webhook={corp_id,agent_secret,token,encoding_aes_key,corp_agent_id}, WeCom ws={bot_id,bot_secret}, Feishu={app_id,app_secret,verification_token,encrypt_key}';
+
+-- Add im_channel_id column to im_channel_sessions for linking
+ALTER TABLE im_channel_sessions ADD COLUMN IF NOT EXISTS im_channel_id VARCHAR(36) DEFAULT '';
+CREATE INDEX IF NOT EXISTS idx_im_channel_sessions_channel ON im_channel_sessions (im_channel_id) WHERE im_channel_id != '';
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000021] IM channel integration setup completed successfully!'; END $$;

migrations/versioned/000022_message_images.down.sql

@@ -0,0 +1 @@
+ALTER TABLE messages DROP COLUMN IF EXISTS images;

migrations/versioned/000022_message_images.up.sql

@@ -0,0 +1 @@
+ALTER TABLE messages ADD COLUMN IF NOT EXISTS images JSONB DEFAULT '[]';

migrations/versioned/000023_im_channel_kb_id.down.sql

@@ -0,0 +1 @@
+ALTER TABLE im_channels DROP COLUMN IF EXISTS knowledge_base_id;

migrations/versioned/000023_im_channel_kb_id.up.sql

@@ -0,0 +1,3 @@
+-- Add knowledge_base_id column to im_channels table.
+-- When set, file messages received on this channel will be saved to the specified knowledge base.
+ALTER TABLE im_channels ADD COLUMN IF NOT EXISTS knowledge_base_id VARCHAR(36) DEFAULT '';

migrations/versioned/000024_im_channel_bot_identity.down.sql

@@ -0,0 +1,2 @@
+DROP INDEX IF EXISTS idx_im_channels_bot_identity;
+ALTER TABLE im_channels DROP COLUMN IF EXISTS bot_identity;

migrations/versioned/000024_im_channel_bot_identity.up.sql

@@ -0,0 +1,17 @@
+-- Migration: 000024_im_channel_bot_identity
+-- Description: Add bot_identity column to im_channels for duplicate bot prevention.
+-- bot_identity is a computed unique key derived from credentials, ensuring each bot
+-- can only be connected to one active (non-deleted) channel.
+DO $$ BEGIN RAISE NOTICE '[Migration 000024] Adding bot_identity column to im_channels'; END $$;
+
+ALTER TABLE im_channels ADD COLUMN IF NOT EXISTS bot_identity VARCHAR(255) NOT NULL DEFAULT '';
+
+-- Partial unique index: only enforce uniqueness for non-deleted rows with a non-empty bot_identity.
+-- Empty bot_identity (unknown credential format) is excluded from the constraint.
+CREATE UNIQUE INDEX IF NOT EXISTS idx_im_channels_bot_identity
+    ON im_channels (bot_identity)
+    WHERE deleted_at IS NULL AND bot_identity != '';
+
+COMMENT ON COLUMN im_channels.bot_identity IS 'Unique bot identity derived from credentials (e.g. wecom:ws:{bot_id}, feishu:{app_id}). Used to prevent duplicate bot bindings.';
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000024] bot_identity column and unique index created successfully'; END $$;

migrations/versioned/000025_message_channel.down.sql

@@ -0,0 +1,4 @@
+-- Migration: 000025_message_channel (rollback)
+-- Description: Remove channel column from messages and knowledge
+ALTER TABLE messages DROP COLUMN IF EXISTS channel;
+ALTER TABLE knowledges DROP COLUMN IF EXISTS channel;

migrations/versioned/000025_message_channel.up.sql

@@ -0,0 +1,11 @@
+-- Migration: 000025_message_channel
+-- Description: Add channel column to messages and knowledge to track the source channel (web, api, im, etc.)
+DO $$ BEGIN RAISE NOTICE '[Migration 000025] Adding channel column to messages and knowledge'; END $$;
+
+ALTER TABLE messages ADD COLUMN IF NOT EXISTS channel VARCHAR(50) NOT NULL DEFAULT '';
+COMMENT ON COLUMN messages.channel IS 'Source channel of the message: web, api, im, etc.';
+
+ALTER TABLE knowledges ADD COLUMN IF NOT EXISTS channel VARCHAR(50) NOT NULL DEFAULT 'web';
+COMMENT ON COLUMN knowledges.channel IS 'Source channel of the knowledge: web, api, browser_extension, wechat, etc.';
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000025] channel column added to messages and knowledge successfully'; END $$;

migrations/versioned/000026_chunks_query_indexes.down.sql

@@ -0,0 +1,4 @@
+DO $$ BEGIN RAISE NOTICE '[Migration 000026] Dropping chunks query indexes'; END $$;
+
+DROP INDEX IF EXISTS idx_chunks_kb_tenant;
+DROP INDEX IF EXISTS idx_chunks_knowledge_enabled;

migrations/versioned/000026_chunks_query_indexes.up.sql

@@ -0,0 +1,18 @@
+-- ============================================================================
+-- Migration 000026: Add missing indexes for chunks query performance
+-- ============================================================================
+-- Covers grep_chunks tool queries that filter by (knowledge_base_id, tenant_id)
+-- and aggregate by (knowledge_id) with is_enabled/deleted_at filters.
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000026] Adding chunks query indexes'; END $$;
+
+-- Index for main query: WHERE knowledge_base_id = ? AND tenant_id = ?
+-- Also benefits JOIN and OR conditions across multiple KB-tenant pairs
+CREATE INDEX IF NOT EXISTS idx_chunks_kb_tenant
+    ON chunks(knowledge_base_id, tenant_id);
+
+-- Index for count query: WHERE knowledge_id IN (...) AND is_enabled = true AND deleted_at IS NULL
+-- GROUP BY knowledge_id
+CREATE INDEX IF NOT EXISTS idx_chunks_knowledge_enabled
+    ON chunks(knowledge_id, is_enabled, deleted_at);

migrations/versioned/000027_message_rendered_content.down.sql

@@ -0,0 +1,2 @@
+-- Rollback: 000027_message_rendered_content
+ALTER TABLE messages DROP COLUMN IF EXISTS rendered_content;

migrations/versioned/000027_message_rendered_content.up.sql

@@ -0,0 +1,8 @@
+-- Migration: 000027_message_rendered_content
+-- Description: Add rendered_content column to messages to persist RAG-augmented user content for multi-turn context
+DO $$ BEGIN RAISE NOTICE '[Migration 000027] Adding rendered_content column to messages'; END $$;
+
+ALTER TABLE messages ADD COLUMN IF NOT EXISTS rendered_content TEXT NOT NULL DEFAULT '';
+COMMENT ON COLUMN messages.rendered_content IS 'Full RAG-augmented user message sent to LLM, preserving retrieval context across turns';
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000027] rendered_content column added to messages successfully'; END $$;

migrations/versioned/000028_im_thread_session.down.sql

@@ -0,0 +1,8 @@
+DO $$ BEGIN RAISE NOTICE '[Migration 000028] Rolling back thread session support'; END $$;
+
+DROP INDEX IF EXISTS idx_channel_thread_lookup;
+ALTER TABLE im_channel_sessions DROP COLUMN IF EXISTS thread_id;
+ALTER TABLE im_channels DROP CONSTRAINT IF EXISTS chk_im_channels_session_mode;
+ALTER TABLE im_channels DROP COLUMN IF EXISTS session_mode;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000028] Rollback completed'; END $$;

migrations/versioned/000028_im_thread_session.up.sql

@@ -0,0 +1,33 @@
+-- Migration: 000028_im_thread_session
+-- Description: Add thread-based session mode support for IM channels
+DO $$ BEGIN RAISE NOTICE '[Migration 000028] Adding thread session support'; END $$;
+
+-- 1. im_channels: add session_mode column
+ALTER TABLE im_channels
+    ADD COLUMN IF NOT EXISTS session_mode VARCHAR(20) NOT NULL DEFAULT 'user';
+
+-- 2. CHECK constraint (separate statement: ADD COLUMN IF NOT EXISTS does not support inline CONSTRAINT)
+DO $$ BEGIN
+    ALTER TABLE im_channels
+        ADD CONSTRAINT chk_im_channels_session_mode
+        CHECK (session_mode IN ('user', 'thread'));
+EXCEPTION WHEN duplicate_object THEN NULL;
+END $$;
+
+COMMENT ON COLUMN im_channels.session_mode IS
+    'Session resolution mode: user (per user+chat, default) or thread (per thread)';
+
+-- 3. im_channel_sessions: add thread_id column
+ALTER TABLE im_channel_sessions
+    ADD COLUMN IF NOT EXISTS thread_id VARCHAR(128) NOT NULL DEFAULT '';
+
+COMMENT ON COLUMN im_channel_sessions.thread_id IS
+    'Platform thread identifier for thread-based sessions. Empty for user-mode sessions.';
+
+-- 4. Unique index for thread-mode session lookup
+--    Includes chat_id because Telegram message_thread_id is per-chat (topic ID within a group).
+CREATE UNIQUE INDEX IF NOT EXISTS idx_channel_thread_lookup
+    ON im_channel_sessions (platform, chat_id, thread_id, tenant_id)
+    WHERE deleted_at IS NULL AND thread_id != '';
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000028] Thread session support added'; END $$;

migrations/versioned/000029_datasource_tables.down.sql

@@ -0,0 +1,6 @@
+-- Rollback: 000028_datasource_tables
+DROP TRIGGER IF EXISTS trg_sync_logs_updated_at ON sync_logs;
+DROP TRIGGER IF EXISTS trg_data_sources_updated_at ON data_sources;
+
+DROP TABLE IF EXISTS sync_logs;
+DROP TABLE IF EXISTS data_sources;

migrations/versioned/000029_datasource_tables.up.sql

@@ -0,0 +1,60 @@
+-- Migration: 000029_datasource_tables
+-- Description: Create data_sources and sync_logs tables for external data source management
+DO $$ BEGIN RAISE NOTICE '[Migration 000029] Creating data_sources and sync_logs tables'; END $$;
+
+-- Create data_sources table for managing external data source configurations
+CREATE TABLE IF NOT EXISTS data_sources (
+    id VARCHAR(36) NOT NULL PRIMARY KEY,
+    tenant_id BIGINT NOT NULL,
+    knowledge_base_id VARCHAR(36) NOT NULL,
+    name VARCHAR(255) NOT NULL,
+    type VARCHAR(50) NOT NULL,
+    config JSONB,
+    sync_schedule VARCHAR(100),
+    sync_mode VARCHAR(20) DEFAULT 'incremental',
+    status VARCHAR(32) DEFAULT 'active',
+    conflict_strategy VARCHAR(32) DEFAULT 'overwrite',
+    sync_deletions BOOLEAN DEFAULT true,
+    last_sync_at TIMESTAMP NULL,
+    last_sync_cursor JSONB,
+    last_sync_result JSONB,
+    error_message TEXT,
+    sync_log_retention_days INT DEFAULT 30,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_data_sources_tenant_id ON data_sources (tenant_id);
+CREATE INDEX IF NOT EXISTS idx_data_sources_knowledge_base_id ON data_sources (knowledge_base_id);
+CREATE INDEX IF NOT EXISTS idx_data_sources_type ON data_sources (type);
+CREATE INDEX IF NOT EXISTS idx_data_sources_status ON data_sources (status);
+CREATE INDEX IF NOT EXISTS idx_data_sources_deleted_at ON data_sources (deleted_at);
+
+-- Create sync_logs table for tracking sync operation history
+CREATE TABLE IF NOT EXISTS sync_logs (
+    id VARCHAR(36) NOT NULL PRIMARY KEY,
+    data_source_id VARCHAR(36) NOT NULL REFERENCES data_sources(id) ON DELETE CASCADE,
+    tenant_id BIGINT NOT NULL,
+    status VARCHAR(32) NOT NULL,
+    started_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    finished_at TIMESTAMP NULL,
+    items_total INT DEFAULT 0,
+    items_created INT DEFAULT 0,
+    items_updated INT DEFAULT 0,
+    items_deleted INT DEFAULT 0,
+    items_skipped INT DEFAULT 0,
+    items_failed INT DEFAULT 0,
+    error_message TEXT,
+    result JSONB,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS idx_sync_logs_data_source_id ON sync_logs (data_source_id);
+CREATE INDEX IF NOT EXISTS idx_sync_logs_tenant_id ON sync_logs (tenant_id);
+CREATE INDEX IF NOT EXISTS idx_sync_logs_status ON sync_logs (status);
+CREATE INDEX IF NOT EXISTS idx_sync_logs_started_at ON sync_logs (started_at);
+
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000029] data_sources and sync_logs tables created successfully'; END $$;

migrations/versioned/000030_web_search_providers.down.sql

@@ -0,0 +1,3 @@
+-- Rollback migration: 000029_web_search_providers
+DROP TRIGGER IF EXISTS trg_web_search_providers_updated_at ON web_search_providers;
+DROP TABLE IF EXISTS web_search_providers;

migrations/versioned/000030_web_search_providers.up.sql

@@ -0,0 +1,25 @@
+-- Migration: 000030_web_search_providers
+-- Description: Create web_search_providers table for tenant-specific search engine configurations
+DO $$ BEGIN RAISE NOTICE '[Migration 000030] Creating web_search_providers table'; END $$;
+
+-- Create web_search_providers table for managing tenant search engine configurations
+-- Each row represents a configured search provider instance (e.g., "Production Bing", "Test Google")
+-- Agents reference these by ID via custom_agents.config.web_search_provider_id
+CREATE TABLE IF NOT EXISTS web_search_providers (
+    id VARCHAR(36) NOT NULL PRIMARY KEY,
+    tenant_id BIGINT NOT NULL,
+    name VARCHAR(255) NOT NULL,
+    provider VARCHAR(50) NOT NULL,
+    description TEXT,
+    parameters JSONB,
+    is_default BOOLEAN DEFAULT false,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_web_search_providers_tenant_id ON web_search_providers (tenant_id);
+CREATE INDEX IF NOT EXISTS idx_web_search_providers_provider ON web_search_providers (provider);
+CREATE INDEX IF NOT EXISTS idx_web_search_providers_deleted_at ON web_search_providers (deleted_at);
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000030] web_search_providers table created successfully'; END $$;

migrations/versioned/000031_add_asr_config.down.sql

@@ -0,0 +1,7 @@
+-- Migration: 000025_add_asr_config (rollback)
+-- Description: Remove asr_config column from knowledge_bases.
+DO $$ BEGIN RAISE NOTICE '[Migration 000025] Removing asr_config column from knowledge_bases'; END $$;
+
+ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS asr_config;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000025] asr_config column removed successfully'; END $$;

migrations/versioned/000031_add_asr_config.up.sql

@@ -0,0 +1,10 @@
+-- Migration: 000031_add_asr_config
+-- Description: Add asr_config column to knowledge_bases for ASR (Automatic Speech Recognition) configuration.
+-- ASR config stores the model ID and language settings for audio transcription.
+DO $$ BEGIN RAISE NOTICE '[Migration 000031] Adding asr_config column to knowledge_bases'; END $$;
+
+ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS asr_config JSONB;
+
+COMMENT ON COLUMN knowledge_bases.asr_config IS 'ASR (Automatic Speech Recognition) configuration: {"enabled": bool, "model_id": string, "language": string}';
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000031] asr_config column added successfully'; END $$;

migrations/versioned/000032_vector_stores.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS vector_stores;

migrations/versioned/000032_vector_stores.up.sql

@@ -0,0 +1,28 @@
+-- Migration: 000032_vector_stores
+-- Description: Create vector_stores table for tenant-specific vector database configurations
+DO $$ BEGIN RAISE NOTICE '[Migration 000032] Creating vector_stores table'; END $$;
+
+CREATE TABLE IF NOT EXISTS vector_stores (
+    id                VARCHAR(36)  NOT NULL PRIMARY KEY,
+    name              VARCHAR(255) NOT NULL,
+    engine_type       VARCHAR(50)  NOT NULL,
+    connection_config JSONB        NOT NULL DEFAULT '{}',
+    index_config      JSONB        NOT NULL DEFAULT '{}',
+    tenant_id         BIGINT       NOT NULL,
+    created_at        TIMESTAMP    DEFAULT CURRENT_TIMESTAMP,
+    updated_at        TIMESTAMP    DEFAULT CURRENT_TIMESTAMP,
+    deleted_at        TIMESTAMP    NULL
+);
+
+-- Name uniqueness per tenant (excluding soft-deleted rows)
+CREATE UNIQUE INDEX IF NOT EXISTS idx_vector_stores_name_tenant
+    ON vector_stores(name, tenant_id)
+    WHERE deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_vector_stores_tenant_id ON vector_stores(tenant_id);
+CREATE INDEX IF NOT EXISTS idx_vector_stores_engine_type ON vector_stores(engine_type);
+CREATE INDEX IF NOT EXISTS idx_vector_stores_deleted_at ON vector_stores(deleted_at);
+
+-- Note: updated_at is managed by GORM hooks, no database trigger needed.
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000032] vector_stores table created successfully'; END $$;

migrations/versioned/000033_add_video_info_to_chunks.down.sql

@@ -0,0 +1,7 @@
+-- Migration: 000033_add_video_info_to_chunks (down)
+-- Description: Remove video_info column from chunks table.
+DO $$ BEGIN RAISE NOTICE '[Migration 000033 down] Removing video_info column from chunks'; END $$;
+
+ALTER TABLE chunks DROP COLUMN IF EXISTS video_info;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000033 down] video_info column removed successfully'; END $$;

migrations/versioned/000033_add_video_info_to_chunks.up.sql

@@ -0,0 +1,10 @@
+-- Migration: 000033_add_video_info_to_chunks
+-- Description: Add video_info column to chunks table for video multimodal processing.
+-- This column stores JSON-serialized VideoInfo struct containing video analysis results.
+DO $$ BEGIN RAISE NOTICE '[Migration 000033] Adding video_info column to chunks'; END $$;
+
+ALTER TABLE chunks ADD COLUMN IF NOT EXISTS video_info TEXT;
+
+COMMENT ON COLUMN chunks.video_info IS 'Video information in JSON format: {"url": string, "frame_count": int, "has_vlm_analysis": bool, "has_asr": bool, "video_summary": string, "asr_text": string, "frame_descriptions": string[]}';
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000033] video_info column added successfully'; END $$;

migrations/versioned/000034_add_attachments.down.sql

@@ -0,0 +1 @@
+ALTER TABLE messages DROP COLUMN IF EXISTS attachments;

migrations/versioned/000034_add_attachments.up.sql

@@ -0,0 +1 @@
+ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachments JSONB DEFAULT '[]'::jsonb;

migrations/versioned/000035_add_credentials.down.sql

@@ -0,0 +1 @@
+ALTER TABLE tenants DROP COLUMN IF EXISTS credentials;

migrations/versioned/000035_add_credentials.up.sql

@@ -0,0 +1,5 @@
+-- Description: Add credentials column to tenants for third-party provider credentials (e.g. WeKnoraCloud AppID/AppSecret).
+DO $$ BEGIN RAISE NOTICE '[Migration 000035] Adding credentials column to tenants'; END $$;
+
+ALTER TABLE tenants ADD COLUMN IF NOT EXISTS credentials JSONB DEFAULT NULL;
+COMMENT ON COLUMN tenants.credentials IS 'Third-party provider credentials (e.g. WeKnoraCloud AppID/AppSecret); encrypted at application level';

migrations/versioned/000036_kb_vector_store_id.down.sql

@@ -0,0 +1,30 @@
+-- Migration: 000036_kb_vector_store_id (rollback)
+-- Description: Remove vector_store_id column and its index from knowledge_bases.
+--
+-- WARNING: This rollback is destructive. It permanently drops the vector_store_id column
+-- and all KB ↔ VectorStore bindings. To preserve bindings before rollback:
+--
+--   CREATE TABLE knowledge_bases_vsid_backup AS
+--     SELECT id, tenant_id, vector_store_id FROM knowledge_bases
+--     WHERE vector_store_id IS NOT NULL;
+--
+-- This down migration does NOT block execution when non-NULL rows exist, because blocking
+-- during an emergency rollback can make an incident worse. The RAISE NOTICE below logs the
+-- number of rows whose bindings will be lost so operators can reconstruct later if needed.
+DO $$
+DECLARE non_null_count BIGINT;
+BEGIN
+    SELECT COUNT(*) INTO non_null_count
+    FROM knowledge_bases WHERE vector_store_id IS NOT NULL;
+    IF non_null_count > 0 THEN
+        RAISE NOTICE '[Migration 000036 DOWN] Dropping vector_store_id with % non-NULL rows; these bindings will be lost', non_null_count;
+    ELSE
+        RAISE NOTICE '[Migration 000036 DOWN] Removing vector_store_id column from knowledge_bases';
+    END IF;
+END $$;
+
+DROP INDEX IF EXISTS idx_knowledge_bases_tenant_vector_store;
+
+ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS vector_store_id;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000036 DOWN] vector_store_id column removed successfully'; END $$;

migrations/versioned/000036_kb_vector_store_id.up.sql

@@ -0,0 +1,39 @@
+-- Migration: 000036_kb_vector_store_id
+-- Description: Add vector_store_id column to knowledge_bases for per-KB VectorStore binding.
+-- NULL means "use tenant's effective engines" (env store derived from RETRIEVE_DRIVER),
+-- which preserves existing behavior for all pre-existing rows.
+-- No foreign key: env store IDs are virtual (not present in vector_stores), and
+-- VectorStore soft-delete must not cascade into KB references. Referential integrity is
+-- enforced at the application layer.
+-- On PostgreSQL 11+ this ALTER runs as metadata-only (O(1), no row rewrite) because the
+-- column is nullable with no DEFAULT. PostgreSQL 10 and earlier are not supported.
+DO $$ BEGIN RAISE NOTICE '[Migration 000036] Adding vector_store_id column to knowledge_bases'; END $$;
+
+ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS vector_store_id VARCHAR(36);
+
+COMMENT ON COLUMN knowledge_bases.vector_store_id IS
+    'References vector_stores.id. NULL means tenant default (env store derived from RETRIEVE_DRIVER). '
+    'Immutable after creation (enforced at ORM and service layer). No FK by design.';
+
+-- Composite index: tenant_id first to align with the dominant query pattern. All KB queries
+-- start with `WHERE tenant_id = $1`, so the composite index enforces tenant-scoped plans and
+-- hardens against accidental cross-tenant access. A full (non-partial) index is used because
+-- the target end-state has most KBs bound to a store, making a partial index short-lived.
+-- Storage cost is negligible at typical KB row counts.
+CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_vector_store
+    ON knowledge_bases(tenant_id, vector_store_id);
+
+-- Note: CREATE INDEX here holds a SHARE lock briefly. For very large knowledge_bases tables
+-- (hundreds of thousands of rows or more), operators may prefer CREATE INDEX CONCURRENTLY.
+-- This requires running the statement outside a transaction, which the standard migration
+-- runner does not support; execute the CONCURRENTLY variant from a separate operational
+-- script in that case. Simply replacing the statement below with CONCURRENTLY inside this
+-- file will cause the migration to fail and leave schema_migrations in a dirty state.
+
+DO $$
+DECLARE non_null_count BIGINT;
+BEGIN
+    SELECT COUNT(*) INTO non_null_count
+    FROM knowledge_bases WHERE vector_store_id IS NOT NULL;
+    RAISE NOTICE '[Migration 000036] knowledge_bases.vector_store_id added (% non-NULL rows)', non_null_count;
+END $$;

migrations/versioned/000037_wiki_and_indexing.down.sql

@@ -0,0 +1,15 @@
+-- Migration: 000037_wiki_and_indexing (rollback)
+-- Description: Reverse the wiki + indexing schema in the opposite order of the up migration.
+DO $$ BEGIN RAISE NOTICE '[Migration 000037 DOWN] Reverting wiki + indexing schema'; END $$;
+
+ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS indexing_strategy;
+
+DROP TABLE IF EXISTS wiki_page_issues;
+
+DROP TABLE IF EXISTS wiki_folders;
+
+DROP TABLE IF EXISTS wiki_pages;
+
+ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS wiki_config;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000037 DOWN] wiki + indexing schema reverted successfully'; END $$;

migrations/versioned/000037_wiki_and_indexing.up.sql

@@ -0,0 +1,153 @@
+-- Migration: 000037_wiki_and_indexing
+-- Description: Wiki feature schema (wiki_pages table + wiki_config column + wiki_page_issues table)
+-- and the indexing_strategy column on knowledge_bases.
+DO $$ BEGIN RAISE NOTICE '[Migration 000037] Applying wiki + indexing schema'; END $$;
+
+-- ---------------------------------------------------------------------------
+-- 1) wiki_pages table and wiki_config column
+-- Wiki pages are LLM-generated, interlinked markdown documents that form a
+-- persistent wiki for a knowledge base.
+-- ---------------------------------------------------------------------------
+ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS wiki_config JSONB;
+
+COMMENT ON COLUMN knowledge_bases.wiki_config IS 'Wiki configuration: {"auto_ingest": bool, "synthesis_model_id": string, "wiki_language": string, "max_pages_per_ingest": int}';
+
+CREATE TABLE IF NOT EXISTS wiki_pages (
+    id              VARCHAR(36) PRIMARY KEY,
+    tenant_id       BIGINT NOT NULL,
+    knowledge_base_id VARCHAR(36) NOT NULL,
+    slug            VARCHAR(255) NOT NULL,
+    title           VARCHAR(512) NOT NULL DEFAULT '',
+    page_type       VARCHAR(32) NOT NULL DEFAULT 'summary',
+    status          VARCHAR(32) NOT NULL DEFAULT 'published',
+    content         TEXT NOT NULL DEFAULT '',
+    summary         TEXT NOT NULL DEFAULT '',
+    parent_slug     VARCHAR(255) NOT NULL DEFAULT '',
+    -- folder_id is the single source of truth for a page's placement in the
+    -- directory tree (FK to wiki_folders.id; '' = wiki root). category_path /
+    -- wiki_path / depth below are denormalized projections of the folder chain,
+    -- kept in sync on write so list/index/search queries need no folder join.
+    folder_id       VARCHAR(36) NOT NULL DEFAULT '',
+    category_path   JSONB DEFAULT '[]'::JSONB,
+    wiki_path       VARCHAR(1024) NOT NULL DEFAULT '',
+    depth           INT NOT NULL DEFAULT 0,
+    sort_order      INT NOT NULL DEFAULT 0,
+    source_refs     JSONB DEFAULT '[]'::JSONB,
+    chunk_refs      JSONB DEFAULT '[]'::JSONB,
+    in_links        JSONB DEFAULT '[]'::JSONB,
+    out_links       JSONB DEFAULT '[]'::JSONB,
+    page_metadata   JSONB DEFAULT '{}'::JSONB,
+    aliases         JSONB DEFAULT '[]'::JSONB,
+    version         INT NOT NULL DEFAULT 1,
+    created_at      TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    updated_at      TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    deleted_at      TIMESTAMP WITH TIME ZONE
+);
+
+-- slug must be unique within a knowledge base (for non-deleted pages)
+CREATE UNIQUE INDEX IF NOT EXISTS idx_wiki_pages_kb_slug
+    ON wiki_pages (knowledge_base_id, slug)
+    WHERE deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_wiki_pages_kb_id
+    ON wiki_pages (knowledge_base_id);
+
+CREATE INDEX IF NOT EXISTS idx_wiki_pages_page_type
+    ON wiki_pages (knowledge_base_id, page_type);
+
+CREATE INDEX IF NOT EXISTS idx_wiki_pages_parent_slug
+    ON wiki_pages (knowledge_base_id, parent_slug);
+
+CREATE INDEX IF NOT EXISTS idx_wiki_pages_tree
+    ON wiki_pages (knowledge_base_id, page_type, wiki_path, sort_order, title);
+
+CREATE INDEX IF NOT EXISTS idx_wiki_pages_folder
+    ON wiki_pages (knowledge_base_id, folder_id);
+
+CREATE INDEX IF NOT EXISTS idx_wiki_pages_tenant_id
+    ON wiki_pages (tenant_id);
+
+CREATE INDEX IF NOT EXISTS idx_wiki_pages_deleted_at
+    ON wiki_pages (deleted_at);
+
+CREATE INDEX IF NOT EXISTS idx_wiki_pages_fulltext
+    ON wiki_pages USING GIN (to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(content, '')));
+
+-- ---------------------------------------------------------------------------
+-- 1b) wiki_folders table
+-- First-class directory nodes for the wiki browser. A folder exists
+-- independently of any page, so empty folders persist (the user can build a
+-- skeleton and file pages into it later). parent_id forms an adjacency-list
+-- tree ('' = root); path is the materialized "/"-joined name chain kept for
+-- cheap display/sort. wiki_pages.folder_id references id; renaming/moving a
+-- folder updates this row's subtree and the affected pages' cached path.
+-- ---------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS wiki_folders (
+    id                VARCHAR(36) PRIMARY KEY,
+    tenant_id         BIGINT NOT NULL DEFAULT 0,
+    knowledge_base_id VARCHAR(36) NOT NULL,
+    parent_id         VARCHAR(36) NOT NULL DEFAULT '',
+    name              VARCHAR(255) NOT NULL,
+    path              VARCHAR(1024) NOT NULL DEFAULT '',
+    depth             INT NOT NULL DEFAULT 0,
+    sort_order        INT NOT NULL DEFAULT 0,
+    created_at        TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    updated_at        TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    deleted_at        TIMESTAMP WITH TIME ZONE
+);
+
+-- A folder name is unique among its live siblings under the same parent.
+CREATE UNIQUE INDEX IF NOT EXISTS idx_wiki_folders_parent_name
+    ON wiki_folders (knowledge_base_id, parent_id, name)
+    WHERE deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_wiki_folders_parent
+    ON wiki_folders (knowledge_base_id, parent_id);
+
+CREATE INDEX IF NOT EXISTS idx_wiki_folders_deleted_at
+    ON wiki_folders (deleted_at);
+
+-- ---------------------------------------------------------------------------
+-- 2) wiki_page_issues table
+-- Reports issues against generated wiki pages (LLM-flagged or user-flagged).
+-- ---------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS wiki_page_issues (
+    id VARCHAR(36) PRIMARY KEY,
+    tenant_id BIGINT NOT NULL,
+    knowledge_base_id VARCHAR(36) NOT NULL,
+    slug VARCHAR(255) NOT NULL,
+    issue_type VARCHAR(50) NOT NULL,
+    description TEXT NOT NULL,
+    suspected_knowledge_ids JSONB,
+    status VARCHAR(20) DEFAULT 'pending' NOT NULL,
+    reported_by VARCHAR(100) NOT NULL,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    deleted_at TIMESTAMP WITH TIME ZONE
+);
+
+CREATE INDEX IF NOT EXISTS idx_wiki_page_issues_tenant_id ON wiki_page_issues(tenant_id);
+CREATE INDEX IF NOT EXISTS idx_wiki_page_issues_knowledge_base_id ON wiki_page_issues(knowledge_base_id);
+CREATE INDEX IF NOT EXISTS idx_wiki_page_issues_slug ON wiki_page_issues(slug);
+CREATE INDEX IF NOT EXISTS idx_wiki_page_issues_status ON wiki_page_issues(status);
+
+-- ---------------------------------------------------------------------------
+-- 3) indexing_strategy column on knowledge_bases
+-- Controls which indexing pipelines are active (vector, keyword, wiki, graph).
+-- Backfill: existing rows get vector+keyword=true (legacy default behavior);
+-- wiki_enabled / graph_enabled stay false until explicitly enabled.
+-- ---------------------------------------------------------------------------
+ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS indexing_strategy JSONB;
+
+COMMENT ON COLUMN knowledge_bases.indexing_strategy IS 'Indexing pipelines strategy: {"vector_enabled": bool, "keyword_enabled": bool, "wiki_enabled": bool, "graph_enabled": bool}';
+
+UPDATE knowledge_bases
+SET indexing_strategy = jsonb_build_object(
+    'vector_enabled',  TRUE,
+    'keyword_enabled', TRUE,
+    'wiki_enabled',    FALSE,
+    'graph_enabled',   FALSE
+)
+WHERE indexing_strategy IS NULL;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000037] wiki + indexing schema applied successfully'; END $$;

migrations/versioned/000038_im_session_agent_id.down.sql

@@ -0,0 +1,12 @@
+-- Rollback: 000038_im_session_agent_id
+-- Restores the original indexes without agent_id.
+
+DROP INDEX IF EXISTS idx_channel_lookup;
+CREATE UNIQUE INDEX idx_channel_lookup
+    ON im_channel_sessions (platform, user_id, chat_id, tenant_id)
+    WHERE deleted_at IS NULL;
+
+DROP INDEX IF EXISTS idx_channel_thread_lookup;
+CREATE UNIQUE INDEX idx_channel_thread_lookup
+    ON im_channel_sessions (platform, chat_id, thread_id, tenant_id)
+    WHERE deleted_at IS NULL AND thread_id != '';

migrations/versioned/000038_im_session_agent_id.up.sql

@@ -0,0 +1,19 @@
+-- Migration: 000038_im_session_agent_id
+-- Description: Add agent_id to IM channel session unique indexes to isolate sessions per agent.
+-- Fixes: GitHub #1066 (cross-agent session contamination)
+DO $$ BEGIN RAISE NOTICE '[Migration 000038] Adding agent_id to IM channel session indexes'; END $$;
+
+-- 1. User-mode lookup: include agent_id so the same user talking to
+--    different agents gets separate sessions.
+DROP INDEX IF EXISTS idx_channel_lookup;
+CREATE UNIQUE INDEX idx_channel_lookup
+    ON im_channel_sessions (platform, user_id, chat_id, tenant_id, agent_id)
+    WHERE deleted_at IS NULL;
+
+-- 2. Thread-mode lookup: same fix for thread-based sessions.
+DROP INDEX IF EXISTS idx_channel_thread_lookup;
+CREATE UNIQUE INDEX idx_channel_thread_lookup
+    ON im_channel_sessions (platform, chat_id, thread_id, tenant_id, agent_id)
+    WHERE deleted_at IS NULL AND thread_id != '';
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000038] IM session agent_id indexes updated'; END $$;

migrations/versioned/000039_session_user_id_and_pin.down.sql

@@ -0,0 +1,8 @@
+-- Rollback: 000039_session_user_id_and_pin
+
+DROP INDEX IF EXISTS idx_sessions_tenant_user_pin;
+
+ALTER TABLE sessions
+    DROP COLUMN IF EXISTS pinned_at,
+    DROP COLUMN IF EXISTS is_pinned,
+    DROP COLUMN IF EXISTS user_id;

migrations/versioned/000039_session_user_id_and_pin.up.sql

@@ -0,0 +1,21 @@
+-- Migration: 000039_session_user_id_and_pin
+-- Description: Add user_id, is_pinned, pinned_at to sessions for per-user
+--              session ownership and user-level pinning. Existing rows keep
+--              user_id = NULL and stay visible at the tenant level for
+--              backward compatibility.
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000039] Adding user_id/is_pinned/pinned_at to sessions'; END $$;
+
+ALTER TABLE sessions
+    ADD COLUMN IF NOT EXISTS user_id VARCHAR(36),
+    ADD COLUMN IF NOT EXISTS is_pinned BOOLEAN NOT NULL DEFAULT FALSE,
+    ADD COLUMN IF NOT EXISTS pinned_at TIMESTAMP WITH TIME ZONE;
+
+-- Index for the common list query:
+--   WHERE tenant_id = ? AND (user_id = ? OR user_id IS NULL) AND deleted_at IS NULL
+--   ORDER BY is_pinned DESC, pinned_at DESC NULLS LAST, updated_at DESC
+CREATE INDEX IF NOT EXISTS idx_sessions_tenant_user_pin
+    ON sessions (tenant_id, user_id, is_pinned DESC, pinned_at DESC, updated_at DESC)
+    WHERE deleted_at IS NULL;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000039] sessions user_id/pin columns added'; END $$;

migrations/versioned/000040_wiki_log_entries.down.sql

@@ -0,0 +1,9 @@
+-- Migration: 000040_wiki_log_entries (rollback)
+-- Description: Drop the wiki_log_entries table. No data migration was
+--              performed on the way up, so rollback is a plain DROP.
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000040 DOWN] Reverting wiki_log_entries schema'; END $$;
+
+DROP TABLE IF EXISTS wiki_log_entries;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000040 DOWN] wiki_log_entries schema reverted successfully'; END $$;

migrations/versioned/000040_wiki_log_entries.up.sql

@@ -0,0 +1,32 @@
+-- Migration: 000040_wiki_log_entries
+-- Description: Dedicated append-only event table for wiki operation log entries.
+--              Replaces the "single giant TEXT column on slug='log' wiki_pages
+--              row" model that caused O(n^2) write amplification as KBs grew:
+--              every ingest/retract op previously did GetLog + UpdatePage
+--              rewriting the entire (potentially multi-MB) TEXT column.
+--              Here each event is one INSERT; reads paginate by (kb_id, id DESC).
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000040] Applying wiki_log_entries schema'; END $$;
+
+CREATE TABLE IF NOT EXISTS wiki_log_entries (
+    id                BIGSERIAL PRIMARY KEY,
+    tenant_id         BIGINT NOT NULL,
+    knowledge_base_id VARCHAR(36) NOT NULL,
+    action            VARCHAR(32) NOT NULL,
+    knowledge_id      VARCHAR(36) NOT NULL DEFAULT '',
+    doc_title         TEXT NOT NULL DEFAULT '',
+    summary           TEXT NOT NULL DEFAULT '',
+    pages_affected    JSONB NOT NULL DEFAULT '[]'::JSONB,
+    created_at        TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
+);
+
+-- Primary list query: cursor-paginated feed per KB, newest first.
+-- Using id (BIGSERIAL, monotonic) as the cursor sidesteps duplicate-timestamp
+-- tie-breaking that a created_at cursor would require.
+CREATE INDEX IF NOT EXISTS idx_wiki_log_entries_kb_id_desc
+    ON wiki_log_entries (knowledge_base_id, id DESC);
+
+CREATE INDEX IF NOT EXISTS idx_wiki_log_entries_tenant_id
+    ON wiki_log_entries (tenant_id);
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000040] wiki_log_entries schema applied successfully'; END $$;

migrations/versioned/000041_task_queue_and_wiki_indexes.down.sql

@@ -0,0 +1,16 @@
+-- Migration: 000041_task_queue_and_wiki_indexes (rollback)
+-- Description: Drop the generic task queue tables and the new wiki_pages
+--              GIN indexes. Pending rows are NOT migrated back to Redis on
+--              rollback; operators are expected to drain the queue first
+--              (or accept the loss of un-consumed ops).
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000041 DOWN] Reverting task queue + wiki indexes schema'; END $$;
+
+DROP INDEX IF EXISTS idx_wiki_pages_title_trgm;
+DROP INDEX IF EXISTS idx_wiki_pages_source_refs_text;
+DROP INDEX IF EXISTS idx_wiki_pages_source_refs;
+
+DROP TABLE IF EXISTS task_dead_letters;
+DROP TABLE IF EXISTS task_pending_ops;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000041 DOWN] task queue + wiki indexes schema reverted successfully'; END $$;

migrations/versioned/000041_task_queue_and_wiki_indexes.up.sql

@@ -0,0 +1,148 @@
+-- Migration: 000041_task_queue_and_wiki_indexes
+-- Description: Generic task pending queue + dead-letter table, and three
+--              additional GIN indexes on wiki_pages to back the optimized
+--              ingest pipeline (4w-document-scale).
+--
+-- Schemas introduced:
+--   1. task_pending_ops    — durable replacement for the Redis-list-backed
+--                            wiki:pending:<kbID> queue. Designed generically
+--                            (task_type + scope + scope_id) so future
+--                            "debounced batch" task types can reuse it
+--                            without another migration.
+--   2. task_dead_letters   — generic archived-task record. Written by both
+--                            (a) the asynq middleware when a task exhausts
+--                            its retry budget, and (b) the wiki ingest
+--                            service when a per-document op exceeds its
+--                            in-batch retry quota.
+--   3. wiki_pages indexes  — GIN on source_refs (containment + text), and
+--                            trigram on lower(title) for the new pg_trgm
+--                            dedup pre-filter.
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000041] Applying task queue + wiki indexes schema'; END $$;
+
+-- Ensure pg_trgm is loaded before creating the trigram GIN index below.
+-- Migration 000002 also creates this extension, but only inside the
+-- conditional embeddings block (app.skip_embedding gate). Environments that
+-- skip 000002's body — or had pg_trgm install silently fail there — would
+-- otherwise blow up on the gin_trgm_ops index further down. Re-issuing
+-- CREATE EXTENSION IF NOT EXISTS here is a no-op when it's already present
+-- and surfaces a clear error early when the extension genuinely isn't
+-- available, instead of failing midway and leaving task_pending_ops /
+-- task_dead_letters uncreated (see issue #1319).
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+
+-- ---------------------------------------------------------------------------
+-- 1) task_pending_ops
+--
+-- Generic replacement for ad-hoc Redis-list pending queues. Each row is one
+-- pending operation scoped to a (task_type, scope, scope_id) tuple. The
+-- consumer (e.g. wiki ingest batch handler) uses PeekBatch to pull the head
+-- of the list, processes the ops, then DeleteByIDs the consumed rows.
+--
+-- claimed_at backs the concurrent-claim workflow (ClaimBatch): standard-mode
+-- wiki ingest dropped the exclusive per-KB lock in favour of multiple batches
+-- claiming DISJOINT dedup_keys via SELECT ... FOR UPDATE SKIP LOCKED, stamping
+-- claimed_at = NOW() on the rows they take. A claim older than the consumer's
+-- stale threshold (a crashed/abandoned worker) is recoverable by the next
+-- claimer; a fresh claim blocks its whole dedup_key so same-document ops never
+-- split across concurrent batches. Lite mode (no Redis) still serializes per
+-- KB in-process and leaves claimed_at NULL.
+-- ---------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS task_pending_ops (
+    id          BIGSERIAL PRIMARY KEY,
+    tenant_id   BIGINT NOT NULL,
+    task_type   VARCHAR(64) NOT NULL,
+    scope       VARCHAR(32) NOT NULL,
+    scope_id    VARCHAR(64) NOT NULL,
+    op          VARCHAR(32) NOT NULL,
+    dedup_key   VARCHAR(128) NOT NULL DEFAULT '',
+    payload     JSONB NOT NULL DEFAULT '{}'::JSONB,
+    fail_count  INT NOT NULL DEFAULT 0,
+    enqueued_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    claimed_at  TIMESTAMPTZ
+);
+
+COMMENT ON TABLE task_pending_ops IS 'Generic durable pending-op queue keyed by (task_type, scope, scope_id). Replaces ad-hoc Redis-list queues that were vulnerable to TTL eviction.';
+COMMENT ON COLUMN task_pending_ops.task_type IS 'Free-form task identifier, e.g. "wiki:ingest" — should match an asynq task type when applicable.';
+COMMENT ON COLUMN task_pending_ops.scope IS 'Logical scope, e.g. "knowledge_base" / "knowledge" / "tenant". Read together with scope_id.';
+COMMENT ON COLUMN task_pending_ops.dedup_key IS 'Optional service-defined key used by the consumer to de-duplicate equivalent ops within a single batch peek. Empty means no de-dup.';
+COMMENT ON COLUMN task_pending_ops.fail_count IS 'In-batch retry counter: the consumer increments it via IncrFailCount and dead-letters once it exceeds a service-defined cap.';
+COMMENT ON COLUMN task_pending_ops.claimed_at IS 'Concurrent-claim marker: set to NOW() when a consumer claims the row (SELECT ... FOR UPDATE SKIP LOCKED). NULL = unclaimed; a value older than the consumer stale threshold is a crashed/abandoned claim and is recoverable. A fresh claim blocks its whole dedup_key so same-document ops never split across concurrent batches.';
+
+-- Cover the PeekBatch query: the consumer scans rows for one
+-- (task_type, scope, scope_id) tuple ordered by id ASC.
+CREATE INDEX IF NOT EXISTS idx_task_pending_ops_scope
+    ON task_pending_ops (task_type, scope, scope_id, id);
+
+CREATE INDEX IF NOT EXISTS idx_task_pending_ops_tenant
+    ON task_pending_ops (tenant_id);
+
+-- ---------------------------------------------------------------------------
+-- 2) task_dead_letters
+--
+-- Permanent archive for tasks that exhausted retries. Written from two
+-- distinct paths:
+--   (a) The asynq dead-letter middleware (internal/middleware/asynq_dead_letter.go)
+--       inserts a row whenever a task's retry count equals MaxRetry on the
+--       way out — covers every asynq task type uniformly.
+--   (b) The wiki ingest service inserts a row directly when a per-document
+--       op exceeds wikiMaxFailRetries inside a batch — these never escalate
+--       to an asynq retry because they're handled inline.
+--
+-- Operations queries it by scope (e.g. all dead letters for a KB) or by
+-- task_type (e.g. all summary:generation failures in the last 24h). No
+-- TTL — rows are kept for postmortem until manually pruned. failed_at DESC
+-- so newest-first scans are cheap.
+-- ---------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS task_dead_letters (
+    id          BIGSERIAL PRIMARY KEY,
+    tenant_id   BIGINT NOT NULL,
+    task_type   VARCHAR(64) NOT NULL,
+    scope       VARCHAR(32) NOT NULL,
+    scope_id    VARCHAR(64) NOT NULL,
+    related_id  VARCHAR(64) NOT NULL DEFAULT '',
+    payload     JSONB NOT NULL,
+    last_error  TEXT NOT NULL DEFAULT '',
+    fail_count  INT NOT NULL,
+    failed_at   TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+COMMENT ON TABLE task_dead_letters IS 'Permanent archive of tasks that exhausted retries. Written by the asynq dead-letter middleware and by service-level retry handlers (e.g. wiki ingest per-doc retries).';
+COMMENT ON COLUMN task_dead_letters.related_id IS 'Optional secondary identifier. Wiki ingest puts knowledge_id here so retract/ingest dead letters cluster around the source document.';
+COMMENT ON COLUMN task_dead_letters.payload IS 'Raw task payload (asynq.Task.Payload) at the time of failure. Allows manual requeue via SQL + asynq.Client.Enqueue.';
+COMMENT ON COLUMN task_dead_letters.last_error IS 'String form of the error that caused the final retry to fail. Long stack traces are kept verbatim.';
+
+CREATE INDEX IF NOT EXISTS idx_task_dead_letters_scope
+    ON task_dead_letters (scope, scope_id, failed_at DESC);
+
+CREATE INDEX IF NOT EXISTS idx_task_dead_letters_tenant
+    ON task_dead_letters (tenant_id, failed_at DESC);
+
+CREATE INDEX IF NOT EXISTS idx_task_dead_letters_task_type
+    ON task_dead_letters (task_type, failed_at DESC);
+
+-- ---------------------------------------------------------------------------
+-- 3) wiki_pages: source_refs / title trigram indexes
+--
+-- (a) source_refs containment GIN — covers `source_refs @> ?::jsonb`. Used
+--     by ListBySourceRef when looking up pages that cite a given knowledge
+--     id (delete flow, retract reconciliation, getExistingPageSlugsForKnowledge).
+--
+-- (b) source_refs text-fulltext GIN — fallback for the legacy "kid|title"
+--     ref form, which the runtime LIKE pattern needs. Same shape as the
+--     existing fulltext index in 000037 (line 54-55).
+--
+-- (c) title trigram GIN — backs the new pg_trgm-based dedup pre-filter
+--     (selectDedupCandidatePages). pg_trgm extension is already loaded by
+--     migration 000002.
+-- ---------------------------------------------------------------------------
+CREATE INDEX IF NOT EXISTS idx_wiki_pages_source_refs
+    ON wiki_pages USING GIN (source_refs jsonb_path_ops);
+
+CREATE INDEX IF NOT EXISTS idx_wiki_pages_source_refs_text
+    ON wiki_pages USING GIN (to_tsvector('simple', source_refs::text));
+
+CREATE INDEX IF NOT EXISTS idx_wiki_pages_title_trgm
+    ON wiki_pages USING GIN (lower(title) gin_trgm_ops);
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000041] task queue + wiki indexes schema applied successfully'; END $$;

migrations/versioned/000042_mcp_tool_approval.down.sql

@@ -0,0 +1,7 @@
+DO $$ BEGIN RAISE NOTICE '[Migration 000042 DOWN] Dropping mcp_tool_approvals...'; END $$;
+
+DROP INDEX IF EXISTS idx_mcp_tool_approvals_service_id;
+DROP INDEX IF EXISTS idx_mcp_tool_approvals_tenant_svc_tool;
+DROP TABLE IF EXISTS mcp_tool_approvals;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000042 DOWN] Done'; END $$;

migrations/versioned/000042_mcp_tool_approval.up.sql

@@ -0,0 +1,19 @@
+-- MCP tool human-approval flags (per service + tool name, issue #1173)
+DO $$ BEGIN RAISE NOTICE '[Migration 000042] Creating mcp_tool_approvals...'; END $$;
+
+CREATE TABLE IF NOT EXISTS mcp_tool_approvals (
+    id VARCHAR(36) PRIMARY KEY,
+    tenant_id INTEGER NOT NULL,
+    service_id VARCHAR(36) NOT NULL REFERENCES mcp_services(id) ON DELETE CASCADE,
+    tool_name VARCHAR(512) NOT NULL,
+    require_approval BOOLEAN NOT NULL DEFAULT false,
+    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_mcp_tool_approvals_tenant_svc_tool
+    ON mcp_tool_approvals(tenant_id, service_id, tool_name);
+
+CREATE INDEX IF NOT EXISTS idx_mcp_tool_approvals_service_id ON mcp_tool_approvals(service_id);
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000042] mcp_tool_approvals ready'; END $$;

migrations/versioned/000043_tenant_rbac.down.sql

@@ -0,0 +1,32 @@
+-- Migration 000043 down: revert tenant RBAC schema additions.
+--
+-- DANGER: This drops the tenant_members table and the creator_id /
+--         runnable_by_viewer columns. All role assignments, KB ownership
+--         metadata, and viewer-runtime flags are lost UNRECOVERABLY.
+--         Only intended for development rollbacks, never for production.
+--
+-- Operators who really need to run this in a non-dev environment must set
+-- the session-level GUC `weknora.allow_destructive_migration` to 'true'
+-- first (e.g. `SET weknora.allow_destructive_migration = 'true';`).
+-- Otherwise the migration aborts before touching any data.
+DO $$
+BEGIN
+    IF coalesce(current_setting('weknora.allow_destructive_migration', true), 'false') <> 'true' THEN
+        RAISE EXCEPTION
+            '[Migration 000043 down] BLOCKED: refuse to drop tenant_members / creator_id / runnable_by_viewer. '
+            'Set weknora.allow_destructive_migration = ''true'' in this session to override.';
+    END IF;
+    RAISE WARNING '[Migration 000043 down] Reverting tenant RBAC — role data will be lost';
+END $$;
+
+ALTER TABLE custom_agents DROP COLUMN IF EXISTS runnable_by_viewer;
+
+DROP INDEX IF EXISTS idx_knowledge_bases_tenant_creator;
+ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS creator_id;
+
+DROP INDEX IF EXISTS idx_tenant_members_user;
+DROP INDEX IF EXISTS idx_tenant_members_tenant_role;
+DROP INDEX IF EXISTS idx_tenant_members_user_tenant_unique;
+DROP TABLE IF EXISTS tenant_members;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000043 down] tenant RBAC reverted'; END $$;

migrations/versioned/000043_tenant_rbac.up.sql

@@ -0,0 +1,107 @@
+-- Migration: 000043_tenant_rbac
+-- Introduces tenant-level RBAC (issue #1303):
+--   1. tenant_members table holds the (user, tenant) role assignments that replace
+--      the coarse "User.TenantID only" model. A user may now have rows in multiple
+--      tenants with potentially different roles.
+--   2. knowledge_bases.creator_id records who created a KB so Contributors can edit
+--      their own without full tenant-wide edit rights. custom_agents.created_by
+--      already exists and is reused as-is.
+--   3. custom_agents.runnable_by_viewer controls whether TenantRoleViewer users
+--      may start sessions against an agent.
+--
+-- Backfill policy (existing data):
+--   - In each tenant, the earliest-created active user becomes 'owner'; any other
+--     users become 'contributor'. This preserves today's "anyone can create KBs"
+--     behaviour for non-first users while giving each tenant exactly one owner.
+--   - knowledge_bases.creator_id is set to that tenant's owner, so Admins/Owners
+--     keep full control and Contributors do not unexpectedly inherit ownership of
+--     pre-existing resources.
+--   - API-key-only tenants (tenants with no human users) get no membership rows;
+--     the auth middleware auto-promotes the first human authenticating into such
+--     a tenant to Owner.
+DO $$ BEGIN RAISE NOTICE '[Migration 000043] Starting tenant RBAC setup...'; END $$;
+
+-- 1. tenant_members table
+DO $$ BEGIN RAISE NOTICE '[Migration 000043] Creating table: tenant_members'; END $$;
+CREATE TABLE IF NOT EXISTS tenant_members (
+    id          BIGSERIAL PRIMARY KEY,
+    user_id     VARCHAR(36) NOT NULL,
+    tenant_id   INTEGER NOT NULL,
+    role        VARCHAR(20) NOT NULL DEFAULT 'contributor',
+    status      VARCHAR(20) NOT NULL DEFAULT 'active',
+    invited_by  VARCHAR(36),
+    joined_at   TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    created_at  TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at  TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    deleted_at  TIMESTAMP WITH TIME ZONE
+);
+
+-- Partial unique index: at most one non-deleted membership per (user, tenant).
+CREATE UNIQUE INDEX IF NOT EXISTS idx_tenant_members_user_tenant_unique
+    ON tenant_members(user_id, tenant_id)
+    WHERE deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_tenant_members_tenant_role
+    ON tenant_members(tenant_id, role)
+    WHERE deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_tenant_members_user
+    ON tenant_members(user_id)
+    WHERE deleted_at IS NULL;
+
+-- 2. Backfill one membership row per existing active user.
+--    Earliest-created active user per tenant => owner; others => contributor.
+DO $$ BEGIN RAISE NOTICE '[Migration 000043] Backfilling tenant_members rows from users'; END $$;
+INSERT INTO tenant_members (user_id, tenant_id, role, status, joined_at, created_at, updated_at)
+SELECT u.id,
+       u.tenant_id,
+       CASE
+           WHEN u.id = (
+               SELECT u2.id FROM users u2
+                WHERE u2.tenant_id = u.tenant_id
+                  AND u2.is_active = TRUE
+                  AND u2.deleted_at IS NULL
+                ORDER BY u2.created_at ASC, u2.id ASC
+                LIMIT 1
+           ) THEN 'owner'
+           ELSE 'contributor'
+       END AS role,
+       'active',
+       u.created_at,
+       CURRENT_TIMESTAMP,
+       CURRENT_TIMESTAMP
+  FROM users u
+ WHERE u.deleted_at IS NULL
+   AND u.is_active = TRUE
+   AND u.tenant_id IS NOT NULL
+   AND u.tenant_id <> 0
+ON CONFLICT DO NOTHING;
+
+-- 3. knowledge_bases.creator_id
+DO $$ BEGIN RAISE NOTICE '[Migration 000043] Adding creator_id to knowledge_bases'; END $$;
+ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS creator_id VARCHAR(36);
+CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_creator
+    ON knowledge_bases(tenant_id, creator_id);
+
+-- Backfill KB creator to the tenant's owner. Rows in tenants without any human
+-- users (API-key-only) keep creator_id NULL; the application layer treats NULL
+-- creator as "tenant-owned" and requires Admin+ to mutate.
+UPDATE knowledge_bases kb
+   SET creator_id = (
+       SELECT tm.user_id
+         FROM tenant_members tm
+        WHERE tm.tenant_id = kb.tenant_id
+          AND tm.role = 'owner'
+          AND tm.status = 'active'
+          AND tm.deleted_at IS NULL
+        ORDER BY tm.joined_at ASC, tm.id ASC
+        LIMIT 1
+   )
+ WHERE kb.creator_id IS NULL;
+
+-- 4. custom_agents.runnable_by_viewer
+DO $$ BEGIN RAISE NOTICE '[Migration 000043] Adding runnable_by_viewer to custom_agents'; END $$;
+ALTER TABLE custom_agents
+    ADD COLUMN IF NOT EXISTS runnable_by_viewer BOOLEAN NOT NULL DEFAULT TRUE;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000043] tenant RBAC setup ready'; END $$;

migrations/versioned/000044_audit_log.down.sql

@@ -0,0 +1,26 @@
+-- Migration 000044 down: drop audit_logs table.
+--
+-- DANGER: This drops the audit_logs table. All durable RBAC and
+--         resource-action history is lost UNRECOVERABLY. Only intended
+--         for development rollbacks, never for production.
+--
+-- Operators who really need to run this in a non-dev environment must
+-- set the session-level GUC `weknora.allow_destructive_migration` to
+-- 'true' first. Otherwise the migration aborts before touching any data.
+DO $$
+BEGIN
+    IF coalesce(current_setting('weknora.allow_destructive_migration', true), 'false') <> 'true' THEN
+        RAISE EXCEPTION
+            '[Migration 000044 down] BLOCKED: refuse to drop audit_logs. '
+            'Set weknora.allow_destructive_migration = ''true'' in this session to override.';
+    END IF;
+    RAISE WARNING '[Migration 000044 down] Dropping audit_logs — durable history will be lost';
+END $$;
+
+DROP INDEX IF EXISTS idx_audit_logs_created_at;
+DROP INDEX IF EXISTS idx_audit_logs_tenant_action;
+DROP INDEX IF EXISTS idx_audit_logs_actor;
+DROP INDEX IF EXISTS idx_audit_logs_tenant_id_desc;
+DROP TABLE IF EXISTS audit_logs;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000044 down] audit_logs dropped'; END $$;

migrations/versioned/000044_audit_log.up.sql

@@ -0,0 +1,65 @@
+-- Migration: 000044_audit_log
+-- Adds a generic per-tenant audit log table (issue #1303 PR 6).
+--
+-- Scope:
+--   - PR 6 wires the table to RBAC events (member add/remove/role-change/
+--     leave) and middleware-level enforcement denials.
+--   - The schema is intentionally generic (action, target_type, target_id,
+--     details JSONB) so future PRs (KB ops, agent ops, datasource sync)
+--     can plug in new action constants without another migration.
+--
+-- Indexes:
+--   - (tenant_id, id DESC) for the cursor-paginated feed query
+--     ("show this tenant's audit log newest-first").
+--   - (actor_user_id) for "what did Alice do" lookups.
+--   - (tenant_id, action) for action-class filtering and powering the
+--     1-minute sliding-window dedup that LogDenied uses to keep a
+--     probing client from filling the table.
+DO $$ BEGIN RAISE NOTICE '[Migration 000044] Creating table: audit_logs'; END $$;
+
+CREATE TABLE IF NOT EXISTS audit_logs (
+    id              BIGSERIAL PRIMARY KEY,
+    tenant_id       BIGINT NOT NULL,
+    actor_user_id   VARCHAR(36) NOT NULL DEFAULT '',
+    actor_role      VARCHAR(32) NOT NULL DEFAULT '',
+    action          VARCHAR(64) NOT NULL,
+    target_type     VARCHAR(32) NOT NULL DEFAULT '',
+    target_id       VARCHAR(64) NOT NULL DEFAULT '',
+    target_user_id  VARCHAR(36) NOT NULL DEFAULT '',
+    request_path    VARCHAR(512) NOT NULL DEFAULT '',
+    request_method  VARCHAR(16)  NOT NULL DEFAULT '',
+    outcome         VARCHAR(16)  NOT NULL DEFAULT 'success',
+    details         JSONB        NOT NULL DEFAULT '{}'::JSONB,
+    created_at      TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Primary list query: newest-first cursor pagination per tenant. Using id
+-- (BIGSERIAL, monotonic) as the cursor sidesteps the duplicate-timestamp
+-- tie-breaking that a created_at cursor would require.
+CREATE INDEX IF NOT EXISTS idx_audit_logs_tenant_id_desc
+    ON audit_logs (tenant_id, id DESC);
+
+-- Filter by actor (e.g. "what did Alice do"). Useful for incident
+-- response and for user-scoped self-audit screens (out of v1 scope but
+-- the index is cheap and the column is already there).
+CREATE INDEX IF NOT EXISTS idx_audit_logs_actor
+    ON audit_logs (actor_user_id);
+
+-- Powers two queries: (a) the audit-log feed filtered by action class,
+-- and (b) the LogDenied dedup `count rows where (tenant_id, action) and
+-- created_at >= since`. Without this index the dedup count would scan
+-- the entire table on every denied request.
+CREATE INDEX IF NOT EXISTS idx_audit_logs_tenant_action
+    ON audit_logs (tenant_id, action);
+
+-- Powers the daily retention sweep `DELETE FROM audit_logs WHERE
+-- created_at < cutoff`. Without it the sweep would Seq Scan the whole
+-- table on every run, which on a 90-day-retained tenant with bursty
+-- RBAC traffic is enough to blow past the runner's 30s timeout and
+-- never converge. Indexing created_at also keeps the per-day DELETE
+-- bounded to roughly one day's worth of rows once the table reaches
+-- steady state.
+CREATE INDEX IF NOT EXISTS idx_audit_logs_created_at
+    ON audit_logs (created_at);
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000044] audit_logs table ready'; END $$;

migrations/versioned/000045_org_tenant_members.down.sql

@@ -0,0 +1,35 @@
+-- Reverse migration for 000045_org_tenant_members.
+--
+-- Restore organization_members from the parked _pre_plan3 table and drop
+-- the new tenant-scoped table. Because we only RENAMEd the original on
+-- the way up, no data is lost on rollback — the legacy rows are still
+-- intact under the renamed table.
+--
+-- If the parked table no longer exists (e.g. a later destructive
+-- migration already dropped it), this script falls through gracefully:
+-- the IF EXISTS guards leave the database in whatever state it was in.
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000045] Reverting Plan 3'; END $$;
+
+-- Move legacy table back into place.
+ALTER TABLE IF EXISTS organization_members_pre_plan3 RENAME TO organization_members;
+
+-- Restore index names so the application code can find them by their
+-- documented identifiers.
+ALTER INDEX IF EXISTS idx_org_members_org_user_pre_plan3      RENAME TO idx_org_members_org_user;
+ALTER INDEX IF EXISTS idx_org_members_user_id_pre_plan3       RENAME TO idx_org_members_user_id;
+ALTER INDEX IF EXISTS idx_org_members_tenant_id_pre_plan3     RENAME TO idx_org_members_tenant_id;
+ALTER INDEX IF EXISTS idx_org_members_role_pre_plan3          RENAME TO idx_org_members_role;
+
+-- Drop the new table. The unique/secondary indexes go with it.
+DROP TABLE IF EXISTS organization_tenant_members;
+
+-- Drop the partial unique index added in step 5 of up.sql. The data
+-- mutation (pending → rejected) is intentionally NOT reverted: those
+-- duplicates were already wrong under Plan 3 semantics, and we don't
+-- have enough information to safely re-promote which one should win.
+-- The `[Plan 3] superseded` review_message tag makes the affected
+-- rows easy to audit if a manual fix is later needed.
+DROP INDEX IF EXISTS uq_org_join_requests_pending_per_tenant;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000045] Plan 3 reverted'; END $$;

migrations/versioned/000045_org_tenant_members.up.sql

@@ -0,0 +1,163 @@
+-- Migration: 000045_org_tenant_members
+--
+-- Plan 3 of issue #1303: lift "organization membership" from per-user to
+-- per-tenant. Today `organization_members` is keyed on (org_id, user_id) —
+-- if Alice (tenant T) joins Org X, her tenant-mate Bob has no visibility
+-- into anything shared into Org X. That breaks the company-level
+-- collaboration mental model ("our company is in this org") and creates
+-- inconsistency with `tenant_disabled_shared_agents`, which is already
+-- tenant-scoped.
+--
+-- The new table `organization_tenant_members` is keyed on (org_id,
+-- tenant_id). One row says "tenant T participates in Org O at role R,
+-- represented by user U". `representative_user_id` is purely for UI/audit
+-- ("who from this tenant brought us here"); permission checks use only
+-- (org_id, tenant_id, role).
+--
+-- Backfill policy: collapse all `organization_members` rows for the same
+-- (org, tenant) into a single new row. Role = the highest role observed
+-- across the group (admin > editor > viewer). representative_user_id =
+-- the earliest joiner. `RAISE NOTICE` flags any (org, tenant) that had
+-- conflicting roles so operators can audit the resolved choice.
+--
+-- The old `organization_members` table is RENAMEd to
+-- `organization_members_pre_plan3` so a `down.sql` rollback can restore
+-- it. A future destructive migration (gated on
+-- `weknora.allow_destructive_migration`) will DROP it once the new model
+-- is settled.
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000045] Starting Plan 3: org members → org tenant members'; END $$;
+
+-- 1. New table.
+DO $$ BEGIN RAISE NOTICE '[Migration 000045] Creating table: organization_tenant_members'; END $$;
+CREATE TABLE IF NOT EXISTS organization_tenant_members (
+    id                      VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(),
+    organization_id         VARCHAR(36) NOT NULL REFERENCES organizations(id) ON DELETE CASCADE,
+    tenant_id               INTEGER NOT NULL,
+    role                    VARCHAR(32) NOT NULL DEFAULT 'viewer',
+    representative_user_id  VARCHAR(36) NOT NULL DEFAULT '',
+    joined_at               TIMESTAMP WITH TIME ZONE,
+    created_at              TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at              TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_org_tenant_members_unique
+    ON organization_tenant_members (organization_id, tenant_id);
+
+CREATE INDEX IF NOT EXISTS idx_org_tenant_members_by_tenant
+    ON organization_tenant_members (tenant_id);
+
+CREATE INDEX IF NOT EXISTS idx_org_tenant_members_role
+    ON organization_tenant_members (organization_id, role);
+
+COMMENT ON TABLE organization_tenant_members IS 'Plan 3: Tenants (not users) are organization members.';
+COMMENT ON COLUMN organization_tenant_members.role IS 'Tenant role inside the org: admin | editor | viewer.';
+COMMENT ON COLUMN organization_tenant_members.representative_user_id IS 'Display-only: the user who first brought this tenant into the org.';
+
+-- 2. Surface conflicting roles before backfilling so operators see them
+--    in the migration log. The collapse strategy is "max role wins"
+--    (admin > editor > viewer); this is a permission *promotion* for
+--    any user who was at a lower role inside their tenant, so we shout
+--    loudly when it happens.
+DO $$
+DECLARE
+    rec RECORD;
+BEGIN
+    FOR rec IN
+        SELECT organization_id, tenant_id,
+               COUNT(*)                  AS user_count,
+               ARRAY_AGG(DISTINCT role)  AS observed_roles,
+               CASE
+                   WHEN BOOL_OR(role = 'admin')  THEN 'admin'
+                   WHEN BOOL_OR(role = 'editor') THEN 'editor'
+                   ELSE 'viewer'
+               END AS resolved_role
+        FROM organization_members
+        GROUP BY organization_id, tenant_id
+    LOOP
+        IF rec.user_count > 1 AND ARRAY_LENGTH(rec.observed_roles, 1) > 1 THEN
+            RAISE NOTICE '[Migration 000045] dedup org=% tenant=% chose role=% from observed=%, user_count=%',
+                rec.organization_id, rec.tenant_id, rec.resolved_role, rec.observed_roles, rec.user_count;
+        END IF;
+    END LOOP;
+END $$;
+
+-- 3. Backfill the new table from the old one.
+DO $$ BEGIN RAISE NOTICE '[Migration 000045] Backfilling organization_tenant_members from organization_members'; END $$;
+INSERT INTO organization_tenant_members (
+    organization_id, tenant_id, role, representative_user_id, joined_at, created_at, updated_at
+)
+SELECT
+    om.organization_id,
+    om.tenant_id,
+    CASE
+        WHEN BOOL_OR(om.role = 'admin')  THEN 'admin'
+        WHEN BOOL_OR(om.role = 'editor') THEN 'editor'
+        ELSE 'viewer'
+    END AS role,
+    -- Representative: earliest joiner in the group, fallback to lowest user_id
+    -- so the choice is deterministic across re-runs.
+    (SELECT om2.user_id
+       FROM organization_members om2
+      WHERE om2.organization_id = om.organization_id
+        AND om2.tenant_id       = om.tenant_id
+      ORDER BY om2.created_at ASC, om2.user_id ASC
+      LIMIT 1) AS representative_user_id,
+    MIN(om.created_at) AS joined_at,
+    CURRENT_TIMESTAMP,
+    CURRENT_TIMESTAMP
+FROM organization_members om
+GROUP BY om.organization_id, om.tenant_id
+ON CONFLICT DO NOTHING;
+
+-- 4. Park the old table so `down.sql` can restore it. We do NOT drop it
+--    here so a botched rollout can be reversed without re-creating data.
+--    A follow-up destructive migration removes it in a later release.
+DO $$ BEGIN RAISE NOTICE '[Migration 000045] Renaming organization_members → organization_members_pre_plan3'; END $$;
+ALTER TABLE organization_members RENAME TO organization_members_pre_plan3;
+
+-- The unique-index name must move with the table; renaming the table does
+-- not rename the indices in older PG versions, so do it explicitly.
+ALTER INDEX IF EXISTS idx_org_members_org_user      RENAME TO idx_org_members_org_user_pre_plan3;
+ALTER INDEX IF EXISTS idx_org_members_user_id       RENAME TO idx_org_members_user_id_pre_plan3;
+ALTER INDEX IF EXISTS idx_org_members_tenant_id     RENAME TO idx_org_members_tenant_id_pre_plan3;
+ALTER INDEX IF EXISTS idx_org_members_role          RENAME TO idx_org_members_role_pre_plan3;
+
+-- 5. Dedup pending join/upgrade requests at the (org, tenant, type)
+--    level. Plan 3 lifts the dedup key from per-user to per-tenant
+--    (see GetPendingRequestByTenantAndType), so any leftover duplicate
+--    pending rows from the old per-user model are now ambiguous: one
+--    row would silently shadow the others and admins would see a
+--    "ghost" entry remaining in the list after they approved one.
+--
+--    Strategy: keep the earliest pending row per (org, tenant, type),
+--    mark the rest as 'rejected' with a system review_message so the
+--    audit trail makes it clear this was a Plan 3 cleanup, not a
+--    human reject. Then add a partial unique index to keep the
+--    invariant going forward.
+DO $$ BEGIN RAISE NOTICE '[Migration 000045] Deduping pending join/upgrade requests at (org, tenant, type) level'; END $$;
+WITH ranked AS (
+    SELECT id,
+           ROW_NUMBER() OVER (
+               PARTITION BY organization_id, tenant_id, request_type
+               ORDER BY created_at ASC, id ASC
+           ) AS rn
+      FROM organization_join_requests
+     WHERE status = 'pending'
+)
+UPDATE organization_join_requests r
+   SET status         = 'rejected',
+       review_message = '[Plan 3] superseded: another pending request from the same tenant was kept',
+       updated_at     = CURRENT_TIMESTAMP
+  FROM ranked
+ WHERE r.id      = ranked.id
+   AND ranked.rn > 1;
+
+-- Partial unique index: only one pending row per (org, tenant, type).
+-- Postgres lets approved/rejected rows coexist freely so the historical
+-- audit trail is preserved.
+CREATE UNIQUE INDEX IF NOT EXISTS uq_org_join_requests_pending_per_tenant
+    ON organization_join_requests (organization_id, tenant_id, request_type)
+    WHERE status = 'pending';
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000045] Plan 3 setup ready'; END $$;

migrations/versioned/000046_org_owner_tenant_id.down.sql

@@ -0,0 +1,13 @@
+-- Reverse migration for 000046_org_owner_tenant_id.
+--
+-- Drop the index first, then the column. The data backfilled in up.sql
+-- is recoverable from `users.tenant_id` so we don't park it.
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000046] Dropping organizations.owner_tenant_id'; END $$;
+
+DROP INDEX IF EXISTS idx_organizations_owner_tenant;
+
+ALTER TABLE organizations
+    DROP COLUMN IF EXISTS owner_tenant_id;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000046] organizations.owner_tenant_id removed'; END $$;

migrations/versioned/000046_org_owner_tenant_id.up.sql

@@ -0,0 +1,78 @@
+-- Migration: 000046_org_owner_tenant_id
+--
+-- Plan 3 follow-up to #1303: pin the org's "owning tenant" in the
+-- organizations table itself instead of recomputing it on every
+-- permission check from the owner user's *current* tenant. The old
+-- `isOwnerTenant(org, t) == (owner.user.TenantID == t)` rule breaks
+-- silently when the owner user moves to a different tenant: the
+-- formerly-protected owning tenant becomes removable, and the new
+-- tenant the owner moved to (which probably isn't even in OTM) starts
+-- being treated as untouchable. Storing owner_tenant_id at create time
+-- and never changing it post-hoc gives us a single source of truth.
+--
+-- Backfill strategy:
+--   1. derive from the owner user's current tenant (works for the
+--      vast majority of orgs);
+--   2. for orgs whose owner user is gone, fall back to the earliest
+--      Admin tenant in OTM — deterministic across re-runs;
+--   3. anything still NULL means the org is genuinely orphaned (no
+--      owner user, no admin tenant) — abort the migration with a loud
+--      RAISE EXCEPTION so the operator deals with it manually.
+--      We refuse to silently ship an unconstrained NOT NULL since it
+--      would mean some org rows are unreachable.
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000046] Adding organizations.owner_tenant_id'; END $$;
+
+ALTER TABLE organizations
+    ADD COLUMN IF NOT EXISTS owner_tenant_id BIGINT;
+
+-- Pass 1: owner user still exists.
+UPDATE organizations o
+   SET owner_tenant_id = u.tenant_id
+  FROM users u
+ WHERE o.owner_id        = u.id
+   AND o.owner_tenant_id IS NULL;
+
+-- Pass 2: orphan owner — pick the earliest Admin tenant in OTM.
+UPDATE organizations o
+   SET owner_tenant_id = sub.tenant_id
+  FROM (
+        SELECT DISTINCT ON (otm.organization_id)
+               otm.organization_id,
+               otm.tenant_id
+          FROM organization_tenant_members otm
+         WHERE otm.role = 'admin'
+         ORDER BY otm.organization_id, otm.created_at ASC, otm.tenant_id ASC
+       ) sub
+ WHERE sub.organization_id = o.id
+   AND o.owner_tenant_id   IS NULL;
+
+-- Surface anything still unresolved. We deliberately fail the migration
+-- here instead of guessing — making the column NOT NULL with a bogus
+-- backfill would corrupt permission checks for those orgs forever.
+DO $$
+DECLARE
+    orphan_count INT;
+BEGIN
+    SELECT COUNT(*) INTO orphan_count
+      FROM organizations
+     WHERE owner_tenant_id IS NULL
+       AND deleted_at      IS NULL;
+    IF orphan_count > 0 THEN
+        RAISE EXCEPTION
+            '[Migration 000046] % orphan organization(s) have no resolvable owner_tenant_id (owner user missing AND no admin tenant in OTM). Either soft-delete them or backfill manually before retrying. Inspect with: SELECT id, name, owner_id FROM organizations WHERE owner_tenant_id IS NULL AND deleted_at IS NULL;',
+            orphan_count;
+    END IF;
+END $$;
+
+-- Lock in the invariant.
+ALTER TABLE organizations
+    ALTER COLUMN owner_tenant_id SET NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_organizations_owner_tenant
+    ON organizations (owner_tenant_id);
+
+COMMENT ON COLUMN organizations.owner_tenant_id IS
+    'Plan 3 (#1303): owning tenant; cannot be removed/downgraded from OTM.';
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000046] organizations.owner_tenant_id ready'; END $$;

migrations/versioned/000047_user_resource_favorites.down.sql

@@ -0,0 +1,8 @@
+-- Reverse of 000047_user_resource_favorites.
+DO $$ BEGIN RAISE NOTICE '[Migration 000047] Dropping table: user_resource_favorites'; END $$;
+
+DROP INDEX IF EXISTS idx_user_resource_favorites_tenant_id;
+DROP INDEX IF EXISTS idx_user_resource_favorites_user_tenant_type_created_at;
+DROP TABLE IF EXISTS user_resource_favorites;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000047] user_resource_favorites table dropped'; END $$;

migrations/versioned/000047_user_resource_favorites.up.sql

@@ -0,0 +1,43 @@
+-- Migration: 000047_user_resource_favorites
+-- Per-(user, tenant) starred resources (knowledge bases and custom agents).
+--
+-- Why (user_id, tenant_id) instead of (user_id) alone:
+--   A user can be a member of multiple tenants with very different
+--   resource sets; a KB called "design specs" in tenant A is not the
+--   same thing as "design specs" in tenant B and the user almost
+--   certainly wants to star them independently. Scoping by tenant also
+--   simplifies cleanup: drop a tenant -> drop its favorites cleanly via
+--   the (tenant_id) index without orphaning rows.
+--
+-- Why a single table instead of one per resource_type:
+--   Adding "favorite a mcp tool" or "favorite a wiki page" later is a
+--   new constant string, not a new schema migration. The PK includes
+--   resource_type so different types with colliding ids can coexist.
+--
+-- No FK to knowledge_bases / custom_agents:
+--   Favorites survive across share revocations / re-grants and across
+--   the eventual soft-delete -> hard-delete window. Hydrating with a
+--   LEFT JOIN at read time is cheap (<= ~30 rows per user) and lets us
+--   silently drop entries for resources the user can no longer see.
+DO $$ BEGIN RAISE NOTICE '[Migration 000047] Creating table: user_resource_favorites'; END $$;
+
+CREATE TABLE IF NOT EXISTS user_resource_favorites (
+    user_id        VARCHAR(36) NOT NULL,
+    tenant_id      BIGINT      NOT NULL,
+    resource_type  VARCHAR(16) NOT NULL,   -- 'kb' | 'agent' (extensible)
+    resource_id    VARCHAR(64) NOT NULL,
+    created_at     TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY (user_id, tenant_id, resource_type, resource_id)
+);
+
+-- Primary read path: "list this user's favorites of this type in this
+-- tenant, newest first". The PK already covers (user_id, tenant_id, …)
+-- equality, but we want created_at DESC ordering without a sort step.
+CREATE INDEX IF NOT EXISTS idx_user_resource_favorites_user_tenant_type_created_at
+    ON user_resource_favorites (user_id, tenant_id, resource_type, created_at DESC);
+
+-- Cleanup path when a tenant is deleted: bulk DELETE WHERE tenant_id = ?
+CREATE INDEX IF NOT EXISTS idx_user_resource_favorites_tenant_id
+    ON user_resource_favorites (tenant_id);
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000047] user_resource_favorites table ready'; END $$;

migrations/versioned/000048_tenant_invitations.down.sql

@@ -0,0 +1,9 @@
+-- Reverse of 000048_tenant_invitations.
+DO $$ BEGIN RAISE NOTICE '[Migration 000048] Dropping table: tenant_invitations'; END $$;
+
+DROP INDEX IF EXISTS idx_tenant_invitations_invitee;
+DROP INDEX IF EXISTS idx_tenant_invitations_tenant;
+DROP INDEX IF EXISTS idx_tenant_invitations_unique_pending;
+DROP TABLE IF EXISTS tenant_invitations;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000048] tenant_invitations table dropped'; END $$;

migrations/versioned/000048_tenant_invitations.up.sql

@@ -0,0 +1,59 @@
+-- Migration: 000048_tenant_invitations
+-- Adds an explicit "invitation -> accept" flow on top of the
+-- tenant-member RBAC model introduced in 000043. Previously, POST
+-- /tenants/:id/members wrote an active tenant_members row in one shot
+-- so the invitee was added without their knowledge or consent. The
+-- table below records the pending intent; the row is only promoted
+-- into a real tenant_members entry after the invitee accepts.
+--
+-- Why a separate table instead of reusing tenant_members.status='invited':
+--   - Mixing the two states pollutes every existing tenant_members
+--     read path with an extra "AND status='active'" filter, and the
+--     PRs that introduced kb / agent ownership already assume any
+--     non-deleted membership is active.
+--   - Declined / revoked / expired invitations carry forensic value
+--     (e.g. "did Bob ever try to add Eve?"); keeping them in their
+--     own table lets us retain the full history without touching the
+--     authoritative roster.
+--   - Re-inviting the same user is a natural use case; a separate
+--     table accumulates rows naturally with a partial unique index
+--     guarding the in-flight one.
+--
+-- Status machine:
+--   pending  -> accepted  (terminal; invitee accepts)
+--   pending  -> declined  (terminal; invitee rejects)
+--   pending  -> revoked   (terminal; owner cancels)
+--   pending  -> expired   (terminal; lazy sweep after expires_at)
+DO $$ BEGIN RAISE NOTICE '[Migration 000048] Creating table: tenant_invitations'; END $$;
+
+CREATE TABLE IF NOT EXISTS tenant_invitations (
+    id              BIGSERIAL   PRIMARY KEY,
+    tenant_id       INTEGER     NOT NULL,
+    invitee_user_id VARCHAR(36) NOT NULL,
+    invited_by      VARCHAR(36),
+    role            VARCHAR(20) NOT NULL,
+    status          VARCHAR(20) NOT NULL DEFAULT 'pending',
+    message         VARCHAR(500),
+    expires_at      TIMESTAMP WITH TIME ZONE NOT NULL,
+    responded_at    TIMESTAMP WITH TIME ZONE,
+    created_at      TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at      TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    deleted_at      TIMESTAMP WITH TIME ZONE
+);
+
+-- Partial unique index: at most one PENDING invitation per (tenant,
+-- invitee). Terminal-state rows (accepted/declined/revoked/expired)
+-- can accumulate freely so the history of past invites stays intact.
+CREATE UNIQUE INDEX IF NOT EXISTS idx_tenant_invitations_unique_pending
+    ON tenant_invitations(tenant_id, invitee_user_id)
+    WHERE status = 'pending' AND deleted_at IS NULL;
+
+-- Read paths:
+--   - Tenant management UI: list invitations for a tenant.
+--   - "My invitations" inbox: list invitations for a user.
+CREATE INDEX IF NOT EXISTS idx_tenant_invitations_tenant
+    ON tenant_invitations(tenant_id) WHERE deleted_at IS NULL;
+CREATE INDEX IF NOT EXISTS idx_tenant_invitations_invitee
+    ON tenant_invitations(invitee_user_id) WHERE deleted_at IS NULL;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000048] tenant_invitations table ready'; END $$;

migrations/versioned/000049_user_preferences.down.sql

@@ -0,0 +1,8 @@
+-- Rollback: drop users.preferences (data loss is acceptable; it only
+-- carries UI preferences, not anything users can't reset by clicking).
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000049 DOWN] Dropping users.preferences...'; END $$;
+
+ALTER TABLE users DROP COLUMN IF EXISTS preferences;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000049 DOWN] Done.'; END $$;

migrations/versioned/000049_user_preferences.up.sql

@@ -0,0 +1,24 @@
+-- Migration: 000049_user_preferences
+-- Adds a user-scoped JSON preferences blob so per-user UI toggles
+-- (memory feature, future preferences) can be persisted server-side and
+-- sync across devices/browsers. Previously the only carrier for such
+-- toggles was the browser's localStorage, which intentionally does NOT
+-- cross devices — switching machines silently reset every preference.
+--
+-- A single jsonb column keeps the schema flat: new keys can be added by
+-- writing through the existing PUT /auth/me/preferences endpoint, no DDL
+-- per knob. Server reads/writes are partial updates (key-merge), so an
+-- old client that only knows about a subset of keys can't accidentally
+-- wipe newer ones it doesn't understand.
+--
+-- Defaults to '{}' (NOT NULL) so existing rows behave as "no preference
+-- set yet" without forcing every read path to handle NULL.
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000049] Adding users.preferences column...'; END $$;
+
+ALTER TABLE users
+    ADD COLUMN IF NOT EXISTS preferences JSONB NOT NULL DEFAULT '{}'::jsonb;
+
+COMMENT ON COLUMN users.preferences IS 'Per-user JSON preferences (memory toggle, future UI knobs)';
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000049] Done.'; END $$;