xinmi/internal/application/service/knowledge_clone_move.go

package service

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"path/filepath"
	"slices"
	"strings"
	"time"

	"github.com/Tencent/WeKnora/internal/accesspolicy"
	"github.com/Tencent/WeKnora/internal/application/service/retriever"
	werrors "github.com/Tencent/WeKnora/internal/errors"
	"github.com/Tencent/WeKnora/internal/knowledgeversion"
	"github.com/Tencent/WeKnora/internal/logger"
	"github.com/Tencent/WeKnora/internal/tracing/langfuse"
	"github.com/Tencent/WeKnora/internal/types"
	"github.com/Tencent/WeKnora/internal/types/interfaces"
	"github.com/google/uuid"
	"github.com/hibiken/asynq"
	"github.com/redis/go-redis/v9"
	"golang.org/x/sync/errgroup"
)

// copyOwnedObject copies srcPath into a NEW object owned by the destination
// tenant, returning the new provider:// (resource) path.
//
// Extracted/embedded chunk images MUST land in the tenant's exports/ namespace,
// because GET /knowledge-bases/:id/files only serves objects that pass
// ValidateKBScopedStoragePath (i.e. {tenant}/exports/...). CopyFile writes to
// the knowledge-scoped upload layout ({tenant}/{knowledgeID}/...) used for raw
// source files, which the KB proxy rejects — so a clone that used CopyFile
// produced images that could no longer be rendered. Instead, read the source
// bytes and re-save them via SaveBytes, exactly mirroring how the original
// images were persisted during ingestion (see image_resolver.saveReferencedImage),
// so the copy is a genuine independent object in the servable namespace.
func copyOwnedObject(
	ctx context.Context,
	srcSvc, dstSvc interfaces.FileService,
	srcPath string,
	tenantID uint64,
	knowledgeID string,
) (string, error) {
	_ = knowledgeID // exports objects are tenant-scoped, not knowledge-scoped
	rc, err := srcSvc.GetFile(ctx, srcPath)
	if err != nil {
		return "", fmt.Errorf("read source image %q: %w", srcPath, err)
	}
	defer rc.Close()
	const maxCopiedImageBytes = 64 << 20
	data, err := io.ReadAll(io.LimitReader(rc, maxCopiedImageBytes+1))
	if err != nil {
		return "", fmt.Errorf("buffer source image %q: %w", srcPath, err)
	}
	if len(data) > maxCopiedImageBytes {
		return "", fmt.Errorf("embedded image exceeds the 64 MiB copy limit")
	}

	fileName := uuid.New().String() + imageExtForCopy(srcPath, data)
	newPath, err := dstSvc.SaveBytes(ctx, data, tenantID, fileName, false)
	if err != nil {
		return "", fmt.Errorf("save copied image for %q: %w", srcPath, err)
	}
	if newPath == "" || newPath == srcPath {
		return "", fmt.Errorf("image storage did not create an independent object")
	}
	return newPath, nil
}

// imageExtForCopy resolves the file extension to use for a copied image. It
// prefers an image extension already present on the source path, then falls
// back to sniffing the content bytes, and finally defaults to ".png" (matching
// image_resolver's default) so the object is always served with a sane type.
func imageExtForCopy(srcPath string, data []byte) string {
	if ext := strings.ToLower(filepath.Ext(srcPath)); isImageExt(ext) {
		return ext
	}
	switch http.DetectContentType(data) {
	case "image/png":
		return ".png"
	case "image/jpeg":
		return ".jpg"
	case "image/gif":
		return ".gif"
	case "image/webp":
		return ".webp"
	case "image/bmp":
		return ".bmp"
	}
	return ".png"
}

func isImageExt(ext string) bool {
	switch ext {
	case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".svg":
		return true
	default:
		return false
	}
}

// cloneChunkImageInfo parses a chunk's image_info JSON, copies every referenced
// object into a NEW object owned by (tenantID, knowledgeID), and returns the
// re-serialized image_info plus the list of newly-created object URLs (for
// rollback on failure). urlCache dedups identical source objects across chunks
// so the same source image is copied at most once per clone AND accumulates the
// full old->new URL mapping so callers can rewrite in-content Markdown image
// references (see rewriteContentImageURLs).
//
// An empty srcImageInfo yields ("", nil, nil). A JSON parse failure returns an
// error (the clone fails) rather than silently inheriting the shared-reference
// bug. When an image's OriginalURL points at the same object as its URL (the
// common case for extracted images), OriginalURL is rewritten to the new path
// too; an OriginalURL from a different/external source is preserved.
func cloneChunkImageInfo(
	ctx context.Context,
	dstSvc interfaces.FileService,
	srcImageInfo string,
	tenantID uint64,
	knowledgeID string,
	urlCache map[string]string,
) (newImageInfo string, copiedURLs []string, err error) {
	if srcImageInfo == "" {
		return "", nil, nil
	}

	var images []*types.ImageInfo
	if err := json.Unmarshal([]byte(srcImageInfo), &images); err != nil {
		return "", nil, fmt.Errorf("failed to parse chunk image_info JSON: %w", err)
	}

	for _, img := range images {
		if img == nil || img.URL == "" {
			continue
		}
		originalMatchedURL := img.OriginalURL == img.URL

		newURL, cached := urlCache[img.URL]
		if !cached {
			newURL, err = copyOwnedObject(ctx, dstSvc, dstSvc, img.URL, tenantID, knowledgeID)
			if err != nil {
				return "", copiedURLs, fmt.Errorf("failed to copy chunk image %q: %w", img.URL, err)
			}
			urlCache[img.URL] = newURL
			copiedURLs = append(copiedURLs, newURL)
		}

		if originalMatchedURL {
			img.OriginalURL = newURL
		}
		img.URL = newURL
	}

	out, err := json.Marshal(images)
	if err != nil {
		return "", copiedURLs, fmt.Errorf("failed to re-serialize chunk image_info: %w", err)
	}
	return string(out), copiedURLs, nil
}

// rewriteContentImageURLs replaces every occurrence of an old image URL with its
// new (copied) URL in content, using the old->new mapping accumulated in
// urlCache. It is the second half of the image deep-copy: chunk Content embeds
// image URLs as Markdown ![](url) references, but for document knowledge the
// image objects live in independent image_ocr/image_caption child chunks — the
// parent text chunk carries the ![](url) reference with an empty image_info. So
// the old->new mapping is only known after every chunk's image_info has been
// processed; this rewrite must therefore run as a final pass once urlCache is
// complete, over ALL cloned chunks, not per-chunk.
//
// Replacements are applied longest-old-URL first so a URL that is a prefix of
// another is not partially rewritten. Entries whose old==new are skipped.
func rewriteContentImageURLs(content string, urlCache map[string]string) string {
	if content == "" || len(urlCache) == 0 {
		return content
	}
	oldURLs := make([]string, 0, len(urlCache))
	for oldURL, newURL := range urlCache {
		if oldURL == "" || oldURL == newURL {
			continue
		}
		oldURLs = append(oldURLs, oldURL)
	}
	slices.SortFunc(oldURLs, func(a, b string) int { return len(b) - len(a) })
	for _, oldURL := range oldURLs {
		content = strings.ReplaceAll(content, oldURL, urlCache[oldURL])
	}
	return content
}

// cleanupCopiedObjects deletes objects that were newly created during a clone
// that subsequently failed, to avoid orphaning storage. It is best-effort:
// delete errors are logged but never returned (the original clone error wins).
func cleanupCopiedObjects(ctx context.Context, svc interfaces.FileService, paths []string) {
	if len(paths) == 0 || svc == nil {
		return
	}
	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
	defer cancel()
	logger.Infof(ctx, "Cleaning up %d copied objects after clone failure", len(paths))
	for _, p := range paths {
		if p == "" {
			continue
		}
		if err := svc.DeleteFile(ctx, p); err != nil {
			logger.Errorf(ctx, "Failed to clean up copied object %s: %v", p, err)
		}
	}
}

func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error {
	if err := checkKnowledgeCopyLineage(ctx, srcID); err != nil {
		return err
	}
	srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, srcID, dstID)
	if err != nil {
		logger.Errorf(ctx, "Failed to copy knowledge base: %v", err)
		return err
	}

	addKnowledge, err := s.repo.AminusB(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
	if err != nil {
		logger.Errorf(ctx, "Failed to get knowledge: %v", err)
		return err
	}

	delKnowledge, err := s.repo.AminusB(ctx, dstKB.TenantID, dstKB.ID, srcKB.TenantID, srcKB.ID)
	if err != nil {
		logger.Errorf(ctx, "Failed to get knowledge: %v", err)
		return err
	}
	logger.Infof(ctx, "Knowledge after update to add: %d, delete: %d", len(addKnowledge), len(delKnowledge))

	batch := 10
	g, gctx := errgroup.WithContext(ctx)
	for ids := range slices.Chunk(delKnowledge, batch) {
		g.Go(func() error {
			err := s.DeleteKnowledgeList(gctx, ids)
			if err != nil {
				logger.Errorf(gctx, "delete partial knowledge %v: %v", ids, err)
				return err
			}
			return nil
		})
	}
	err = g.Wait()
	if err != nil {
		logger.Errorf(ctx, "delete total knowledge %d: %v", len(delKnowledge), err)
		return err
	}

	// Copy context out of auto-stop task
	g, gctx = errgroup.WithContext(ctx)
	g.SetLimit(batch)
	for _, knowledge := range addKnowledge {
		g.Go(func() error {
			srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge)
			if err != nil {
				logger.Errorf(gctx, "get knowledge %s: %v", knowledge, err)
				return err
			}
			err = s.cloneKnowledge(gctx, srcKn, dstKB)
			if err != nil {
				logger.Errorf(gctx, "clone knowledge %s: %v", knowledge, err)
				return err
			}
			return nil
		})
	}
	err = g.Wait()
	if err != nil {
		logger.Errorf(ctx, "add total knowledge %d: %v", len(addKnowledge), err)
		return err
	}
	return nil
}

// CloneChunk clone chunks from one knowledge to another
// This method transfers a chunk from a source knowledge document to a target knowledge document
// It handles the creation of new chunks in the target knowledge and updates the vector database accordingly
// Parameters:
//   - ctx: Context with authentication and request information
//   - src: Source knowledge document containing the chunk to move
//   - dst: Target knowledge document where the chunk will be moved
//
// Returns:
//   - error: Any error encountered during the move operation
//
// This method handles the chunk transfer logic, including creating new chunks in the target knowledge
// and updating the vector database representation of the moved chunks.
// It also ensures that the chunk's relationships (like pre and next chunk IDs) are maintained
// by mapping the source chunk IDs to the new target chunk IDs.
func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowledge) (err error) {
	return s.cloneChunk(ctx, src, dst, map[string]string{})
}

func (s *knowledgeService) cloneChunk(ctx context.Context, src, dst *types.Knowledge, tagIDMapping map[string]string, versionMappings ...map[string]string) (err error) {
	if err := checkKnowledgeCopyLineage(ctx, src.KnowledgeBaseID); err != nil {
		return err
	}
	chunkPage := 1
	chunkPageSize := 100
	srcTodst := map[string]string{}
	targetChunks := make([]*types.Chunk, 0, 10)
	chunkType := []types.ChunkType{
		types.ChunkTypeText, types.ChunkTypeParentText, types.ChunkTypeSummary,
		types.ChunkTypeImageCaption, types.ChunkTypeImageOCR,
	}
	versionCopy := knowledgeversion.IsWorker(ctx)
	if versionCopy {
		chunkType = append(chunkType, types.ChunkTypeFAQ, types.ChunkTypeEntity, types.ChunkTypeRelationship,
			types.ChunkTypeWebSearch, types.ChunkTypeTableSummary, types.ChunkTypeTableColumn, types.ChunkTypeWikiPage)
	}

	// Resolve the destination FileService so extracted images can be copied
	// into objects owned by the destination knowledge. urlCache dedups identical
	// source images across chunks; copiedURLs accumulates new objects so they can
	// be cleaned up if the clone fails partway through.
	dstKB, dstKBErr := s.kbService.GetKnowledgeBaseByID(ctx, dst.KnowledgeBaseID)
	if dstKBErr != nil {
		return fmt.Errorf("failed to load destination knowledge base for image copy: %w", dstKBErr)
	}
	dstSvc := s.resolveFileService(ctx, dstKB)
	if versionCopy {
		dstSvc = &versionTrackingFileService{FileService: dstSvc, service: s, knowledge: dst}
	}
	urlCache := map[string]string{}
	var copiedURLs []string
	var references *versionReferenceCopier
	defer func() {
		if err != nil {
			if references != nil {
				copiedURLs = append(copiedURLs, references.copied...)
			}
			cleanupCopiedObjects(ctx, dstSvc, copiedURLs)
		}
	}()
	if versionCopy {
		if src.FilePath != "" && dst.FilePath != "" {
			urlCache[src.FilePath] = dst.FilePath
		}
		srcKB, loadErr := s.kbService.GetKnowledgeBaseByID(ctx, src.KnowledgeBaseID)
		if loadErr != nil {
			return loadErr
		}
		references = &versionReferenceCopier{ctx: ctx, source: func(path string) interfaces.FileService {
			return s.resolveFileServiceForPath(ctx, srcKB, path)
		}, destination: dstSvc, tenantID: dst.TenantID, knowledgeID: dst.ID, urls: urlCache}
		for _, field := range []struct {
			source types.JSON
			target *types.JSON
		}{
			{src.Metadata, &dst.Metadata}, {src.CustomMetadata, &dst.CustomMetadata},
		} {
			value, copyErr := references.json(field.source)
			if copyErr != nil {
				return copyErr
			}
			*field.target = value
		}
	}

	for {
		var sourceChunks []*types.Chunk
		var listErr error
		if versionCopy {
			if chunkPage > 1 {
				break
			}
			// The paginated browse API excludes stored parent chunks. Version
			// copies must include them to preserve the original context tree.
			sourceChunks, listErr = s.chunkRepo.ListChunksByKnowledgeIDAndTypes(ctx, src.TenantID, src.ID, chunkType)
		} else {
			sourceChunks, _, listErr = s.chunkRepo.ListPagedChunksByKnowledgeID(ctx,
				src.TenantID,
				src.ID,
				&types.Pagination{
					Page:     chunkPage,
					PageSize: chunkPageSize,
				},
				chunkType,
				nil,
				"",
				"",
				"",
				"",
				nil,
			)
		}
		chunkPage++
		if listErr != nil {
			return listErr
		}
		if len(sourceChunks) == 0 {
			break
		}
		now := time.Now()
		for _, sourceChunk := range sourceChunks {
			if versionCopy {
				switch sourceChunk.ChunkType {
				case types.ChunkTypeText, types.ChunkTypeParentText, types.ChunkTypeSummary,
					types.ChunkTypeImageCaption, types.ChunkTypeImageOCR, types.ChunkTypeFAQ:
				default:
					return fmt.Errorf("当前知识分块类型尚不支持独立版本复制: %s", sourceChunk.ChunkType)
				}
				if sourceChunk.IndexStatus != "" && sourceChunk.IndexStatus != "ready" {
					return fmt.Errorf("知识分块尚未完成索引: %s", sourceChunk.ID)
				}
			}
			// Map TagID to target knowledge base
			targetTagID := ""
			if sourceChunk.TagID != "" {
				if mappedTagID, ok := tagIDMapping[sourceChunk.TagID]; ok {
					targetTagID = mappedTagID
				} else {
					if versionCopy {
						return fmt.Errorf("知识分类未完整复制: %s", sourceChunk.TagID)
					}
					// Try to find or create the tag in target knowledge base
					targetTagID = s.getOrCreateTagInTarget(ctx, src.TenantID, dst.TenantID, dst.KnowledgeBaseID, sourceChunk.TagID, tagIDMapping)
				}
			}

			// Deep-copy extracted images into objects owned by the destination
			// knowledge so deleting the source never breaks this clone. Content
			// URL rewriting happens in a final pass below, once urlCache holds
			// the complete old->new mapping (image objects live in independent
			// child chunks, so a parent text chunk's ![](url) reference cannot be
			// rewritten until its child image chunk has been processed).
			var newImageInfo string
			var copied []string
			var copyErr error
			if versionCopy {
				var imageJSON types.JSON
				imageJSON, copyErr = references.json([]byte(sourceChunk.ImageInfo))
				newImageInfo = string(imageJSON)
			} else {
				newImageInfo, copied, copyErr = cloneChunkImageInfo(
					ctx, dstSvc, sourceChunk.ImageInfo, dst.TenantID, dst.ID, urlCache)
			}
			copiedURLs = append(copiedURLs, copied...)
			if copyErr != nil {
				err = fmt.Errorf("clone chunk image copy failed: %w", copyErr)
				return err
			}

			targetChunk := &types.Chunk{
				ID:              uuid.New().String(),
				TenantID:        dst.TenantID,
				KnowledgeID:     dst.ID,
				KnowledgeBaseID: dst.KnowledgeBaseID,
				TagID:           targetTagID,
				Content:         sourceChunk.Content,
				SourceContent:   sourceChunk.SourceContent,
				ContentRevision: sourceChunk.ContentRevision,
				IndexStatus:     sourceChunk.IndexStatus,
				LastEditorID:    sourceChunk.LastEditorID,
				ContextHeader:   sourceChunk.ContextHeader,
				ChunkIndex:      sourceChunk.ChunkIndex,
				IsEnabled:       sourceChunk.IsEnabled,
				Flags:           sourceChunk.Flags,
				Status:          sourceChunk.Status,
				StartAt:         sourceChunk.StartAt,
				EndAt:           sourceChunk.EndAt,
				PreChunkID:      sourceChunk.PreChunkID,
				NextChunkID:     sourceChunk.NextChunkID,
				ChunkType:       sourceChunk.ChunkType,
				ParentChunkID:   sourceChunk.ParentChunkID,
				Metadata:        sourceChunk.Metadata,
				ContentHash:     sourceChunk.ContentHash,
				ImageInfo:       newImageInfo,
				CreatedAt:       now,
				UpdatedAt:       now,
			}
			if versionCopy {
				for _, value := range []*string{&targetChunk.Content, &targetChunk.SourceContent, &targetChunk.ContextHeader} {
					copied, copyErr := references.text(*value)
					if copyErr != nil {
						return copyErr
					}
					*value = copied
				}
				targetChunk.Metadata, copyErr = references.json(sourceChunk.Metadata)
				if copyErr != nil {
					return copyErr
				}
			}
			targetChunks = append(targetChunks, targetChunk)
			srcTodst[sourceChunk.ID] = targetChunk.ID
		}
	}
	for _, targetChunk := range targetChunks {
		// Rewrite in-content Markdown image URLs now that urlCache holds the
		// complete old->new mapping across all chunks. This fixes parent text
		// chunks whose ![](url) reference points at a source object copied while
		// processing an independent image_ocr/image_caption child chunk.
		targetChunk.Content = rewriteContentImageURLs(targetChunk.Content, urlCache)
		targetChunk.SourceContent = rewriteContentImageURLs(targetChunk.SourceContent, urlCache)
		if val, ok := srcTodst[targetChunk.PreChunkID]; ok {
			targetChunk.PreChunkID = val
		} else {
			targetChunk.PreChunkID = ""
		}
		if val, ok := srcTodst[targetChunk.NextChunkID]; ok {
			targetChunk.NextChunkID = val
		} else {
			targetChunk.NextChunkID = ""
		}
		if val, ok := srcTodst[targetChunk.ParentChunkID]; ok {
			targetChunk.ParentChunkID = val
		} else {
			targetChunk.ParentChunkID = ""
		}
	}
	if references != nil && len(references.copied) > 0 {
		if len(targetChunks) == 0 {
			return fmt.Errorf("带有附件的知识没有可复制分块")
		}
		// Native deletion collects image_info. Keep metadata-only attachments
		// in that inventory as well so later deletion can reclaim every copy.
		var inventory []*types.ImageInfo
		if targetChunks[0].ImageInfo != "" {
			if err := json.Unmarshal([]byte(targetChunks[0].ImageInfo), &inventory); err != nil {
				return err
			}
		}
		seen := map[string]bool{}
		for _, img := range inventory {
			if img != nil {
				seen[img.URL] = true
			}
		}
		for _, path := range references.copied {
			if !seen[path] {
				inventory = append(inventory, &types.ImageInfo{URL: path, OriginalURL: path})
				seen[path] = true
			}
			if _, resourceReference := types.ParseResourcePath(path); resourceReference && s.resourceCatalog != nil {
				if err := s.resourceCatalog.Bind(ctx, path, types.ResourceOwnerKnowledge, dst.ID, types.ResourceRelationExtractedImage); err != nil {
					return err
				}
			}
		}
		encoded, marshalErr := json.Marshal(inventory)
		if marshalErr != nil {
			return marshalErr
		}
		targetChunks[0].ImageInfo = string(encoded)
	}
	for chunks := range slices.Chunk(targetChunks, chunkPageSize) {
		var state []types.Chunk
		if versionCopy {
			for _, chunk := range chunks {
				state = append(state, *chunk)
			}
		}
		err := s.chunkRepo.CreateChunks(ctx, chunks)
		if err != nil {
			return err
		}
		if versionCopy {
			var corrected []*types.Chunk
			for i, chunk := range chunks {
				if chunk.IsEnabled != state[i].IsEnabled || chunk.Flags != state[i].Flags {
					chunk.IsEnabled, chunk.Flags = state[i].IsEnabled, state[i].Flags
					corrected = append(corrected, chunk)
				}
			}
			if len(corrected) > 0 {
				if err := s.chunkRepo.UpdateChunks(ctx, corrected); err != nil {
					return err
				}
			}
		}
	}

	tenantID := types.MustTenantIDFromContext(ctx)
	// Route CopyIndices via the source KB's bound store. This function does
	// not handle cross-store copies — embeddings written by different
	// VectorStore backends are not bit-compatible, so callers that allow
	// source/target KBs to bind to different stores must perform their own
	// cross-store migration before invoking this.
	var sourceStoreID *string
	if srcKB, loadErr := s.kbService.GetKnowledgeBaseByID(ctx, src.KnowledgeBaseID); loadErr == nil && srcKB != nil {
		sourceStoreID = srcKB.VectorStoreID
	}
	retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
		ctx, s.retrieveEngine, s.ownership, tenantID, sourceStoreID)
	if err != nil {
		return err
	}
	embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, dst.EmbeddingModelID)
	if err != nil {
		return err
	}
	if err := retrieveEngine.CopyIndices(ctx, src.KnowledgeBaseID, dst.KnowledgeBaseID,
		map[string]string{src.ID: dst.ID},
		srcTodst,
		embeddingModel.GetDimensions(),
		dst.Type,
	); err != nil {
		return err
	}
	if knowledgeversion.IsWorker(ctx) {
		tags := make(map[string]string, len(targetChunks))
		enabled := make(map[string]bool, len(targetChunks))
		for _, chunk := range targetChunks {
			tags[chunk.ID] = chunk.TagID
			enabled[chunk.ID] = chunk.IsEnabled && src.EnableStatus != "disabled"
		}
		if len(tags) > 0 {
			if err := retrieveEngine.BatchUpdateChunkTagID(ctx, tags); err != nil {
				return err
			}
			if err := retrieveEngine.BatchUpdateChunkEnabledStatus(ctx, enabled); err != nil {
				return err
			}
		}
	}
	for _, mapping := range versionMappings {
		for source, target := range srcTodst {
			mapping[source] = target
		}
	}
	return nil
}

func checkKnowledgeCopyLineage(ctx context.Context, sourceID string) error {
	if knowledgeversion.IsWorker(ctx) || types.IsSystemAdminFromContext(ctx) || types.TenantRoleFromContext(ctx).HasPermission(types.TenantRoleAdmin) {
		return nil
	}
	snapshot := accesspolicy.FromContext(ctx)
	if snapshot != nil && (snapshot.Publications["knowledge:"+sourceID] || snapshot.NoExport["knowledge:"+sourceID]) {
		return werrors.NewForbiddenError("全局知识库及其个人版本不能复制或移动到其他知识库，请使用个人版本")
	}
	return nil
}

const (
	kbCloneProgressKeyPrefix = "kb_clone_progress:"
	kbCloneProgressTTL       = 24 * time.Hour
)

// getKBCloneProgressKey returns the Redis key for storing KB clone progress
func getKBCloneProgressKey(taskID string) string {
	return kbCloneProgressKeyPrefix + taskID
}

// ProcessKBClone handles Asynq knowledge base clone tasks
func (s *knowledgeService) ProcessKBClone(ctx context.Context, t *asynq.Task) error {
	var payload types.KBClonePayload
	if err := json.Unmarshal(t.Payload(), &payload); err != nil {
		return fmt.Errorf("failed to unmarshal KB clone payload: %w", err)
	}
	ctx = payload.Initiator.Apply(ctx)
	ctx = withKBActivityTask(ctx, payload.TaskID, kbActivityTrigger(ctx))

	// Add tenant ID to context
	ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)

	// Get tenant info and add to context
	tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
	if err != nil {
		logger.Errorf(ctx, "Failed to get tenant info: %v", err)
		return fmt.Errorf("failed to get tenant info: %w", err)
	}
	ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)

	// Check if this is the last retry
	retryCount, _ := asynq.GetRetryCount(ctx)
	maxRetry, _ := asynq.GetMaxRetry(ctx)
	isLastRetry := retryCount >= maxRetry

	logger.Infof(ctx, "Processing KB clone task: %s, source: %s, target: %s, retry: %d/%d",
		payload.TaskID, payload.SourceID, payload.TargetID, retryCount, maxRetry)

	// Helper function to handle errors - only mark as failed on last retry
	handleError := func(progress *types.KBCloneProgress, err error, message string) {
		if isLastRetry {
			progress.Status = types.KBCloneStatusFailed
			progress.Error = err.Error()
			progress.Message = message
			progress.UpdatedAt = time.Now().Unix()
			_ = s.saveKBCloneProgress(ctx, progress)
			recordKBActivity(ctx, s.audit, payload.TenantID, payload.TargetID, types.AuditActionKBCloneFailed,
				"knowledge_base", payload.TargetID, types.AuditOutcomeFailed,
				map[string]any{"source_kb_id": payload.SourceID, "task_id": payload.TaskID})
		}
	}

	// Update progress to processing
	progress := &types.KBCloneProgress{
		TaskID:    payload.TaskID,
		SourceID:  payload.SourceID,
		TargetID:  payload.TargetID,
		Status:    types.KBCloneStatusProcessing,
		Progress:  0,
		Message:   "Starting knowledge base clone...",
		UpdatedAt: time.Now().Unix(),
	}
	if err := s.saveKBCloneProgress(ctx, progress); err != nil {
		logger.Errorf(ctx, "Failed to update KB clone progress: %v", err)
	}

	// Get source and target knowledge bases
	srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, payload.SourceID, payload.TargetID)
	if err != nil {
		logger.Errorf(ctx, "Failed to copy knowledge base: %v", err)
		handleError(progress, err, "Failed to copy knowledge base configuration")
		return err
	}
	if retryCount == 0 {
		recordKBActivity(ctx, s.audit, payload.TenantID, payload.TargetID, types.AuditActionKBCloneStarted,
			"knowledge_base", payload.TargetID, types.AuditOutcomeAccepted,
			map[string]any{"source_kb_id": payload.SourceID, "task_id": payload.TaskID})
	}

	// Use different sync strategies based on knowledge base type
	if srcKB.Type == types.KnowledgeBaseTypeFAQ {
		if err := s.cloneFAQKnowledgeBase(ctx, srcKB, dstKB, progress, handleError); err != nil {
			return err
		}
		recordKBActivity(ctx, s.audit, payload.TenantID, payload.TargetID, types.AuditActionKBCloneCompleted,
			"knowledge_base", payload.TargetID, types.AuditOutcomeSuccess,
			map[string]any{"source_kb_id": payload.SourceID, "task_id": payload.TaskID, "total": progress.Total})
		return nil
	}

	// Document type: use Knowledge-level diff based on file_hash
	addKnowledge, err := s.repo.AminusB(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
	if err != nil {
		logger.Errorf(ctx, "Failed to get knowledge to add: %v", err)
		handleError(progress, err, "Failed to calculate knowledge difference")
		return err
	}

	delKnowledge, err := s.repo.AminusB(ctx, dstKB.TenantID, dstKB.ID, srcKB.TenantID, srcKB.ID)
	if err != nil {
		logger.Errorf(ctx, "Failed to get knowledge to delete: %v", err)
		handleError(progress, err, "Failed to calculate knowledge difference")
		return err
	}

	totalOperations := len(addKnowledge) + len(delKnowledge)
	progress.Total = totalOperations
	progress.Message = fmt.Sprintf("Found %d knowledge to add, %d to delete", len(addKnowledge), len(delKnowledge))
	progress.UpdatedAt = time.Now().Unix()
	_ = s.saveKBCloneProgress(ctx, progress)

	logger.Infof(ctx, "Knowledge after update to add: %d, delete: %d", len(addKnowledge), len(delKnowledge))

	processedCount := 0
	batch := 10

	// Delete knowledge in target that doesn't exist in source
	g, gctx := errgroup.WithContext(ctx)
	for ids := range slices.Chunk(delKnowledge, batch) {
		g.Go(func() error {
			err := s.DeleteKnowledgeList(gctx, ids)
			if err != nil {
				logger.Errorf(gctx, "delete partial knowledge %v: %v", ids, err)
				return err
			}
			return nil
		})
	}
	if err := g.Wait(); err != nil {
		logger.Errorf(ctx, "delete total knowledge %d: %v", len(delKnowledge), err)
		handleError(progress, err, "Failed to delete knowledge")
		return err
	}

	processedCount += len(delKnowledge)
	if totalOperations > 0 {
		progress.Progress = processedCount * 100 / totalOperations
	}
	progress.Processed = processedCount
	progress.Message = fmt.Sprintf("Deleted %d knowledge, cloning %d...", len(delKnowledge), len(addKnowledge))
	progress.UpdatedAt = time.Now().Unix()
	_ = s.saveKBCloneProgress(ctx, progress)

	// Clone knowledge from source to target
	g, gctx = errgroup.WithContext(ctx)
	g.SetLimit(batch)
	for _, knowledge := range addKnowledge {
		g.Go(func() error {
			srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge)
			if err != nil {
				logger.Errorf(gctx, "get knowledge %s: %v", knowledge, err)
				return err
			}
			err = s.cloneKnowledge(gctx, srcKn, dstKB)
			if err != nil {
				logger.Errorf(gctx, "clone knowledge %s: %v", knowledge, err)
				return err
			}

			// Update progress
			processedCount++
			if totalOperations > 0 {
				progress.Progress = processedCount * 100 / totalOperations
			}
			progress.Processed = processedCount
			progress.Message = fmt.Sprintf("Cloned %d/%d knowledge", processedCount-len(delKnowledge), len(addKnowledge))
			progress.UpdatedAt = time.Now().Unix()
			_ = s.saveKBCloneProgress(ctx, progress)

			return nil
		})
	}
	if err := g.Wait(); err != nil {
		logger.Errorf(ctx, "add total knowledge %d: %v", len(addKnowledge), err)
		handleError(progress, err, "Failed to clone knowledge")
		return err
	}

	// Mark as completed
	progress.Status = types.KBCloneStatusCompleted
	progress.Progress = 100
	progress.Processed = totalOperations
	progress.Message = "Knowledge base clone completed successfully"
	progress.UpdatedAt = time.Now().Unix()
	if err := s.saveKBCloneProgress(ctx, progress); err != nil {
		logger.Errorf(ctx, "Failed to update KB clone progress to completed: %v", err)
	}

	logger.Infof(ctx, "KB clone task completed: %s", payload.TaskID)
	recordKBActivity(ctx, s.audit, payload.TenantID, payload.TargetID, types.AuditActionKBCloneCompleted,
		"knowledge_base", payload.TargetID, types.AuditOutcomeSuccess,
		map[string]any{"source_kb_id": payload.SourceID, "task_id": payload.TaskID, "total": totalOperations})
	return nil
}

// cloneFAQKnowledgeBase handles FAQ knowledge base cloning with chunk-level incremental sync
func (s *knowledgeService) cloneFAQKnowledgeBase(
	ctx context.Context,
	srcKB, dstKB *types.KnowledgeBase,
	progress *types.KBCloneProgress,
	handleError func(*types.KBCloneProgress, error, string),
) (retErr error) {
	// Deep-copy extracted FAQ images into objects owned by the destination KB.
	// urlCache dedups identical source images across chunks; copiedURLs tracks
	// new objects for best-effort cleanup if the clone fails partway through.
	dstSvc := s.resolveFileService(ctx, dstKB)
	imageURLCache := map[string]string{}
	var copiedImageURLs []string
	defer func() {
		if retErr != nil {
			cleanupCopiedObjects(ctx, dstSvc, copiedImageURLs)
		}
	}()

	// Get source FAQ knowledge first (FAQ KB has exactly one Knowledge entry)
	srcKnowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, srcKB.TenantID, srcKB.ID)
	if err != nil {
		logger.Errorf(ctx, "Failed to get source FAQ knowledge: %v", err)
		handleError(progress, err, "Failed to get source FAQ knowledge")
		return err
	}
	if len(srcKnowledgeList) == 0 {
		// Source has no FAQ knowledge, nothing to clone
		progress.Status = types.KBCloneStatusCompleted
		progress.Progress = 100
		progress.Message = "Source FAQ knowledge base is empty"
		progress.UpdatedAt = time.Now().Unix()
		_ = s.saveKBCloneProgress(ctx, progress)
		return nil
	}
	srcKnowledge := srcKnowledgeList[0]

	// Get chunk-level differences based on content_hash.
	diff, err := s.chunkRepo.FAQChunkDiff(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
	if err != nil {
		logger.Errorf(ctx, "Failed to calculate FAQ chunk difference: %v", err)
		handleError(progress, err, "Failed to calculate FAQ chunk difference")
		return err
	}
	chunksToAdd := diff.ChunksToAdd
	chunksToDelete := diff.ChunksToDelete

	tagIDMapping := map[string]string{}
	resolveFAQTag := func(srcTagID string) string {
		if srcTagID == "" {
			return ""
		}
		if id, ok := tagIDMapping[srcTagID]; ok {
			return id
		}
		return s.getOrCreateTagInTarget(ctx, srcKB.TenantID, dstKB.TenantID, dstKB.ID, srcTagID, tagIDMapping)
	}

	syncPlan, err := s.buildFAQStatusSyncPlan(ctx, srcKB.TenantID, dstKB.TenantID, diff.MatchedPairs, resolveFAQTag)
	if err != nil {
		logger.Errorf(ctx, "Failed to build FAQ status sync plan: %v", err)
		handleError(progress, err, "Failed to build FAQ status sync plan")
		return err
	}
	chunksToUpdate := syncPlan.Pairs

	totalOperations := len(chunksToAdd) + len(chunksToDelete) + len(chunksToUpdate)
	progress.Total = totalOperations
	progress.Message = fmt.Sprintf(
		"Found %d FAQ entries to add, %d to delete, %d to update status",
		len(chunksToAdd), len(chunksToDelete), len(chunksToUpdate),
	)
	progress.UpdatedAt = time.Now().Unix()
	_ = s.saveKBCloneProgress(ctx, progress)

	logger.Infof(ctx, "FAQ chunks to add: %d, delete: %d, update status: %d",
		len(chunksToAdd), len(chunksToDelete), len(chunksToUpdate))

	if totalOperations == 0 {
		progress.Status = types.KBCloneStatusCompleted
		progress.Progress = 100
		progress.Message = "FAQ knowledge base is already in sync"
		progress.UpdatedAt = time.Now().Unix()
		_ = s.saveKBCloneProgress(ctx, progress)
		return nil
	}

	// Route the FAQ clone through the source KB's bound store. Same
	// constraint as CloneChunk: callers must ensure source and target share
	// the same VectorStore (cross-store FAQ clone is not handled here).
	var sourceStoreID *string
	if srcKB != nil {
		sourceStoreID = srcKB.VectorStoreID
	}
	retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
		ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), sourceStoreID)
	if err != nil {
		logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
		handleError(progress, err, "Failed to initialize retrieve engine")
		return err
	}

	// Get embedding model
	embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, dstKB.EmbeddingModelID)
	if err != nil {
		logger.Errorf(ctx, "Failed to get embedding model: %v", err)
		handleError(progress, err, "Failed to get embedding model")
		return err
	}

	processedCount := 0

	// Delete FAQ chunks that don't exist in source
	if len(chunksToDelete) > 0 {
		// Delete from vector store
		if err := retrieveEngine.DeleteByChunkIDList(ctx, chunksToDelete, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil {
			logger.Errorf(ctx, "Failed to delete FAQ chunks from vector store: %v", err)
			handleError(progress, err, "Failed to delete FAQ entries from vector store")
			return err
		}
		// Delete from database
		if err := s.chunkRepo.DeleteChunks(ctx, dstKB.TenantID, chunksToDelete); err != nil {
			logger.Errorf(ctx, "Failed to delete FAQ chunks from database: %v", err)
			handleError(progress, err, "Failed to delete FAQ entries from database")
			return err
		}
		processedCount += len(chunksToDelete)
		if totalOperations > 0 {
			progress.Progress = processedCount * 100 / totalOperations
		}
		progress.Processed = processedCount
		progress.Message = fmt.Sprintf("Deleted %d FAQ entries, adding %d...", len(chunksToDelete), len(chunksToAdd))
		progress.UpdatedAt = time.Now().Unix()
		_ = s.saveKBCloneProgress(ctx, progress)
	}

	// Get or create the FAQ knowledge entry in destination
	dstKnowledge, err := s.getOrCreateFAQKnowledge(ctx, dstKB, srcKnowledge)
	if err != nil {
		logger.Errorf(ctx, "Failed to get or create FAQ knowledge: %v", err)
		handleError(progress, err, "Failed to prepare FAQ knowledge entry")
		return err
	}

	// Clone FAQ chunks from source to destination
	batch := 50
	for i := 0; i < len(chunksToAdd); i += batch {
		end := i + batch
		if end > len(chunksToAdd) {
			end = len(chunksToAdd)
		}
		batchIDs := chunksToAdd[i:end]

		// Get source chunks
		srcChunks, err := s.chunkRepo.ListChunksByID(ctx, srcKB.TenantID, batchIDs)
		if err != nil {
			logger.Errorf(ctx, "Failed to get source FAQ chunks: %v", err)
			handleError(progress, err, "Failed to get source FAQ entries")
			return err
		}

		// Create new chunks for destination
		newChunks := make([]*types.Chunk, 0, len(srcChunks))
		for _, srcChunk := range srcChunks {
			// Map TagID to target knowledge base
			targetTagID := ""
			if srcChunk.TagID != "" {
				targetTagID = resolveFAQTag(srcChunk.TagID)
			}

			// Deep-copy extracted images into objects owned by the destination
			// FAQ knowledge so deleting the source never breaks this clone.
			newImageInfo, copied, copyErr := cloneChunkImageInfo(
				ctx, dstSvc, srcChunk.ImageInfo, dstKB.TenantID, dstKnowledge.ID, imageURLCache)
			if copyErr != nil {
				logger.Errorf(ctx, "Failed to copy FAQ chunk images: %v", copyErr)
				handleError(progress, copyErr, "Failed to copy FAQ entry images")
				retErr = copyErr
				return retErr
			}
			copiedImageURLs = append(copiedImageURLs, copied...)

			newChunk := &types.Chunk{
				ID:              uuid.New().String(),
				TenantID:        dstKB.TenantID,
				KnowledgeID:     dstKnowledge.ID,
				KnowledgeBaseID: dstKB.ID,
				TagID:           targetTagID,
				Content:         rewriteContentImageURLs(srcChunk.Content, imageURLCache),
				ChunkIndex:      srcChunk.ChunkIndex,
				IsEnabled:       srcChunk.IsEnabled,
				Flags:           srcChunk.Flags,
				ChunkType:       types.ChunkTypeFAQ,
				Metadata:        srcChunk.Metadata,
				ContentHash:     srcChunk.ContentHash,
				ImageInfo:       newImageInfo,
				Status:          int(types.ChunkStatusStored), // Initially stored, will be indexed
				CreatedAt:       time.Now(),
				UpdatedAt:       time.Now(),
			}
			newChunks = append(newChunks, newChunk)
		}

		// Save to database
		if err := s.chunkRepo.CreateChunks(ctx, newChunks); err != nil {
			logger.Errorf(ctx, "Failed to create FAQ chunks: %v", err)
			handleError(progress, err, "Failed to create FAQ entries")
			return err
		}

		// Index in vector store using existing method
		// This will index standard question + similar questions based on FAQConfig
		if err := s.indexFAQChunks(ctx, dstKB, dstKnowledge, newChunks, embeddingModel, false, false); err != nil {
			logger.Errorf(ctx, "Failed to index FAQ chunks: %v", err)
			handleError(progress, err, "Failed to index FAQ entries")
			return err
		}

		// Update chunk status to indexed
		for _, chunk := range newChunks {
			chunk.Status = int(types.ChunkStatusIndexed)
		}
		if err := s.chunkService.UpdateChunks(ctx, newChunks); err != nil {
			logger.Warnf(ctx, "Failed to update FAQ chunks status: %v", err)
			// Don't fail the whole operation for status update failure
		}

		processedCount += len(batchIDs)
		if totalOperations > 0 {
			progress.Progress = processedCount * 100 / totalOperations
		}
		progress.Processed = processedCount
		progress.Message = fmt.Sprintf("Added %d/%d FAQ entries", processedCount-len(chunksToDelete), len(chunksToAdd))
		progress.UpdatedAt = time.Now().Unix()
		_ = s.saveKBCloneProgress(ctx, progress)
	}

	for i := 0; i < len(chunksToUpdate); i += batch {
		end := i + batch
		if end > len(chunksToUpdate) {
			end = len(chunksToUpdate)
		}
		if err := s.syncFAQChunkStatusBatch(
			ctx, dstKB, chunksToUpdate[i:end], syncPlan.SrcByID, syncPlan.DstByID, resolveFAQTag); err != nil {
			logger.Errorf(ctx, "Failed to sync FAQ status fields: %v", err)
			handleError(progress, err, "Failed to sync FAQ status fields")
			return err
		}
		processedCount += end - i
		if totalOperations > 0 {
			progress.Progress = processedCount * 100 / totalOperations
		}
		progress.Processed = processedCount
		progress.Message = fmt.Sprintf("Updated %d/%d FAQ status fields", processedCount-len(chunksToDelete)-len(chunksToAdd), len(chunksToUpdate))
		progress.UpdatedAt = time.Now().Unix()
		_ = s.saveKBCloneProgress(ctx, progress)
	}

	// Mark as completed
	progress.Status = types.KBCloneStatusCompleted
	progress.Progress = 100
	progress.Processed = totalOperations
	progress.Message = "FAQ knowledge base clone completed successfully"
	progress.UpdatedAt = time.Now().Unix()
	if err := s.saveKBCloneProgress(ctx, progress); err != nil {
		logger.Errorf(ctx, "Failed to update KB clone progress to completed: %v", err)
	}

	return nil
}

// getOrCreateFAQKnowledge gets or creates the FAQ knowledge entry for a knowledge base
// If srcKnowledge is provided, it will copy relevant fields from source when creating new knowledge
func (s *knowledgeService) getOrCreateFAQKnowledge(ctx context.Context, kb *types.KnowledgeBase, srcKnowledge *types.Knowledge) (*types.Knowledge, error) {
	// FAQ knowledge base should have exactly one Knowledge entry
	knowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, kb.TenantID, kb.ID)
	if err != nil {
		return nil, err
	}

	if len(knowledgeList) > 0 {
		return knowledgeList[0], nil
	}

	// Create a new FAQ knowledge entry, copying from source if available
	knowledge := &types.Knowledge{
		ID:               uuid.New().String(),
		TenantID:         kb.TenantID,
		KnowledgeBaseID:  kb.ID,
		Type:             types.KnowledgeTypeFAQ,
		Channel:          types.ChannelWeb,
		Title:            "FAQ",
		ParseStatus:      "completed",
		EnableStatus:     "enabled",
		EmbeddingModelID: kb.EmbeddingModelID,
	}

	// Copy additional fields from source knowledge if available
	if srcKnowledge != nil {
		knowledge.Title = srcKnowledge.Title
		knowledge.Description = srcKnowledge.Description
		knowledge.Source = srcKnowledge.Source
		knowledge.Channel = srcKnowledge.Channel
		knowledge.Metadata = srcKnowledge.Metadata
	}

	if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
		return nil, err
	}
	return knowledge, nil
}

// saveKBCloneProgress saves the KB clone progress to Redis
func (s *knowledgeService) saveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error {
	key := getKBCloneProgressKey(progress.TaskID)
	data, err := json.Marshal(progress)
	if err != nil {
		return fmt.Errorf("failed to marshal progress: %w", err)
	}
	return s.redisClient.Set(ctx, key, data, kbCloneProgressTTL).Err()
}

// SaveKBCloneProgress saves the KB clone progress to Redis (public method for handler use)
func (s *knowledgeService) SaveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error {
	return s.saveKBCloneProgress(ctx, progress)
}

// GetKBCloneProgress retrieves the progress of a knowledge base clone task
func (s *knowledgeService) GetKBCloneProgress(ctx context.Context, taskID string) (*types.KBCloneProgress, error) {
	key := getKBCloneProgressKey(taskID)
	data, err := s.redisClient.Get(ctx, key).Bytes()
	if err != nil {
		if errors.Is(err, redis.Nil) {
			return nil, werrors.NewNotFoundError("KB clone task not found")
		}
		return nil, fmt.Errorf("failed to get progress from Redis: %w", err)
	}

	var progress types.KBCloneProgress
	if err := json.Unmarshal(data, &progress); err != nil {
		return nil, fmt.Errorf("failed to unmarshal progress: %w", err)
	}
	return &progress, nil
}

const (
	knowledgeMoveProgressKeyPrefix = "knowledge_move_progress:"
	knowledgeMoveProgressTTL       = 24 * time.Hour
)

func getKnowledgeMoveProgressKey(taskID string) string {
	return knowledgeMoveProgressKeyPrefix + taskID
}

func (s *knowledgeService) saveKnowledgeMoveProgress(ctx context.Context, progress *types.KnowledgeMoveProgress) error {
	key := getKnowledgeMoveProgressKey(progress.TaskID)
	data, err := json.Marshal(progress)
	if err != nil {
		return fmt.Errorf("failed to marshal move progress: %w", err)
	}
	return s.redisClient.Set(ctx, key, data, knowledgeMoveProgressTTL).Err()
}

// SaveKnowledgeMoveProgress saves the knowledge move progress to Redis (public method for handler use)
func (s *knowledgeService) SaveKnowledgeMoveProgress(ctx context.Context, progress *types.KnowledgeMoveProgress) error {
	return s.saveKnowledgeMoveProgress(ctx, progress)
}

// GetKnowledgeMoveProgress retrieves the progress of a knowledge move task
func (s *knowledgeService) GetKnowledgeMoveProgress(ctx context.Context, taskID string) (*types.KnowledgeMoveProgress, error) {
	key := getKnowledgeMoveProgressKey(taskID)
	data, err := s.redisClient.Get(ctx, key).Bytes()
	if err != nil {
		if errors.Is(err, redis.Nil) {
			return nil, werrors.NewNotFoundError("Knowledge move task not found")
		}
		return nil, fmt.Errorf("failed to get move progress from Redis: %w", err)
	}

	var progress types.KnowledgeMoveProgress
	if err := json.Unmarshal(data, &progress); err != nil {
		return nil, fmt.Errorf("failed to unmarshal move progress: %w", err)
	}
	return &progress, nil
}

// ProcessKnowledgeMove handles Asynq knowledge move tasks
func (s *knowledgeService) ProcessKnowledgeMove(ctx context.Context, t *asynq.Task) error {
	var payload types.KnowledgeMovePayload
	if err := json.Unmarshal(t.Payload(), &payload); err != nil {
		return fmt.Errorf("failed to unmarshal knowledge move payload: %w", err)
	}
	ctx = payload.Initiator.Apply(ctx)
	ctx = withKBActivityTask(ctx, payload.TaskID, kbActivityTrigger(ctx))

	// Add tenant ID to context
	ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)

	// Get tenant info and add to context
	tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
	if err != nil {
		logger.Errorf(ctx, "ProcessKnowledgeMove: failed to get tenant info: %v", err)
		return fmt.Errorf("failed to get tenant info: %w", err)
	}
	ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)

	// Check if this is the last retry
	retryCount, _ := asynq.GetRetryCount(ctx)
	maxRetry, _ := asynq.GetMaxRetry(ctx)
	isLastRetry := retryCount >= maxRetry

	logger.Infof(ctx, "ProcessKnowledgeMove: task=%s, source=%s, target=%s, mode=%s, count=%d, retry=%d/%d",
		payload.TaskID, payload.SourceKBID, payload.TargetKBID, payload.Mode, len(payload.KnowledgeIDs), retryCount, maxRetry)

	// Helper function to handle errors - only mark as failed on last retry
	handleError := func(progress *types.KnowledgeMoveProgress, err error, message string) {
		if isLastRetry {
			progress.Status = types.KBCloneStatusFailed
			progress.Error = err.Error()
			progress.Message = message
			progress.UpdatedAt = time.Now().Unix()
			_ = s.saveKnowledgeMoveProgress(ctx, progress)
			for _, kbID := range []string{payload.SourceKBID, payload.TargetKBID} {
				recordKBActivity(ctx, s.audit, payload.TenantID, kbID, types.AuditActionKnowledgeMoveFailed,
					"knowledge_move", payload.TaskID, types.AuditOutcomeFailed,
					map[string]any{"source_kb_id": payload.SourceKBID, "target_kb_id": payload.TargetKBID,
						"task_id": payload.TaskID, "count": len(payload.KnowledgeIDs), "mode": payload.Mode})
			}
		}
	}
	if retryCount == 0 {
		for _, kbID := range []string{payload.SourceKBID, payload.TargetKBID} {
			recordKBActivity(ctx, s.audit, payload.TenantID, kbID, types.AuditActionKnowledgeMoveStarted,
				"knowledge_move", payload.TaskID, types.AuditOutcomeAccepted,
				map[string]any{"source_kb_id": payload.SourceKBID, "target_kb_id": payload.TargetKBID,
					"task_id": payload.TaskID, "count": len(payload.KnowledgeIDs), "mode": payload.Mode})
		}
	}

	// Update progress to processing
	progress := &types.KnowledgeMoveProgress{
		TaskID:     payload.TaskID,
		SourceKBID: payload.SourceKBID,
		TargetKBID: payload.TargetKBID,
		Status:     types.KBCloneStatusProcessing,
		Total:      len(payload.KnowledgeIDs),
		Progress:   0,
		Message:    "Starting knowledge move...",
		UpdatedAt:  time.Now().Unix(),
	}
	_ = s.saveKnowledgeMoveProgress(ctx, progress)

	// Get source and target knowledge bases
	sourceKB, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.SourceKBID)
	if err != nil {
		handleError(progress, err, "Failed to get source knowledge base")
		return err
	}
	targetKB, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.TargetKBID)
	if err != nil {
		handleError(progress, err, "Failed to get target knowledge base")
		return err
	}

	// Validate compatibility
	if sourceKB.Type != targetKB.Type {
		err := fmt.Errorf("type mismatch: source=%s, target=%s", sourceKB.Type, targetKB.Type)
		handleError(progress, err, "Source and target knowledge bases must be the same type")
		return err
	}
	if sourceKB.EmbeddingModelID != targetKB.EmbeddingModelID {
		err := fmt.Errorf("embedding model mismatch: source=%s, target=%s", sourceKB.EmbeddingModelID, targetKB.EmbeddingModelID)
		handleError(progress, err, "Source and target must use the same embedding model")
		return err
	}

	// Process each knowledge item
	for i, knowledgeID := range payload.KnowledgeIDs {
		err := s.moveOneKnowledge(ctx, knowledgeID, sourceKB, targetKB, payload.Mode)
		if err != nil {
			logger.Errorf(ctx, "ProcessKnowledgeMove: failed to move knowledge %s: %v", knowledgeID, err)
			progress.Failed++
		}
		progress.Processed = i + 1
		if progress.Total > 0 {
			progress.Progress = progress.Processed * 100 / progress.Total
		}
		progress.Message = fmt.Sprintf("Moved %d/%d knowledge items", progress.Processed, progress.Total)
		progress.UpdatedAt = time.Now().Unix()
		_ = s.saveKnowledgeMoveProgress(ctx, progress)
	}

	// Mark as completed
	if progress.Failed > 0 && progress.Failed == progress.Total {
		progress.Status = types.KBCloneStatusFailed
		progress.Message = fmt.Sprintf("Knowledge move failed: all %d items failed", progress.Total)
	} else {
		progress.Status = types.KBCloneStatusCompleted
		progress.Message = fmt.Sprintf("Knowledge move completed: %d/%d succeeded", progress.Processed-progress.Failed, progress.Total)
	}
	progress.Progress = 100
	progress.UpdatedAt = time.Now().Unix()
	_ = s.saveKnowledgeMoveProgress(ctx, progress)

	logger.Infof(ctx, "ProcessKnowledgeMove: task=%s completed, processed=%d, failed=%d", payload.TaskID, progress.Processed, progress.Failed)
	outcome := types.AuditOutcomeSuccess
	action := types.AuditActionKnowledgeMoveCompleted
	if progress.Failed == progress.Total && progress.Total > 0 {
		outcome = types.AuditOutcomeFailed
		action = types.AuditActionKnowledgeMoveFailed
	} else if progress.Failed > 0 {
		outcome = types.AuditOutcomePartial
	}
	for _, kbID := range []string{payload.SourceKBID, payload.TargetKBID} {
		recordKBActivity(ctx, s.audit, payload.TenantID, kbID, action,
			"knowledge_move", payload.TaskID, outcome,
			map[string]any{"source_kb_id": payload.SourceKBID, "target_kb_id": payload.TargetKBID,
				"task_id": payload.TaskID, "count": progress.Total, "failed": progress.Failed, "mode": payload.Mode})
	}
	return nil
}

// moveOneKnowledge moves a single knowledge item from source KB to target KB.
func (s *knowledgeService) moveOneKnowledge(
	ctx context.Context,
	knowledgeID string,
	sourceKB, targetKB *types.KnowledgeBase,
	mode string,
) error {
	if err := checkKnowledgeCopyLineage(ctx, sourceKB.ID); err != nil {
		return err
	}
	tenantID := ctx.Value(types.TenantIDContextKey).(uint64)

	// Get the knowledge item
	knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
	if err != nil {
		return fmt.Errorf("failed to get knowledge %s: %w", knowledgeID, err)
	}

	// Only move completed items
	if knowledge.ParseStatus != types.ParseStatusCompleted {
		return fmt.Errorf("knowledge %s is not in completed status (current: %s)", knowledgeID, knowledge.ParseStatus)
	}

	// Reject a cross-store reuse_vectors move BEFORE mutating status, so a
	// rejected move leaves the knowledge untouched (Completed) rather than
	// stranded in Processing. reuse_vectors copies indices through the source
	// store only; a cross-store copy would corrupt vector data. The handler
	// rejects this synchronously — this is defense-in-depth for directly
	// enqueued tasks. Cross-store moves must use reparse mode.
	if mode == "reuse_vectors" && !sourceKB.SharesStoreWith(targetKB) {
		return fmt.Errorf(
			"reuse_vectors move across different vector stores is not supported "+
				"(source KB %s, target KB %s); use reparse mode", sourceKB.ID, targetKB.ID)
	}

	// Mark as processing during move
	knowledge.ParseStatus = types.ParseStatusProcessing
	if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
		return fmt.Errorf("failed to mark knowledge as processing: %w", err)
	}

	// From the source KB's point of view the document is leaving for good, so it
	// needs the same wiki reconciliation a delete performs: wiki_pages carry
	// source_refs back to this knowledge and are what the folder tree and the
	// wiki graph are built from, and nothing below touches them. This must run
	// while KnowledgeBaseID still points at the source and before any chunk is
	// removed, since the cleanup matches pages by chunk_refs.
	if sourceKB.IsWikiEnabled() {
		s.cleanupWikiOnKnowledgeDelete(ctx, knowledge)
	}

	switch mode {
	case "reuse_vectors":
		if err := s.moveKnowledgeReuseVectors(ctx, knowledge, sourceKB, targetKB); err != nil {
			return err
		}
		// reparse re-ingests through KnowledgePostProcess once the new chunks
		// land; reuse_vectors keeps the existing chunks and never re-enters that
		// pipeline, so the target KB has to be told about the document here.
		if targetKB.IsWikiEnabled() {
			EnqueueWikiIngest(ctx, s.task, s.taskPendingRepo, tenantID, targetKB.ID, knowledge.ID)
		}
		return nil
	case "reparse":
		return s.moveKnowledgeReparse(ctx, knowledge, sourceKB, targetKB)
	default:
		return fmt.Errorf("unknown move mode: %s", mode)
	}
}

// moveKnowledgeReuseVectors moves knowledge by copying vector indices and updating DB references.
func (s *knowledgeService) moveKnowledgeReuseVectors(
	ctx context.Context,
	knowledge *types.Knowledge,
	sourceKB, targetKB *types.KnowledgeBase,
) error {
	tenantID := ctx.Value(types.TenantIDContextKey).(uint64)

	// reuse_vectors copies index entries directly between KBs, which only works
	// inside the same VectorStore backend (CopyIndices is routed through the
	// source store). A cross-store reuse_vectors move would write target-KB rows
	// into the source store and then delete the source indices, corrupting data.
	// The MoveKnowledge handler rejects this up front; this is defense-in-depth
	// for any path that enqueues a move task directly. Cross-store moves must use
	// reparse mode (moveKnowledgeReparse), which re-indexes into the target store.
	if !sourceKB.SharesStoreWith(targetKB) {
		return fmt.Errorf(
			"reuse_vectors move across different vector stores is not supported "+
				"(source KB %s, target KB %s); use reparse mode", sourceKB.ID, targetKB.ID)
	}

	// 1. Get old chunk IDs for vector index copy mapping
	oldChunks, err := s.chunkRepo.ListChunksByKnowledgeID(ctx, tenantID, knowledge.ID)
	if err != nil {
		return fmt.Errorf("failed to list chunks: %w", err)
	}

	// Build identity mapping (same chunk IDs, just moving between KBs)
	chunkIDMapping := make(map[string]string, len(oldChunks))
	for _, c := range oldChunks {
		chunkIDMapping[c.ID] = c.ID
	}

	// 2. Copy vector indices from source KB to target KB
	if len(chunkIDMapping) > 0 && knowledge.EmbeddingModelID != "" {
		// Same VectorStore backend is guaranteed by the SharesStoreWith guard at
		// the top of this function, so routing CopyIndices through the source
		// KB's binding also resolves the target's store.
		var sourceStoreID *string
		if sourceKB != nil {
			sourceStoreID = sourceKB.VectorStoreID
		}
		retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
			ctx, s.retrieveEngine, s.ownership, tenantID, sourceStoreID)
		if err != nil {
			return fmt.Errorf("failed to init retrieve engine: %w", err)
		}
		embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, knowledge.EmbeddingModelID)
		if err != nil {
			return fmt.Errorf("failed to get embedding model: %w", err)
		}

		// Copy indices from source KB to target KB
		knowledgeIDMapping := map[string]string{knowledge.ID: knowledge.ID}
		if err := retrieveEngine.CopyIndices(ctx, sourceKB.ID, targetKB.ID,
			knowledgeIDMapping, chunkIDMapping,
			embeddingModel.GetDimensions(), sourceKB.Type,
		); err != nil {
			return fmt.Errorf("failed to copy indices: %w", err)
		}

		// Delete indices from source KB
		if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID},
			embeddingModel.GetDimensions(), sourceKB.Type,
		); err != nil {
			logger.Warnf(ctx, "moveKnowledgeReuseVectors: failed to delete old indices for knowledge %s: %v", knowledge.ID, err)
			// Non-fatal: indices will be orphaned but won't affect correctness
		}
	}

	// 3. Update chunks' knowledge_base_id in DB
	if err := s.chunkRepo.MoveChunksByKnowledgeID(ctx, tenantID, knowledge.ID, targetKB.ID); err != nil {
		return fmt.Errorf("failed to move chunks: %w", err)
	}

	// 4. Update knowledge record (tags are KB-scoped; clear relations before moving)
	if err := s.repo.DeleteKnowledgeTagRelations(ctx, knowledge.ID); err != nil {
		return fmt.Errorf("failed to clear knowledge tag relations: %w", err)
	}
	knowledge.KnowledgeBaseID = targetKB.ID
	knowledge.ParseStatus = types.ParseStatusCompleted
	knowledge.UpdatedAt = time.Now()
	if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
		return fmt.Errorf("failed to update knowledge: %w", err)
	}

	return nil
}

// moveKnowledgeReparse moves knowledge to target KB and re-parses it with target KB's configuration.
func (s *knowledgeService) moveKnowledgeReparse(
	ctx context.Context,
	knowledge *types.Knowledge,
	_, targetKB *types.KnowledgeBase,
) error {
	tenantID := ctx.Value(types.TenantIDContextKey).(uint64)

	// 1. Clean up existing chunks and vector indices
	if err := s.cleanupKnowledgeResources(ctx, knowledge); err != nil {
		logger.Warnf(ctx, "moveKnowledgeReparse: cleanup partial error for knowledge %s: %v", knowledge.ID, err)
		// Continue - partial cleanup is acceptable
	}

	// 2. Update knowledge to belong to target KB (tags are KB-scoped; clear relations)
	if err := s.repo.DeleteKnowledgeTagRelations(ctx, knowledge.ID); err != nil {
		return fmt.Errorf("failed to clear knowledge tag relations: %w", err)
	}
	knowledge.KnowledgeBaseID = targetKB.ID
	knowledge.EmbeddingModelID = targetKB.EmbeddingModelID
	knowledge.ParseStatus = types.ParseStatusPending
	knowledge.EnableStatus = "disabled"
	knowledge.Description = ""
	knowledge.ProcessedAt = nil
	knowledge.UpdatedAt = time.Now()
	if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
		return fmt.Errorf("failed to update knowledge: %w", err)
	}

	// 3. Enqueue document processing task with target KB's configuration
	if knowledge.IsManual() {
		meta, err := knowledge.ManualMetadata()
		if err != nil || meta == nil {
			return fmt.Errorf("failed to get manual metadata for reparse: %w", err)
		}
		s.triggerManualProcessing(ctx, targetKB, knowledge, meta.Content, false)
		return nil
	}

	if knowledge.FilePath != "" {
		enableMultimodel := targetKB.IsMultimodalEnabled()
		enableQuestionGeneration := false
		questionCount := 3
		if targetKB.QuestionGenerationConfig != nil && targetKB.QuestionGenerationConfig.Enabled {
			enableQuestionGeneration = true
			if targetKB.QuestionGenerationConfig.QuestionCount > 0 {
				questionCount = targetKB.QuestionGenerationConfig.QuestionCount
			}
		}

		lang := types.LanguageFromContextOrDefault(ctx)
		taskPayload := types.DocumentProcessPayload{
			TenantID:                 tenantID,
			KnowledgeID:              knowledge.ID,
			KnowledgeBaseID:          targetKB.ID,
			FilePath:                 knowledge.FilePath,
			FileName:                 knowledge.FileName,
			FileType:                 getFileType(knowledge.FileName),
			EnableMultimodel:         enableMultimodel,
			EnableQuestionGeneration: enableQuestionGeneration,
			QuestionCount:            questionCount,
			Language:                 lang,
		}

		langfuse.InjectTracing(ctx, &taskPayload)
		payloadBytes, err := json.Marshal(taskPayload)
		if err != nil {
			return fmt.Errorf("failed to marshal document process payload: %w", err)
		}

		task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes,
			documentProcessTaskOptions(s.config, asynq.MaxRetry(3))...)
		info, err := s.task.Enqueue(task)
		if err != nil {
			return fmt.Errorf("failed to enqueue document process task: %w", err)
		}
		logger.Infof(ctx, "moveKnowledgeReparse: enqueued reparse task id=%s for knowledge=%s", info.ID, knowledge.ID)
	}

	return nil
}

// getOrCreateTagInTarget finds or creates a tag in the target knowledge base based on the source tag.
// It looks up the source tag by ID, then tries to find a tag with the same name in the target KB.
// If not found, it creates a new tag with the same properties.
// The mapping is cached in tagIDMapping for subsequent lookups.
func (s *knowledgeService) getOrCreateTagInTarget(
	ctx context.Context,
	srcTenantID, dstTenantID uint64,
	dstKnowledgeBaseID string,
	srcTagID string,
	tagIDMapping map[string]string,
) string {
	// Get source tag
	srcTag, err := s.tagRepo.GetByID(ctx, srcTenantID, srcTagID)
	if err != nil || srcTag == nil {
		logger.Warnf(ctx, "Failed to get source tag %s: %v", srcTagID, err)
		tagIDMapping[srcTagID] = "" // Cache empty result to avoid repeated lookups
		return ""
	}

	// Try to find existing tag with same name in target KB
	dstTag, err := s.tagRepo.GetByName(ctx, dstTenantID, dstKnowledgeBaseID, srcTag.Name)
	if err == nil && dstTag != nil {
		tagIDMapping[srcTagID] = dstTag.ID
		return dstTag.ID
	}

	// Create new tag in target KB
	// "未分类" tag should have the lowest sort order to appear first
	sortOrder := srcTag.SortOrder
	if srcTag.Name == types.UntaggedTagName {
		sortOrder = -1
	}
	newTag := &types.KnowledgeTag{
		ID:              uuid.New().String(),
		TenantID:        dstTenantID,
		KnowledgeBaseID: dstKnowledgeBaseID,
		Name:            srcTag.Name,
		Color:           srcTag.Color,
		SortOrder:       sortOrder,
		CreatedAt:       time.Now(),
		UpdatedAt:       time.Now(),
	}
	if err := s.tagRepo.Create(ctx, newTag); err != nil {
		logger.Warnf(ctx, "Failed to create tag %s in target KB: %v", srcTag.Name, err)
		tagIDMapping[srcTagID] = "" // Cache empty result
		return ""
	}

	tagIDMapping[srcTagID] = newTag.ID
	logger.Infof(ctx, "Created tag %s (ID: %s) in target KB %s", newTag.Name, newTag.ID, dstKnowledgeBaseID)
	return newTag.ID
}