xinmi/internal/application/service/knowledge_version_copy.go

package service

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/Tencent/WeKnora/internal/knowledgeversion"
	"github.com/Tencent/WeKnora/internal/searchutil"
	"github.com/Tencent/WeKnora/internal/types"
	"github.com/Tencent/WeKnora/internal/types/interfaces"
	"github.com/google/uuid"
	"gorm.io/gorm"
)

// CloneKnowledgeVersion copies into a fresh, inaccessible staging KB. Unlike
// incremental synchronization it does not consider equal file hashes to mean
// equal edited chunks. The durable caller owns retries and orphan cleanup.
func (s *knowledgeService) CloneKnowledgeVersion(ctx context.Context, source, target string, progress func(int, int) error) error {
	if !knowledgeversion.IsWorker(ctx) || source == target {
		return fmt.Errorf("knowledge version copy requires an isolated worker target")
	}
	tenantID := types.MustTenantIDFromContext(ctx)
	tenant, err := s.tenantRepo.GetTenantByID(ctx, tenantID)
	if err != nil {
		return err
	}
	ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
	srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, source, target)
	if err != nil {
		return err
	}
	if dstKB.VersionState != "copying" {
		return fmt.Errorf("target is not a version staging base")
	}
	if srcKB.TenantID != tenantID || dstKB.TenantID != tenantID {
		return fmt.Errorf("knowledge version copy must remain in the same workspace")
	}
	if !srcKB.SharesStoreWith(dstKB) {
		return fmt.Errorf("knowledge version copy requires the same vector store")
	}
	if srcKB.Type == types.KnowledgeBaseTypeWiki || srcKB.ExtractConfig != nil && srcKB.ExtractConfig.Enabled {
		return fmt.Errorf("Wiki 和知识图谱暂不支持独立版本复制")
	}
	existing, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, tenantID, target)
	if err != nil {
		return err
	}
	if len(existing) != 0 {
		return fmt.Errorf("version copy target must be empty")
	}
	docs, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, tenantID, source)
	if err != nil {
		return err
	}
	sourceChunkCount, err := s.chunkRepo.CountChunksByKnowledgeBaseID(ctx, tenantID, source)
	if err != nil {
		return err
	}
	for _, src := range docs {
		if src.ParseStatus != types.ParseStatusCompleted || src.PendingSubtasksCount > 0 {
			return fmt.Errorf("知识 %s 尚未完成索引，请完成处理后再复制", src.Title)
		}
	}
	tagMapping := map[string]string{}
	for page := 1; ; page++ {
		tags, total, err := s.tagRepo.ListByKB(ctx, tenantID, source, &types.Pagination{Page: page, PageSize: 100}, "")
		if err != nil {
			return err
		}
		for _, tag := range tags {
			copy := *tag
			copy.ID = uuid.NewString()
			copy.SeqID = 0
			copy.KnowledgeBaseID = target
			copy.CreatedAt = time.Now()
			copy.UpdatedAt = copy.CreatedAt
			if err := s.tagRepo.Create(ctx, &copy); err != nil {
				return err
			}
			tagMapping[tag.ID] = copy.ID
		}
		if int64(page*100) >= total || len(tags) == 0 {
			break
		}
	}
	if progress == nil {
		progress = func(int, int) error { return nil }
	}
	if err := progress(0, len(docs)); err != nil {
		return err
	}
	documentMapping, chunkMapping := map[string]string{}, map[string]string{}
	for i, src := range docs {
		if err := ctx.Err(); err != nil {
			return err
		}
		dst := *src
		dst.ID = uuid.NewString()
		dst.KnowledgeBaseID = target
		dst.CreatedAt, dst.UpdatedAt = time.Now(), time.Now()
		dst.DeletedAt = gorm.DeletedAt{}
		dst.ParseStatus = types.ParseStatusProcessing
		dst.EnableStatus = "disabled"
		dst.Tags = nil
		var rawFiles interfaces.FileService
		if src.FilePath != "" {
			// The native file service streams local copies and gives the new
			// document its own object; large raw files never enter a byte slice.
			rawFiles = s.resolveFileServiceForPath(ctx, srcKB, src.FilePath)
			dst.FilePath, err = rawFiles.CopyFile(ctx, src.FilePath, tenantID, dst.ID)
			if err != nil {
				return fmt.Errorf("复制原文件 %s: %w", src.Title, err)
			}
			if dst.FilePath == "" || dst.FilePath == src.FilePath {
				return fmt.Errorf("storage returned a shared source object")
			}
		}
		// Do not persist source-owned metadata: a failed attempt is later
		// deleted by the durable worker, which must only encounter owned files.
		dst.Metadata, dst.CustomMetadata, dst.LastFAQImportResult = nil, nil, nil
		creator, ok := s.repo.(interface {
			CreateVersionKnowledge(context.Context, *types.Knowledge) error
		})
		if !ok {
			cleanupCopiedObjects(ctx, rawFiles, []string{dst.FilePath})
			return fmt.Errorf("knowledge repository does not support atomic version storage accounting")
		}
		if err := creator.CreateVersionKnowledge(ctx, &dst); err != nil {
			cleanupCopiedObjects(ctx, rawFiles, []string{dst.FilePath})
			return err
		}
		if err := s.cloneChunk(ctx, src, &dst, tagMapping, chunkMapping); err != nil {
			return err
		}
		documentMapping[src.ID] = dst.ID
		tagIDs := []string{}
		sourceTags, err := s.repo.GetKnowledgeTags(ctx, []string{src.ID})
		if err != nil {
			return err
		}
		for _, tag := range sourceTags[src.ID] {
			id := tagMapping[tag.ID]
			if id == "" {
				return fmt.Errorf("复制知识分类失败: %s", tag.Name)
			}
			tagIDs = append(tagIDs, id)
		}
		if err := s.repo.SetKnowledgeTags(ctx, dst.ID, tagIDs); err != nil {
			return err
		}
		dst.ParseStatus, dst.EnableStatus = types.ParseStatusCompleted, src.EnableStatus
		if err := s.preserveKnowledgeVersionInventory(ctx, &dst); err != nil {
			return err
		}
		if err := s.repo.UpdateKnowledge(ctx, &dst); err != nil {
			return err
		}
		if err := progress(i+1, len(docs)); err != nil {
			return err
		}
	}
	targetChunkCount, err := s.chunkRepo.CountChunksByKnowledgeBaseID(ctx, tenantID, target)
	if err != nil {
		return err
	}
	if targetChunkCount != sourceChunkCount {
		return fmt.Errorf("知识分块未完整复制: 原始 %d 个，副本 %d 个", sourceChunkCount, targetChunkCount)
	}
	graphCopier, ok := s.repo.(interface {
		CopyVersionGraph(context.Context, string, string, map[string]string, map[string]string) error
	})
	if !ok {
		return fmt.Errorf("knowledge repository cannot preserve versioned graph evidence")
	}
	return graphCopier.CopyVersionGraph(ctx, source, target, documentMapping, chunkMapping)
}

// versionReferenceCopier handles document metadata as structured JSON and
// discovers Markdown/HTML images using the same parser as knowledge previews.
type versionReferenceCopier struct {
	ctx         context.Context
	source      func(string) interfaces.FileService
	destination interfaces.FileService
	tenantID    uint64
	knowledgeID string
	urls        map[string]string
	copied      []string
}

func managedVersionReference(value string) bool {
	if _, ok := types.ParseResourcePath(value); ok {
		return true
	}
	if _, _, ok := types.ParseStorageBackendPath(value); ok {
		return true
	}
	return types.ParseProviderScheme(value) != ""
}

func (c *versionReferenceCopier) copyURL(value string) error {
	if !managedVersionReference(value) || c.urls[value] != "" {
		return nil
	}
	path, err := copyOwnedObject(c.ctx, c.source(value), c.destination, value, c.tenantID, c.knowledgeID)
	if err != nil {
		return err
	}
	c.urls[value] = path
	c.copied = append(c.copied, path)
	return nil
}

func (c *versionReferenceCopier) text(value string) (string, error) {
	if managedVersionReference(value) && !strings.ContainsAny(value, "\n\r\t ") {
		if err := c.copyURL(value); err != nil {
			return "", err
		}
	}
	for url := range searchutil.ImageURLsInContent(value) {
		if err := c.copyURL(url); err != nil {
			return "", err
		}
	}
	return rewriteContentImageURLs(value, c.urls), nil
}

func (c *versionReferenceCopier) jsonValue(value any) (any, error) {
	switch v := value.(type) {
	case string:
		return c.text(v)
	case []any:
		for i, item := range v {
			cloned, err := c.jsonValue(item)
			if err != nil {
				return nil, err
			}
			v[i] = cloned
		}
	case map[string]any:
		for key, item := range v {
			if key == knowledgeVersionCleanupKey {
				delete(v, key)
				continue
			}
			cloned, err := c.jsonValue(item)
			if err != nil {
				return nil, err
			}
			v[key] = cloned
		}
	}
	return value, nil
}

func (c *versionReferenceCopier) json(data []byte) (types.JSON, error) {
	if len(data) == 0 {
		return nil, nil
	}
	decoder := json.NewDecoder(strings.NewReader(string(data)))
	decoder.UseNumber()
	var value any
	if err := decoder.Decode(&value); err != nil {
		return nil, fmt.Errorf("invalid knowledge metadata: %w", err)
	}
	cloned, err := c.jsonValue(value)
	if err != nil {
		return nil, err
	}
	return json.Marshal(cloned)
}