WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/application/service/knowledge_version_copy.go

@@ -0,0 +1,273 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/Tencent/WeKnora/internal/knowledgeversion"
+	"github.com/Tencent/WeKnora/internal/searchutil"
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/Tencent/WeKnora/internal/types/interfaces"
+	"github.com/google/uuid"
+	"gorm.io/gorm"
+)
+
+// CloneKnowledgeVersion copies into a fresh, inaccessible staging KB. Unlike
+// incremental synchronization it does not consider equal file hashes to mean
+// equal edited chunks. The durable caller owns retries and orphan cleanup.
+func (s *knowledgeService) CloneKnowledgeVersion(ctx context.Context, source, target string, progress func(int, int) error) error {
+	if !knowledgeversion.IsWorker(ctx) || source == target {
+		return fmt.Errorf("knowledge version copy requires an isolated worker target")
+	}
+	tenantID := types.MustTenantIDFromContext(ctx)
+	tenant, err := s.tenantRepo.GetTenantByID(ctx, tenantID)
+	if err != nil {
+		return err
+	}
+	ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
+	srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, source, target)
+	if err != nil {
+		return err
+	}
+	if dstKB.VersionState != "copying" {
+		return fmt.Errorf("target is not a version staging base")
+	}
+	if srcKB.TenantID != tenantID || dstKB.TenantID != tenantID {
+		return fmt.Errorf("knowledge version copy must remain in the same workspace")
+	}
+	if !srcKB.SharesStoreWith(dstKB) {
+		return fmt.Errorf("knowledge version copy requires the same vector store")
+	}
+	if srcKB.Type == types.KnowledgeBaseTypeWiki || srcKB.ExtractConfig != nil && srcKB.ExtractConfig.Enabled {
+		return fmt.Errorf("Wiki 和知识图谱暂不支持独立版本复制")
+	}
+	existing, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, tenantID, target)
+	if err != nil {
+		return err
+	}
+	if len(existing) != 0 {
+		return fmt.Errorf("version copy target must be empty")
+	}
+	docs, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, tenantID, source)
+	if err != nil {
+		return err
+	}
+	sourceChunkCount, err := s.chunkRepo.CountChunksByKnowledgeBaseID(ctx, tenantID, source)
+	if err != nil {
+		return err
+	}
+	for _, src := range docs {
+		if src.ParseStatus != types.ParseStatusCompleted || src.PendingSubtasksCount > 0 {
+			return fmt.Errorf("知识 %s 尚未完成索引，请完成处理后再复制", src.Title)
+		}
+	}
+	tagMapping := map[string]string{}
+	for page := 1; ; page++ {
+		tags, total, err := s.tagRepo.ListByKB(ctx, tenantID, source, &types.Pagination{Page: page, PageSize: 100}, "")
+		if err != nil {
+			return err
+		}
+		for _, tag := range tags {
+			copy := *tag
+			copy.ID = uuid.NewString()
+			copy.SeqID = 0
+			copy.KnowledgeBaseID = target
+			copy.CreatedAt = time.Now()
+			copy.UpdatedAt = copy.CreatedAt
+			if err := s.tagRepo.Create(ctx, &copy); err != nil {
+				return err
+			}
+			tagMapping[tag.ID] = copy.ID
+		}
+		if int64(page*100) >= total || len(tags) == 0 {
+			break
+		}
+	}
+	if progress == nil {
+		progress = func(int, int) error { return nil }
+	}
+	if err := progress(0, len(docs)); err != nil {
+		return err
+	}
+	documentMapping, chunkMapping := map[string]string{}, map[string]string{}
+	for i, src := range docs {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		dst := *src
+		dst.ID = uuid.NewString()
+		dst.KnowledgeBaseID = target
+		dst.CreatedAt, dst.UpdatedAt = time.Now(), time.Now()
+		dst.DeletedAt = gorm.DeletedAt{}
+		dst.ParseStatus = types.ParseStatusProcessing
+		dst.EnableStatus = "disabled"
+		dst.Tags = nil
+		var rawFiles interfaces.FileService
+		if src.FilePath != "" {
+			// The native file service streams local copies and gives the new
+			// document its own object; large raw files never enter a byte slice.
+			rawFiles = s.resolveFileServiceForPath(ctx, srcKB, src.FilePath)
+			dst.FilePath, err = rawFiles.CopyFile(ctx, src.FilePath, tenantID, dst.ID)
+			if err != nil {
+				return fmt.Errorf("复制原文件 %s: %w", src.Title, err)
+			}
+			if dst.FilePath == "" || dst.FilePath == src.FilePath {
+				return fmt.Errorf("storage returned a shared source object")
+			}
+		}
+		// Do not persist source-owned metadata: a failed attempt is later
+		// deleted by the durable worker, which must only encounter owned files.
+		dst.Metadata, dst.CustomMetadata, dst.LastFAQImportResult = nil, nil, nil
+		creator, ok := s.repo.(interface {
+			CreateVersionKnowledge(context.Context, *types.Knowledge) error
+		})
+		if !ok {
+			cleanupCopiedObjects(ctx, rawFiles, []string{dst.FilePath})
+			return fmt.Errorf("knowledge repository does not support atomic version storage accounting")
+		}
+		if err := creator.CreateVersionKnowledge(ctx, &dst); err != nil {
+			cleanupCopiedObjects(ctx, rawFiles, []string{dst.FilePath})
+			return err
+		}
+		if err := s.cloneChunk(ctx, src, &dst, tagMapping, chunkMapping); err != nil {
+			return err
+		}
+		documentMapping[src.ID] = dst.ID
+		tagIDs := []string{}
+		sourceTags, err := s.repo.GetKnowledgeTags(ctx, []string{src.ID})
+		if err != nil {
+			return err
+		}
+		for _, tag := range sourceTags[src.ID] {
+			id := tagMapping[tag.ID]
+			if id == "" {
+				return fmt.Errorf("复制知识分类失败: %s", tag.Name)
+			}
+			tagIDs = append(tagIDs, id)
+		}
+		if err := s.repo.SetKnowledgeTags(ctx, dst.ID, tagIDs); err != nil {
+			return err
+		}
+		dst.ParseStatus, dst.EnableStatus = types.ParseStatusCompleted, src.EnableStatus
+		if err := s.preserveKnowledgeVersionInventory(ctx, &dst); err != nil {
+			return err
+		}
+		if err := s.repo.UpdateKnowledge(ctx, &dst); err != nil {
+			return err
+		}
+		if err := progress(i+1, len(docs)); err != nil {
+			return err
+		}
+	}
+	targetChunkCount, err := s.chunkRepo.CountChunksByKnowledgeBaseID(ctx, tenantID, target)
+	if err != nil {
+		return err
+	}
+	if targetChunkCount != sourceChunkCount {
+		return fmt.Errorf("知识分块未完整复制: 原始 %d 个，副本 %d 个", sourceChunkCount, targetChunkCount)
+	}
+	graphCopier, ok := s.repo.(interface {
+		CopyVersionGraph(context.Context, string, string, map[string]string, map[string]string) error
+	})
+	if !ok {
+		return fmt.Errorf("knowledge repository cannot preserve versioned graph evidence")
+	}
+	return graphCopier.CopyVersionGraph(ctx, source, target, documentMapping, chunkMapping)
+}
+
+// versionReferenceCopier handles document metadata as structured JSON and
+// discovers Markdown/HTML images using the same parser as knowledge previews.
+type versionReferenceCopier struct {
+	ctx         context.Context
+	source      func(string) interfaces.FileService
+	destination interfaces.FileService
+	tenantID    uint64
+	knowledgeID string
+	urls        map[string]string
+	copied      []string
+}
+
+func managedVersionReference(value string) bool {
+	if _, ok := types.ParseResourcePath(value); ok {
+		return true
+	}
+	if _, _, ok := types.ParseStorageBackendPath(value); ok {
+		return true
+	}
+	return types.ParseProviderScheme(value) != ""
+}
+
+func (c *versionReferenceCopier) copyURL(value string) error {
+	if !managedVersionReference(value) || c.urls[value] != "" {
+		return nil
+	}
+	path, err := copyOwnedObject(c.ctx, c.source(value), c.destination, value, c.tenantID, c.knowledgeID)
+	if err != nil {
+		return err
+	}
+	c.urls[value] = path
+	c.copied = append(c.copied, path)
+	return nil
+}
+
+func (c *versionReferenceCopier) text(value string) (string, error) {
+	if managedVersionReference(value) && !strings.ContainsAny(value, "\n\r\t ") {
+		if err := c.copyURL(value); err != nil {
+			return "", err
+		}
+	}
+	for url := range searchutil.ImageURLsInContent(value) {
+		if err := c.copyURL(url); err != nil {
+			return "", err
+		}
+	}
+	return rewriteContentImageURLs(value, c.urls), nil
+}
+
+func (c *versionReferenceCopier) jsonValue(value any) (any, error) {
+	switch v := value.(type) {
+	case string:
+		return c.text(v)
+	case []any:
+		for i, item := range v {
+			cloned, err := c.jsonValue(item)
+			if err != nil {
+				return nil, err
+			}
+			v[i] = cloned
+		}
+	case map[string]any:
+		for key, item := range v {
+			if key == knowledgeVersionCleanupKey {
+				delete(v, key)
+				continue
+			}
+			cloned, err := c.jsonValue(item)
+			if err != nil {
+				return nil, err
+			}
+			v[key] = cloned
+		}
+	}
+	return value, nil
+}
+
+func (c *versionReferenceCopier) json(data []byte) (types.JSON, error) {
+	if len(data) == 0 {
+		return nil, nil
+	}
+	decoder := json.NewDecoder(strings.NewReader(string(data)))
+	decoder.UseNumber()
+	var value any
+	if err := decoder.Decode(&value); err != nil {
+		return nil, fmt.Errorf("invalid knowledge metadata: %w", err)
+	}
+	cloned, err := c.jsonValue(value)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(cloned)
+}