WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

2026-09-22 16:19:06 +08:00
commit f7fc992f57
3929 changed files with 1006607 additions and 0 deletions
@@ -0,0 +1,430 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/Tencent/WeKnora/internal/logger"
+	"github.com/Tencent/WeKnora/internal/types"
+	"github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// WikiLintIssueType defines the type of lint issue
+type WikiLintIssueType string
+
+const (
+	LintIssueOrphanPage      WikiLintIssueType = "orphan_page"
+	LintIssueBrokenLink      WikiLintIssueType = "broken_link"
+	LintIssueStaleRef        WikiLintIssueType = "stale_ref"
+	LintIssueMissingCrossRef WikiLintIssueType = "missing_cross_ref"
+	LintIssueEmptyContent    WikiLintIssueType = "empty_content"
+	LintIssueDuplicateSlug   WikiLintIssueType = "duplicate_slug"
+)
+
+// WikiLintIssueSeverity defines the severity of a lint issue
+type WikiLintIssueSeverity string
+
+const (
+	SeverityInfo    WikiLintIssueSeverity = "info"
+	SeverityWarning WikiLintIssueSeverity = "warning"
+	SeverityError   WikiLintIssueSeverity = "error"
+)
+
+// WikiLintIssue represents a single lint finding
+type WikiLintIssue struct {
+	Type     WikiLintIssueType     `json:"type"`
+	Severity WikiLintIssueSeverity `json:"severity"`
+	PageSlug string                `json:"page_slug"`
+	// TargetSlug identifies the other page involved in the issue (e.g. the
+	// broken link target, or the entity slug for a missing cross-ref). It is
+	// the structured field used by AutoFix instead of parsing Description.
+	TargetSlug  string `json:"target_slug,omitempty"`
+	Description string `json:"description"`
+	AutoFixable bool   `json:"auto_fixable"`
+}
+
+// WikiLintReport is the complete lint report for a wiki KB
+type WikiLintReport struct {
+	KnowledgeBaseID string           `json:"knowledge_base_id"`
+	Issues          []WikiLintIssue  `json:"issues"`
+	HealthScore     int              `json:"health_score"` // 0-100
+	Stats           *types.WikiStats `json:"stats"`
+	Summary         string           `json:"summary"`
+}
+
+// WikiLintService provides wiki health checking capabilities
+type WikiLintService struct {
+	wikiService      interfaces.WikiPageService
+	kbService        interfaces.KnowledgeBaseService
+	knowledgeService interfaces.KnowledgeService
+}
+
+// NewWikiLintService creates a new wiki lint service
+func NewWikiLintService(
+	wikiService interfaces.WikiPageService,
+	kbService interfaces.KnowledgeBaseService,
+	knowledgeService interfaces.KnowledgeService,
+) *WikiLintService {
+	return &WikiLintService{
+		wikiService:      wikiService,
+		kbService:        kbService,
+		knowledgeService: knowledgeService,
+	}
+}
+
+// lintCursorBatch is the per-batch limit for the streaming page walk.
+// Picked at 200 because wiki pages can carry multi-KB content blobs
+// and 200 rows × ~20KB ≈ 4MB resident at a time, which is well within
+// what we want to hold while running per-page checks.
+const lintCursorBatch = 200
+
+// RunLint performs a comprehensive health check on a wiki knowledge base.
+//
+// At 4w-document scale the legacy "load every page in one shot"
+// approach was the dominant tail in this method (and intermittently
+// caused OOM in production). We now walk the page set via
+// ListPagesCursor in lintCursorBatch-sized windows, accumulating
+// issues incrementally — memory stays bounded regardless of KB size.
+//
+// We also drop the GetGraph(Limit:0) call that the legacy path used
+// to compute the live-slug set. ListAllSlugs is a one-column projection
+// over the same predicate (kbID + status<>archived), so it gives the
+// same answer at a fraction of the cost.
+func (s *WikiLintService) RunLint(ctx context.Context, kbID string) (*WikiLintReport, error) {
+	// Validate KB
+	kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
+	if err != nil {
+		return nil, fmt.Errorf("get KB: %w", err)
+	}
+	if !kb.IsWikiEnabled() {
+		return nil, fmt.Errorf("KB %s is not a wiki type", kbID)
+	}
+
+	// Get stats
+	stats, err := s.wikiService.GetStats(ctx, kbID)
+	if err != nil {
+		return nil, fmt.Errorf("get stats: %w", err)
+	}
+
+	// Compute the live-slug set from the cheap one-column projection.
+	// This replaces a full GetGraph call (which materialized every node
+	// + edge) with a single Pluck("slug") query.
+	liveSlugs, err := s.wikiService.ListAllSlugs(ctx, kbID)
+	if err != nil {
+		return nil, fmt.Errorf("list all slugs: %w", err)
+	}
+	slugSet := make(map[string]bool, len(liveSlugs))
+	for _, slug := range liveSlugs {
+		slugSet[slug] = true
+	}
+
+	var issues []WikiLintIssue
+	healthScore := 100
+	knowledgeLive := make(map[string]bool) // kid -> exists; cached across pages
+
+	// First-pass walk: orphan / broken-link / empty / stale-ref
+	// detection. Each check is independent of order; we accumulate
+	// issues per-batch and the cursor walk keeps memory bounded.
+	//
+	// We collect entity / concept titles in this pass too so the
+	// missing-cross-ref check (which is intrinsically O(N×M) in
+	// distinct entities × pages) doesn't need a second walk to find
+	// candidates. The check itself runs in a second walk because it
+	// needs the full entity-title set to compare against any page.
+	entitySlugs := make(map[string]string) // slug -> title
+
+	cursor := ""
+	for {
+		pages, next, err := s.wikiService.ListPagesCursor(ctx, kbID, cursor, lintCursorBatch)
+		if err != nil {
+			return nil, fmt.Errorf("list pages cursor: %w", err)
+		}
+		if len(pages) == 0 {
+			break
+		}
+		for _, page := range pages {
+			// Track entity/concept titles for the second pass.
+			if page.PageType == types.WikiPageTypeEntity || page.PageType == types.WikiPageTypeConcept {
+				entitySlugs[page.Slug] = page.Title
+			}
+
+			// Check 1: Orphan pages (no inbound links, excluding system pages).
+			if page.PageType != types.WikiPageTypeIndex {
+				if len(page.InLinks) == 0 {
+					issues = append(issues, WikiLintIssue{
+						Type:        LintIssueOrphanPage,
+						Severity:    SeverityWarning,
+						PageSlug:    page.Slug,
+						Description: fmt.Sprintf("Page '%s' has no inbound links — it's disconnected from the wiki", page.Title),
+						AutoFixable: false,
+					})
+				}
+			}
+
+			// Check 2: Broken links — outlinks pointing at slugs that
+			// don't exist in the live set.
+			for _, outLink := range page.OutLinks {
+				if !slugSet[outLink] {
+					issues = append(issues, WikiLintIssue{
+						Type:        LintIssueBrokenLink,
+						Severity:    SeverityError,
+						PageSlug:    page.Slug,
+						TargetSlug:  outLink,
+						Description: fmt.Sprintf("Page '%s' links to [[%s]] which does not exist", page.Title, outLink),
+						AutoFixable: true,
+					})
+				}
+			}
+
+			// Check 3: Empty content.
+			content := strings.TrimSpace(page.Content)
+			if len(content) < 50 {
+				issues = append(issues, WikiLintIssue{
+					Type:        LintIssueEmptyContent,
+					Severity:    SeverityWarning,
+					PageSlug:    page.Slug,
+					Description: fmt.Sprintf("Page '%s' has very little content (%d chars)", page.Title, len(content)),
+					AutoFixable: true,
+				})
+			}
+
+			// Check 4: Stale source refs — source_refs pointing at
+			// soft-deleted knowledge. Cached knowledgeLive lookup keeps
+			// per-kid checks O(1) after the first batch encounters
+			// each id.
+			if s.knowledgeService != nil && page.PageType != types.WikiPageTypeIndex {
+				for _, ref := range page.SourceRefs {
+					kid := ref
+					if i := strings.Index(ref, "|"); i > 0 {
+						kid = ref[:i]
+					}
+					if kid == "" {
+						continue
+					}
+					live, seen := knowledgeLive[kid]
+					if !seen {
+						kn, err := s.knowledgeService.GetKnowledgeByIDOnly(ctx, kid)
+						live = err == nil && kn != nil
+						knowledgeLive[kid] = live
+					}
+					if !live {
+						issues = append(issues, WikiLintIssue{
+							Type:        LintIssueStaleRef,
+							Severity:    SeverityError,
+							PageSlug:    page.Slug,
+							TargetSlug:  kid,
+							Description: fmt.Sprintf("Page '%s' references deleted knowledge %s", page.Title, kid),
+							AutoFixable: true,
+						})
+					}
+				}
+			}
+		}
+		if next == "" {
+			break
+		}
+		cursor = next
+	}
+
+	// Second-pass walk: missing-cross-ref check. This needs the full
+	// entitySlugs map (built in pass 1), so it has to be a separate
+	// pass — but it's still streaming.
+	cursor = ""
+	for {
+		pages, next, err := s.wikiService.ListPagesCursor(ctx, kbID, cursor, lintCursorBatch)
+		if err != nil {
+			return nil, fmt.Errorf("list pages cursor (pass 2): %w", err)
+		}
+		if len(pages) == 0 {
+			break
+		}
+		for _, page := range pages {
+			lowerContent := strings.ToLower(page.Content)
+			outLinkSet := make(map[string]struct{}, len(page.OutLinks))
+			for _, l := range page.OutLinks {
+				outLinkSet[l] = struct{}{}
+			}
+			for slug, title := range entitySlugs {
+				if slug == page.Slug || title == "" {
+					continue
+				}
+				if !strings.Contains(lowerContent, strings.ToLower(title)) {
+					continue
+				}
+				if _, linked := outLinkSet[slug]; linked {
+					continue
+				}
+				issues = append(issues, WikiLintIssue{
+					Type:        LintIssueMissingCrossRef,
+					Severity:    SeverityInfo,
+					PageSlug:    page.Slug,
+					TargetSlug:  slug,
+					Description: fmt.Sprintf("Page '%s' mentions '%s' but doesn't link to [[%s]]", page.Title, title, slug),
+					AutoFixable: false,
+				})
+			}
+		}
+		if next == "" {
+			break
+		}
+		cursor = next
+	}
+
+	// Calculate health score
+	if stats.TotalPages > 0 {
+		// Penalize for orphans
+		orphanPct := float64(stats.OrphanCount) / float64(stats.TotalPages) * 100
+		if orphanPct > 50 {
+			healthScore -= 25
+		} else if orphanPct > 25 {
+			healthScore -= 10
+		}
+
+		// Penalize for broken links
+		brokenCount := 0
+		for _, issue := range issues {
+			if issue.Type == LintIssueBrokenLink {
+				brokenCount++
+			}
+		}
+		healthScore -= brokenCount * 5
+
+		// Penalize for no links at all
+		if stats.TotalLinks == 0 && stats.TotalPages > 2 {
+			healthScore -= 15
+		}
+
+		// Penalize for empty pages
+		emptyCount := 0
+		for _, issue := range issues {
+			if issue.Type == LintIssueEmptyContent {
+				emptyCount++
+			}
+		}
+		healthScore -= emptyCount * 3
+	}
+
+	if healthScore < 0 {
+		healthScore = 0
+	}
+
+	// Generate summary
+	var summary strings.Builder
+	errorCount := 0
+	warningCount := 0
+	infoCount := 0
+	for _, issue := range issues {
+		switch issue.Severity {
+		case SeverityError:
+			errorCount++
+		case SeverityWarning:
+			warningCount++
+		case SeverityInfo:
+			infoCount++
+		}
+	}
+
+	if len(issues) == 0 {
+		summary.WriteString("Wiki is healthy! No issues found.")
+	} else {
+		fmt.Fprintf(&summary, "Found %d issues: %d errors, %d warnings, %d suggestions.",
+			len(issues), errorCount, warningCount, infoCount)
+	}
+
+	report := &WikiLintReport{
+		KnowledgeBaseID: kbID,
+		Issues:          issues,
+		HealthScore:     healthScore,
+		Stats:           stats,
+		Summary:         summary.String(),
+	}
+
+	logger.Infof(ctx, "wiki lint: KB %s — health score %d/100, %d issues", kbID, healthScore, len(issues))
+
+	return report, nil
+}
+
+// AutoFix attempts to automatically fix fixable issues
+func (s *WikiLintService) AutoFix(ctx context.Context, kbID string) (int, error) {
+	report, err := s.RunLint(ctx, kbID)
+	if err != nil {
+		return 0, err
+	}
+
+	fixed := 0
+	for _, issue := range report.Issues {
+		if !issue.AutoFixable {
+			continue
+		}
+
+		switch issue.Type {
+		case LintIssueBrokenLink:
+			// Replace [[broken-slug]] with plain text so the reference text is
+			// preserved but no longer renders as a dangling wiki link.
+			if issue.TargetSlug == "" {
+				continue
+			}
+			page, err := s.wikiService.GetPageBySlug(ctx, kbID, issue.PageSlug)
+			if err != nil {
+				continue
+			}
+			target := issue.TargetSlug
+			page.Content = strings.ReplaceAll(page.Content, "[["+target+"]]", target)
+			if err := s.wikiService.UpdateAutoLinkedContent(ctx, page); err == nil {
+				fixed++
+			}
+
+		case LintIssueEmptyContent:
+			// Archive pages with very little content instead of deleting
+			page, err := s.wikiService.GetPageBySlug(ctx, kbID, issue.PageSlug)
+			if err != nil {
+				continue
+			}
+			// Don't archive the index page.
+			if page.PageType == types.WikiPageTypeIndex {
+				continue
+			}
+			page.Status = types.WikiPageStatusArchived
+			if _, err := s.wikiService.UpdatePage(ctx, page); err == nil {
+				fixed++
+			}
+
+		case LintIssueStaleRef:
+			// Strip source_refs that point at soft-deleted knowledge. If the
+			// page has no other live sources, delete it outright — leaving
+			// an orphan summary page is worse than removing it, because the
+			// model would still link to it from other pages and the
+			// wiki_read_source_doc drill-down would always fail.
+			if issue.TargetSlug == "" {
+				continue
+			}
+			page, err := s.wikiService.GetPageBySlug(ctx, kbID, issue.PageSlug)
+			if err != nil || page == nil {
+				continue
+			}
+			if page.PageType == types.WikiPageTypeIndex {
+				continue
+			}
+			remaining := removeSourceRef(page.SourceRefs, issue.TargetSlug)
+			if len(remaining) == 0 {
+				if err := s.wikiService.DeletePage(ctx, kbID, page.Slug); err == nil {
+					fixed++
+				}
+			} else if len(remaining) != len(page.SourceRefs) {
+				page.SourceRefs = remaining
+				if err := s.wikiService.UpdatePageMeta(ctx, page); err == nil {
+					fixed++
+				}
+			}
+		}
+	}
+
+	// Rebuild links after fixes
+	if fixed > 0 {
+		_ = s.wikiService.RebuildLinks(ctx, kbID)
+	}
+
+	logger.Infof(ctx, "wiki auto-fix: KB %s — fixed %d issues", kbID, fixed)
+	return fixed, nil
+}