xinmi/internal/application/service/wiki_lint.go

package service

import (
	"context"
	"fmt"
	"strings"

	"github.com/Tencent/WeKnora/internal/logger"
	"github.com/Tencent/WeKnora/internal/types"
	"github.com/Tencent/WeKnora/internal/types/interfaces"
)

// WikiLintIssueType defines the type of lint issue
type WikiLintIssueType string

const (
	LintIssueOrphanPage      WikiLintIssueType = "orphan_page"
	LintIssueBrokenLink      WikiLintIssueType = "broken_link"
	LintIssueStaleRef        WikiLintIssueType = "stale_ref"
	LintIssueMissingCrossRef WikiLintIssueType = "missing_cross_ref"
	LintIssueEmptyContent    WikiLintIssueType = "empty_content"
	LintIssueDuplicateSlug   WikiLintIssueType = "duplicate_slug"
)

// WikiLintIssueSeverity defines the severity of a lint issue
type WikiLintIssueSeverity string

const (
	SeverityInfo    WikiLintIssueSeverity = "info"
	SeverityWarning WikiLintIssueSeverity = "warning"
	SeverityError   WikiLintIssueSeverity = "error"
)

// WikiLintIssue represents a single lint finding
type WikiLintIssue struct {
	Type     WikiLintIssueType     `json:"type"`
	Severity WikiLintIssueSeverity `json:"severity"`
	PageSlug string                `json:"page_slug"`
	// TargetSlug identifies the other page involved in the issue (e.g. the
	// broken link target, or the entity slug for a missing cross-ref). It is
	// the structured field used by AutoFix instead of parsing Description.
	TargetSlug  string `json:"target_slug,omitempty"`
	Description string `json:"description"`
	AutoFixable bool   `json:"auto_fixable"`
}

// WikiLintReport is the complete lint report for a wiki KB
type WikiLintReport struct {
	KnowledgeBaseID string           `json:"knowledge_base_id"`
	Issues          []WikiLintIssue  `json:"issues"`
	HealthScore     int              `json:"health_score"` // 0-100
	Stats           *types.WikiStats `json:"stats"`
	Summary         string           `json:"summary"`
}

// WikiLintService provides wiki health checking capabilities
type WikiLintService struct {
	wikiService      interfaces.WikiPageService
	kbService        interfaces.KnowledgeBaseService
	knowledgeService interfaces.KnowledgeService
}

// NewWikiLintService creates a new wiki lint service
func NewWikiLintService(
	wikiService interfaces.WikiPageService,
	kbService interfaces.KnowledgeBaseService,
	knowledgeService interfaces.KnowledgeService,
) *WikiLintService {
	return &WikiLintService{
		wikiService:      wikiService,
		kbService:        kbService,
		knowledgeService: knowledgeService,
	}
}

// lintCursorBatch is the per-batch limit for the streaming page walk.
// Picked at 200 because wiki pages can carry multi-KB content blobs
// and 200 rows × ~20KB ≈ 4MB resident at a time, which is well within
// what we want to hold while running per-page checks.
const lintCursorBatch = 200

// RunLint performs a comprehensive health check on a wiki knowledge base.
//
// At 4w-document scale the legacy "load every page in one shot"
// approach was the dominant tail in this method (and intermittently
// caused OOM in production). We now walk the page set via
// ListPagesCursor in lintCursorBatch-sized windows, accumulating
// issues incrementally — memory stays bounded regardless of KB size.
//
// We also drop the GetGraph(Limit:0) call that the legacy path used
// to compute the live-slug set. ListAllSlugs is a one-column projection
// over the same predicate (kbID + status<>archived), so it gives the
// same answer at a fraction of the cost.
func (s *WikiLintService) RunLint(ctx context.Context, kbID string) (*WikiLintReport, error) {
	// Validate KB
	kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
	if err != nil {
		return nil, fmt.Errorf("get KB: %w", err)
	}
	if !kb.IsWikiEnabled() {
		return nil, fmt.Errorf("KB %s is not a wiki type", kbID)
	}

	// Get stats
	stats, err := s.wikiService.GetStats(ctx, kbID)
	if err != nil {
		return nil, fmt.Errorf("get stats: %w", err)
	}

	// Compute the live-slug set from the cheap one-column projection.
	// This replaces a full GetGraph call (which materialized every node
	// + edge) with a single Pluck("slug") query.
	liveSlugs, err := s.wikiService.ListAllSlugs(ctx, kbID)
	if err != nil {
		return nil, fmt.Errorf("list all slugs: %w", err)
	}
	slugSet := make(map[string]bool, len(liveSlugs))
	for _, slug := range liveSlugs {
		slugSet[slug] = true
	}

	var issues []WikiLintIssue
	healthScore := 100
	knowledgeLive := make(map[string]bool) // kid -> exists; cached across pages

	// First-pass walk: orphan / broken-link / empty / stale-ref
	// detection. Each check is independent of order; we accumulate
	// issues per-batch and the cursor walk keeps memory bounded.
	//
	// We collect entity / concept titles in this pass too so the
	// missing-cross-ref check (which is intrinsically O(N×M) in
	// distinct entities × pages) doesn't need a second walk to find
	// candidates. The check itself runs in a second walk because it
	// needs the full entity-title set to compare against any page.
	entitySlugs := make(map[string]string) // slug -> title

	cursor := ""
	for {
		pages, next, err := s.wikiService.ListPagesCursor(ctx, kbID, cursor, lintCursorBatch)
		if err != nil {
			return nil, fmt.Errorf("list pages cursor: %w", err)
		}
		if len(pages) == 0 {
			break
		}
		for _, page := range pages {
			// Track entity/concept titles for the second pass.
			if page.PageType == types.WikiPageTypeEntity || page.PageType == types.WikiPageTypeConcept {
				entitySlugs[page.Slug] = page.Title
			}

			// Check 1: Orphan pages (no inbound links, excluding system pages).
			if page.PageType != types.WikiPageTypeIndex {
				if len(page.InLinks) == 0 {
					issues = append(issues, WikiLintIssue{
						Type:        LintIssueOrphanPage,
						Severity:    SeverityWarning,
						PageSlug:    page.Slug,
						Description: fmt.Sprintf("Page '%s' has no inbound links — it's disconnected from the wiki", page.Title),
						AutoFixable: false,
					})
				}
			}

			// Check 2: Broken links — outlinks pointing at slugs that
			// don't exist in the live set.
			for _, outLink := range page.OutLinks {
				if !slugSet[outLink] {
					issues = append(issues, WikiLintIssue{
						Type:        LintIssueBrokenLink,
						Severity:    SeverityError,
						PageSlug:    page.Slug,
						TargetSlug:  outLink,
						Description: fmt.Sprintf("Page '%s' links to [[%s]] which does not exist", page.Title, outLink),
						AutoFixable: true,
					})
				}
			}

			// Check 3: Empty content.
			content := strings.TrimSpace(page.Content)
			if len(content) < 50 {
				issues = append(issues, WikiLintIssue{
					Type:        LintIssueEmptyContent,
					Severity:    SeverityWarning,
					PageSlug:    page.Slug,
					Description: fmt.Sprintf("Page '%s' has very little content (%d chars)", page.Title, len(content)),
					AutoFixable: true,
				})
			}

			// Check 4: Stale source refs — source_refs pointing at
			// soft-deleted knowledge. Cached knowledgeLive lookup keeps
			// per-kid checks O(1) after the first batch encounters
			// each id.
			if s.knowledgeService != nil && page.PageType != types.WikiPageTypeIndex {
				for _, ref := range page.SourceRefs {
					kid := ref
					if i := strings.Index(ref, "|"); i > 0 {
						kid = ref[:i]
					}
					if kid == "" {
						continue
					}
					live, seen := knowledgeLive[kid]
					if !seen {
						kn, err := s.knowledgeService.GetKnowledgeByIDOnly(ctx, kid)
						live = err == nil && kn != nil
						knowledgeLive[kid] = live
					}
					if !live {
						issues = append(issues, WikiLintIssue{
							Type:        LintIssueStaleRef,
							Severity:    SeverityError,
							PageSlug:    page.Slug,
							TargetSlug:  kid,
							Description: fmt.Sprintf("Page '%s' references deleted knowledge %s", page.Title, kid),
							AutoFixable: true,
						})
					}
				}
			}
		}
		if next == "" {
			break
		}
		cursor = next
	}

	// Second-pass walk: missing-cross-ref check. This needs the full
	// entitySlugs map (built in pass 1), so it has to be a separate
	// pass — but it's still streaming.
	cursor = ""
	for {
		pages, next, err := s.wikiService.ListPagesCursor(ctx, kbID, cursor, lintCursorBatch)
		if err != nil {
			return nil, fmt.Errorf("list pages cursor (pass 2): %w", err)
		}
		if len(pages) == 0 {
			break
		}
		for _, page := range pages {
			lowerContent := strings.ToLower(page.Content)
			outLinkSet := make(map[string]struct{}, len(page.OutLinks))
			for _, l := range page.OutLinks {
				outLinkSet[l] = struct{}{}
			}
			for slug, title := range entitySlugs {
				if slug == page.Slug || title == "" {
					continue
				}
				if !strings.Contains(lowerContent, strings.ToLower(title)) {
					continue
				}
				if _, linked := outLinkSet[slug]; linked {
					continue
				}
				issues = append(issues, WikiLintIssue{
					Type:        LintIssueMissingCrossRef,
					Severity:    SeverityInfo,
					PageSlug:    page.Slug,
					TargetSlug:  slug,
					Description: fmt.Sprintf("Page '%s' mentions '%s' but doesn't link to [[%s]]", page.Title, title, slug),
					AutoFixable: false,
				})
			}
		}
		if next == "" {
			break
		}
		cursor = next
	}

	// Calculate health score
	if stats.TotalPages > 0 {
		// Penalize for orphans
		orphanPct := float64(stats.OrphanCount) / float64(stats.TotalPages) * 100
		if orphanPct > 50 {
			healthScore -= 25
		} else if orphanPct > 25 {
			healthScore -= 10
		}

		// Penalize for broken links
		brokenCount := 0
		for _, issue := range issues {
			if issue.Type == LintIssueBrokenLink {
				brokenCount++
			}
		}
		healthScore -= brokenCount * 5

		// Penalize for no links at all
		if stats.TotalLinks == 0 && stats.TotalPages > 2 {
			healthScore -= 15
		}

		// Penalize for empty pages
		emptyCount := 0
		for _, issue := range issues {
			if issue.Type == LintIssueEmptyContent {
				emptyCount++
			}
		}
		healthScore -= emptyCount * 3
	}

	if healthScore < 0 {
		healthScore = 0
	}

	// Generate summary
	var summary strings.Builder
	errorCount := 0
	warningCount := 0
	infoCount := 0
	for _, issue := range issues {
		switch issue.Severity {
		case SeverityError:
			errorCount++
		case SeverityWarning:
			warningCount++
		case SeverityInfo:
			infoCount++
		}
	}

	if len(issues) == 0 {
		summary.WriteString("Wiki is healthy! No issues found.")
	} else {
		fmt.Fprintf(&summary, "Found %d issues: %d errors, %d warnings, %d suggestions.",
			len(issues), errorCount, warningCount, infoCount)
	}

	report := &WikiLintReport{
		KnowledgeBaseID: kbID,
		Issues:          issues,
		HealthScore:     healthScore,
		Stats:           stats,
		Summary:         summary.String(),
	}

	logger.Infof(ctx, "wiki lint: KB %s — health score %d/100, %d issues", kbID, healthScore, len(issues))

	return report, nil
}

// AutoFix attempts to automatically fix fixable issues
func (s *WikiLintService) AutoFix(ctx context.Context, kbID string) (int, error) {
	report, err := s.RunLint(ctx, kbID)
	if err != nil {
		return 0, err
	}

	fixed := 0
	for _, issue := range report.Issues {
		if !issue.AutoFixable {
			continue
		}

		switch issue.Type {
		case LintIssueBrokenLink:
			// Replace [[broken-slug]] with plain text so the reference text is
			// preserved but no longer renders as a dangling wiki link.
			if issue.TargetSlug == "" {
				continue
			}
			page, err := s.wikiService.GetPageBySlug(ctx, kbID, issue.PageSlug)
			if err != nil {
				continue
			}
			target := issue.TargetSlug
			page.Content = strings.ReplaceAll(page.Content, "[["+target+"]]", target)
			if err := s.wikiService.UpdateAutoLinkedContent(ctx, page); err == nil {
				fixed++
			}

		case LintIssueEmptyContent:
			// Archive pages with very little content instead of deleting
			page, err := s.wikiService.GetPageBySlug(ctx, kbID, issue.PageSlug)
			if err != nil {
				continue
			}
			// Don't archive the index page.
			if page.PageType == types.WikiPageTypeIndex {
				continue
			}
			page.Status = types.WikiPageStatusArchived
			if _, err := s.wikiService.UpdatePage(ctx, page); err == nil {
				fixed++
			}

		case LintIssueStaleRef:
			// Strip source_refs that point at soft-deleted knowledge. If the
			// page has no other live sources, delete it outright — leaving
			// an orphan summary page is worse than removing it, because the
			// model would still link to it from other pages and the
			// wiki_read_source_doc drill-down would always fail.
			if issue.TargetSlug == "" {
				continue
			}
			page, err := s.wikiService.GetPageBySlug(ctx, kbID, issue.PageSlug)
			if err != nil || page == nil {
				continue
			}
			if page.PageType == types.WikiPageTypeIndex {
				continue
			}
			remaining := removeSourceRef(page.SourceRefs, issue.TargetSlug)
			if len(remaining) == 0 {
				if err := s.wikiService.DeletePage(ctx, kbID, page.Slug); err == nil {
					fixed++
				}
			} else if len(remaining) != len(page.SourceRefs) {
				page.SourceRefs = remaining
				if err := s.wikiService.UpdatePageMeta(ctx, page); err == nil {
					fixed++
				}
			}
		}
	}

	// Rebuild links after fixes
	if fixed > 0 {
		_ = s.wikiService.RebuildLinks(ctx, kbID)
	}

	logger.Infof(ctx, "wiki auto-fix: KB %s — fixed %d issues", kbID, fixed)
	return fixed, nil
}