WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

docreader/parser/__init__.py

@@ -0,0 +1,40 @@
+"""
+Parser module for WeKnora document processing system.
+
+This module provides document parsers for various file formats including:
+- Microsoft Word documents (.doc, .docx)
+- PDF documents
+- Markdown files
+- Plain text files
+- Images with text content
+- Web pages
+
+The parsers extract content from documents and can split them into
+meaningful chunks for further processing and indexing.
+"""
+
+from .doc_parser import DocParser
+from .docx2_parser import Docx2Parser
+from .excel_parser import ExcelParser
+from .html_parser import HTMLParser
+from .image_parser import ImageParser
+from .markdown_parser import MarkdownParser
+from .parser import Parser
+from .pdf_parser import PDFParser
+from .registry import ParserEngineRegistry, registry
+from .web_parser import WebParser
+
+# Export public classes and modules
+__all__ = [
+    "Docx2Parser",
+    "DocParser",
+    "PDFParser",
+    "MarkdownParser",
+    "ImageParser",
+    "WebParser",
+    "Parser",
+    "ExcelParser",
+    "HTMLParser",
+    "ParserEngineRegistry",
+    "registry",
+]

docreader/parser/base_parser.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+import logging
+import os
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from docreader.models.document import Document
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+class BaseParser(ABC):
+    """Base parser interface.
+
+    After the lightweight refactoring, BaseParser only extracts markdown text
+    and raw image references from documents. Chunking, image storage, OCR,
+    and VLM caption are handled by the Go App module.
+    """
+
+    def __init__(
+        self,
+        file_name: str = "",
+        file_type: Optional[str] = None,
+        **kwargs,
+    ):
+        self.file_name = file_name
+        self.file_type = file_type or os.path.splitext(file_name)[1].lstrip(".")
+
+        logger.info(
+            "Initializing parser for file=%s, type=%s",
+            file_name,
+            self.file_type,
+        )
+
+    @abstractmethod
+    def parse_into_text(self, content: bytes) -> Document:
+        """Parse document content into markdown text.
+
+        Returns:
+            Document with ``content`` (markdown string) and optional
+            ``images`` dict mapping storage-relative paths to base64 data.
+        """
+
+    def parse(self, content: bytes) -> Document:
+        """Parse document and return markdown + image references.
+
+        No chunking, no OCR, no VLM caption — those are done in Go.
+        """
+        logger.info(
+            "Parsing document with %s, bytes: %d",
+            self.__class__.__name__,
+            len(content),
+        )
+        document = self.parse_into_text(content)
+        logger.info(
+            "Extracted %d characters from %s",
+            len(document.content),
+            self.file_name,
+        )
+        return document

docreader/parser/chain_parser.py

@@ -0,0 +1,179 @@
+"""
+Chain Parser Module
+
+This module provides two chain-of-responsibility pattern implementations for document parsing:
+1. FirstParser: Tries multiple parsers sequentially until one succeeds
+2. PipelineParser: Chains parsers where each parser processes the output of the previous one
+"""
+
+import logging
+from typing import Dict, List, Tuple, Type
+
+from docreader.models.document import Document
+from docreader.parser.base_parser import BaseParser
+from docreader.utils import endecode
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+class FirstParser(BaseParser):
+    """
+    First-success parser that tries multiple parsers in sequence.
+
+    This parser attempts to parse content using each registered parser in order.
+    It returns the result from the first parser that successfully produces a valid document.
+    If all parsers fail, it returns an empty Document.
+
+    Usage:
+        # Create a custom FirstParser with specific parser classes
+        CustomParser = FirstParser.create(MarkdownParser, HTMLParser)
+        parser = CustomParser()
+        document = parser.parse_into_text(content_bytes)
+    """
+
+    # Tuple of parser classes to be instantiated
+    _parser_cls: Tuple[Type["BaseParser"], ...] = ()
+
+    def __init__(self, *args, **kwargs):
+        """Initialize FirstParser with configured parser classes."""
+        super().__init__(*args, **kwargs)
+
+        # Instantiate all parser classes into parser instances
+        self._parsers: List[BaseParser] = []
+        for parser_cls in self._parser_cls:
+            parser = parser_cls(*args, **kwargs)
+            self._parsers.append(parser)
+
+    def parse_into_text(self, content: bytes) -> Document:
+        """Parse content using the first parser that succeeds.
+
+        Args:
+            content: Raw bytes content to be parsed
+
+        Returns:
+            Document: Parsed document from the first successful parser,
+                     or an empty Document if all parsers fail
+        """
+        for p in self._parsers:
+            logger.info(f"FirstParser: using parser {p.__class__.__name__}")
+            try:
+                document = p.parse_into_text(content)
+            except Exception:
+                logger.exception(
+                    "FirstParser: parser %s raised exception; trying next parser",
+                    p.__class__.__name__,
+                )
+                continue
+
+            if document.is_valid():
+                logger.info(f"FirstParser: parser {p.__class__.__name__} succeeded")
+                return document
+        return Document()
+
+    @classmethod
+    def create(cls, *parser_classes: Type["BaseParser"]) -> Type["FirstParser"]:
+        """Factory method to create a FirstParser subclass with specific parsers.
+
+        Args:
+            *parser_classes: Variable number of BaseParser subclasses to try in order
+
+        Returns:
+            Type[FirstParser]: A new FirstParser subclass configured with the given parsers
+
+        Example:
+            CustomParser = FirstParser.create(MarkdownParser, HTMLParser)
+            parser = CustomParser()
+        """
+        # Generate a descriptive class name based on parser names
+        names = "_".join([p.__name__ for p in parser_classes])
+        # Dynamically create a new class with the parser configuration
+        return type(f"FirstParser_{names}", (cls,), {"_parser_cls": parser_classes})
+
+
+class PipelineParser(BaseParser):
+    """
+    Pipeline parser that chains multiple parsers sequentially.
+
+    This parser processes content through a series of parsers where each parser
+    receives the output of the previous parser as input. Images from all parsers
+    are accumulated and merged into the final document.
+
+    Usage:
+        # Create a custom PipelineParser with specific parser classes
+        CustomParser = PipelineParser.create(PreParser, MarkdownParser, PostParser)
+        parser = CustomParser()
+        document = parser.parse_into_text(content_bytes)
+    """
+
+    # Tuple of parser classes to be instantiated and chained
+    _parser_cls: Tuple[Type["BaseParser"], ...] = ()
+
+    def __init__(self, *args, **kwargs):
+        """Initialize PipelineParser with configured parser classes."""
+        super().__init__(*args, **kwargs)
+
+        # Instantiate all parser classes into parser instances
+        self._parsers: List[BaseParser] = []
+        for parser_cls in self._parser_cls:
+            parser = parser_cls(*args, **kwargs)
+            self._parsers.append(parser)
+
+    def parse_into_text(self, content: bytes) -> Document:
+        """Parse content through a pipeline of parsers.
+
+        Each parser in the pipeline processes the output of the previous parser.
+        Images from all parsers are accumulated and merged into the final document.
+
+        Args:
+            content: Raw bytes content to be parsed
+
+        Returns:
+            Document: Final document after processing through all parsers,
+                     with accumulated images from all stages
+        """
+        # Accumulate images and metadata from all parsers
+        images: Dict[str, str] = {}
+        metadata: Dict = {}
+        document = Document()
+        for p in self._parsers:
+            logger.info(f"PipelineParser: using parser {p.__class__.__name__}")
+            # Parse content with current parser
+            document = p.parse_into_text(content)
+            # Convert document content back to bytes for next parser
+            content = endecode.encode_bytes(document.content)
+            # Accumulate images and metadata from this parser
+            images.update(document.images)
+            metadata.update(document.metadata)
+        # Merge all accumulated images and metadata into final document
+        document.images.update(images)
+        document.metadata.update(metadata)
+        return document
+
+    @classmethod
+    def create(cls, *parser_classes: Type["BaseParser"]) -> Type["PipelineParser"]:
+        """Factory method to create a PipelineParser subclass with specific parsers.
+
+        Args:
+            *parser_classes: Variable number of BaseParser subclasses to chain in order
+
+        Returns:
+            Type[PipelineParser]: A new PipelineParser subclass configured with the given parsers
+
+        Example:
+            CustomParser = PipelineParser.create(PreprocessParser, MarkdownParser)
+            parser = CustomParser()
+        """
+        # Generate a descriptive class name based on parser names
+        names = "_".join([p.__name__ for p in parser_classes])
+        # Dynamically create a new class with the parser configuration
+        return type(f"PipelineParser_{names}", (cls,), {"_parser_cls": parser_classes})
+
+
+if __name__ == "__main__":
+    from docreader.parser.markdown_parser import MarkdownParser
+
+    # Example: Create and use a FirstParser with MarkdownParser
+    FpCls = FirstParser.create(MarkdownParser)
+    lparser = FpCls()
+    print(lparser.parse_into_text(b"aaa"))

docreader/parser/concurrency.py

@@ -0,0 +1,39 @@
+import logging
+import threading
+from contextlib import contextmanager
+from typing import Dict, Iterator
+
+logger = logging.getLogger(__name__)
+
+_LIMITERS: Dict[str, threading.BoundedSemaphore] = {}
+_LIMITERS_LOCK = threading.Lock()
+
+
+def _get_limiter(name: str, max_workers: int) -> threading.BoundedSemaphore:
+    with _LIMITERS_LOCK:
+        limiter = _LIMITERS.get(name)
+        if limiter is None:
+            limiter = threading.BoundedSemaphore(max_workers)
+            _LIMITERS[name] = limiter
+        return limiter
+
+
+@contextmanager
+def parser_worker_limit(name: str, max_workers: int) -> Iterator[None]:
+    """Limit concurrent access to heavy, process-wide parser operations.
+
+    Set max_workers <= 0 to disable throttling for deployments that have enough
+    CPU/GPU resources and know the parser backend is safe under concurrency.
+    """
+
+    if max_workers <= 0:
+        yield
+        return
+
+    limiter = _get_limiter(name, max_workers)
+    logger.debug("Waiting for %s parser slot (max_workers=%d)", name, max_workers)
+    limiter.acquire()
+    try:
+        yield
+    finally:
+        limiter.release()

docreader/parser/doc_parser.py

@@ -0,0 +1,362 @@
+import logging
+import os
+import subprocess
+import time
+import uuid
+from pathlib import Path
+from typing import List, Optional
+
+import textract
+
+from docreader.config import CONFIG
+from docreader.models.document import Document
+from docreader.parser.docx2_parser import Docx2Parser
+from docreader.utils.tempfile import TempDirContext, TempFileContext
+
+logger = logging.getLogger(__name__)
+
+
+class SandboxExecutor:
+    """Sandbox executor for running commands with proxy configuration"""
+
+    def __init__(self, proxy: Optional[str] = None, default_timeout: int = 60):
+        """Initialize sandbox executor with configuration
+
+        Args:
+            proxy: Proxy URL to use for network access. If None, will use WEB_PROXY environment variable
+            default_timeout: Default timeout in seconds for command execution
+        """
+        # Get proxy from parameter, environment variable, or use default blocking proxy
+        # Use 'or None' to convert empty string to None, then apply default value
+        self.proxy = proxy or CONFIG.external_https_proxy or "http://128.0.0.1:1"
+        self.default_timeout = default_timeout
+
+    def execute_in_sandbox(self, cmd: List[str]) -> tuple:
+        """Execute command in sandbox with proxy configuration
+
+        Args:
+            cmd: Command to execute
+
+        Returns:
+            Tuple of (stdout, stderr, returncode)
+        """
+        # Try different sandbox methods in order of preference
+        sandbox_methods = [
+            self._execute_with_proxy,
+        ]
+
+        for method in sandbox_methods:
+            try:
+                return method(cmd)
+            except Exception as e:
+                logger.warning(f"Sandbox method {method.__name__} failed: {e}")
+                continue
+
+        raise RuntimeError("All sandbox methods failed")
+
+    def _execute_with_proxy(self, cmd: List[str]) -> tuple:
+        """Execute command with proxy configuration
+
+        Args:
+            cmd: Command to execute
+
+        Returns:
+            Tuple of (stdout, stderr, returncode)
+        """
+        # Set up environment with proxy configuration
+        env = os.environ.copy()
+        if self.proxy:
+            env["http_proxy"] = self.proxy
+            env["https_proxy"] = self.proxy
+            env["HTTP_PROXY"] = self.proxy
+            env["HTTPS_PROXY"] = self.proxy
+
+        logger.info(f"Executing command with proxy: {' '.join(cmd)}")
+        if self.proxy:
+            logger.info(f"Using proxy: {self.proxy}")
+
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=env,
+        )
+
+        try:
+            stdout, stderr = process.communicate(timeout=self.default_timeout)
+            return stdout, stderr, process.returncode
+        except subprocess.TimeoutExpired:
+            process.kill()
+            raise RuntimeError(
+                f"Command execution timeout after {self.default_timeout} seconds"
+            )
+
+
+logger = logging.getLogger(__name__)
+
+
+class DocParser(Docx2Parser):
+    """DOC document parser"""
+
+    def __init__(self, *args, **kwargs):
+        """Initialize DOC parser with sandbox executor"""
+        super().__init__(*args, **kwargs)
+        self.sandbox_executor = SandboxExecutor()
+
+    def parse_into_text(self, content: bytes) -> Document:
+        logger.info(f"Parsing DOC document, content size: {len(content)} bytes")
+
+        handle_chain = [
+            # 1. Try to convert to docx format to extract images
+            self._parse_with_docx,
+            # 2. If image extraction is not needed or conversion failed,
+            # try using antiword to extract text
+            self._parse_with_antiword,
+            # 3. If antiword extraction fails, use textract
+            # NOTE: _parse_with_textract is disabled due to SSRF vulnerability
+            # self._parse_with_textract,
+        ]
+
+        # Save byte content as a temporary file
+        with TempFileContext(content, ".doc") as temp_file_path:
+            for handle in handle_chain:
+                try:
+                    document = handle(temp_file_path)
+                    if document:
+                        return document
+                except Exception as e:
+                    logger.warning(f"Failed to parse DOC with {handle.__name__} {e}")
+
+            return Document(content="")
+
+    def _parse_with_docx(self, temp_file_path: str) -> Document:
+        logger.info("Multimodal enabled, attempting to extract images from DOC")
+
+        docx_content = self._try_convert_doc_to_docx(temp_file_path)
+        if not docx_content:
+            raise RuntimeError("Failed to convert DOC to DOCX")
+
+        logger.info("Successfully converted DOC to DOCX, using DocxParser")
+        # Use existing DocxParser to parse the converted docx
+        document = super(Docx2Parser, self).parse_into_text(docx_content)
+        logger.info(f"Extracted {len(document.content)} characters using DocxParser")
+        return document
+
+    def _parse_with_antiword(self, temp_file_path: str) -> Document:
+        logger.info("Attempting to parse DOC file with antiword")
+
+        # Check if antiword is installed
+        antiword_path = self._try_find_antiword()
+        if not antiword_path:
+            raise RuntimeError("antiword not found in PATH")
+
+        # Use antiword to extract text directly in sandbox
+        cmd = [antiword_path, temp_file_path]
+        logger.info("Executing antiword in sandbox with proxy configuration")
+
+        stdout, stderr, returncode = self.sandbox_executor.execute_in_sandbox(cmd)
+
+        if returncode != 0:
+            raise RuntimeError(
+                f"antiword extraction failed: {stderr.decode('utf-8', errors='ignore')}"
+            )
+        text = stdout.decode("utf-8", errors="ignore")
+        logger.info(f"Successfully extracted {len(text)} characters using antiword")
+        return Document(content=text)
+
+    def _parse_with_textract(self, temp_file_path: str) -> Document:
+        logger.info(f"Parsing DOC file with textract: {temp_file_path}")
+        text = textract.process(temp_file_path, method="antiword").decode("utf-8")
+        logger.info(f"Successfully extracted {len(text)} bytes of DOC using textract")
+        return Document(content=str(text))
+
+    def _try_convert_doc_to_docx(self, doc_path: str) -> Optional[bytes]:
+        """Convert DOC file to DOCX format
+
+        Uses LibreOffice/OpenOffice for conversion
+
+        Args:
+            doc_path: DOC file path
+
+        Returns:
+            Byte stream of DOCX file content, or None if conversion fails
+        """
+        logger.info(f"Converting DOC to DOCX: {doc_path}")
+
+        # Check if LibreOffice or OpenOffice is installed
+        soffice_path = self._try_find_soffice()
+        if not soffice_path:
+            return None
+
+        # Execute conversion command
+        logger.info(f"Using {soffice_path} to convert DOC to DOCX")
+
+        # LibreOffice shares a single user profile by default, so concurrent
+        # `soffice` invocations contend for the same profile lock and the loser
+        # silently fails to convert. Give each attempt a dedicated profile dir
+        # and retry a few times so concurrent requests don't fall back to the
+        # lower-fidelity antiword path.
+        max_attempts = 3
+        for attempt in range(1, max_attempts + 1):
+            # Create a temporary directory to store the converted file
+            with TempDirContext() as temp_dir, TempDirContext() as profile_dir:
+                user_installation = Path(profile_dir).as_uri()
+                cmd = [
+                    soffice_path,
+                    "--headless",
+                    f"-env:UserInstallation={user_installation}",
+                    "--convert-to",
+                    "docx",
+                    "--outdir",
+                    temp_dir,
+                    doc_path,
+                ]
+                logger.info(
+                    f"Running command in sandbox (attempt {attempt}/{max_attempts}): "
+                    f"{' '.join(cmd)}"
+                )
+
+                # Execute in sandbox with proxy configuration
+                stdout, stderr, returncode = self.sandbox_executor.execute_in_sandbox(
+                    cmd
+                )
+
+                if returncode != 0:
+                    logger.warning(
+                        f"Error converting DOC to DOCX (attempt {attempt}/"
+                        f"{max_attempts}): {stderr.decode('utf-8', errors='ignore')}"
+                    )
+                    if attempt < max_attempts:
+                        time.sleep(0.5 * attempt)
+                        continue
+                    return None
+
+                # Find the converted file
+                docx_file = [
+                    file for file in os.listdir(temp_dir) if file.endswith(".docx")
+                ]
+                logger.info(
+                    f"Found {len(docx_file)} DOCX file(s) in temporary directory"
+                )
+                for file in docx_file:
+                    converted_file = os.path.join(temp_dir, file)
+                    logger.info(f"Found converted file: {converted_file}")
+
+                    # Read the converted file content
+                    with open(converted_file, "rb") as f:
+                        docx_content = f.read()
+                        logger.info(
+                            f"Successfully read DOCX file, size: {len(docx_content)}"
+                        )
+                        return docx_content
+
+                # Conversion reported success but produced no docx; retry.
+                logger.warning(
+                    f"No DOCX produced despite success (attempt {attempt}/"
+                    f"{max_attempts})"
+                )
+                if attempt < max_attempts:
+                    time.sleep(0.5 * attempt)
+        return None
+
+    def _try_find_executable_path(
+        self,
+        executable_name: str,
+        possible_path: List[str] = [],
+        environment_variable: List[str] = [],
+    ) -> Optional[str]:
+        """Find executable path
+        Args:
+            executable_name: Executable name
+            possible_path: List of possible paths
+            environment_variable: List of environment variables to check
+            Returns:
+                Executable path, or None if not found
+        """
+        # Common executable paths
+        paths: List[str] = []
+        paths.extend(possible_path)
+        paths.extend(os.environ.get(env_var, "") for env_var in environment_variable)
+        paths = list(set(paths))
+
+        # Check if path is set in environment variable
+        for path in paths:
+            if os.path.exists(path):
+                logger.info(f"Found {executable_name} at {path}")
+                return path
+
+        # Try to find in PATH
+        result = subprocess.run(
+            ["which", executable_name], capture_output=True, text=True
+        )
+        if result.returncode == 0 and result.stdout.strip():
+            path = result.stdout.strip()
+            logger.info(f"Found {executable_name} at {path}")
+            return path
+
+        logger.warning(f"Failed to find {executable_name}")
+        return None
+
+    def _try_find_soffice(self) -> Optional[str]:
+        """Find LibreOffice/OpenOffice executable path
+
+        Returns:
+            Executable path, or None if not found
+        """
+        # Common LibreOffice/OpenOffice executable paths
+        possible_paths = [
+            # Linux
+            "/usr/bin/soffice",
+            "/usr/lib/libreoffice/program/soffice",
+            "/opt/libreoffice25.2/program/soffice",
+            # macOS
+            "/Applications/LibreOffice.app/Contents/MacOS/soffice",
+            # Windows
+            "C:\\Program Files\\LibreOffice\\program\\soffice.exe",
+            "C:\\Program Files (x86)\\LibreOffice\\program\\soffice.exe",
+        ]
+        return self._try_find_executable_path(
+            executable_name="soffice",
+            possible_path=possible_paths,
+            environment_variable=["LIBREOFFICE_PATH"],
+        )
+
+    def _try_find_antiword(self) -> Optional[str]:
+        """Find antiword executable path
+
+        Returns:
+            Executable path, or None if not found
+        """
+        # Common antiword executable paths
+        possible_paths = [
+            # Linux/macOS
+            "/usr/bin/antiword",
+            "/usr/local/bin/antiword",
+            # Windows
+            "C:\\Program Files\\Antiword\\antiword.exe",
+            "C:\\Program Files (x86)\\Antiword\\antiword.exe",
+        ]
+        return self._try_find_executable_path(
+            executable_name="antiword",
+            possible_path=possible_paths,
+            environment_variable=["ANTIWORD_PATH"],
+        )
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG)
+
+    file_name = "/path/to/your/test.doc"
+    logger.info(f"Processing file: {file_name}")
+    doc_parser = DocParser(
+        file_name=file_name,
+        enable_multimodal=True,
+        chunk_size=512,
+        chunk_overlap=60,
+    )
+    with open(file_name, "rb") as f:
+        content = f.read()
+
+    document = doc_parser.parse_into_text(content)
+    logger.info(f"Processing complete, extracted text length: {len(document.content)}")
+    logger.info(f"Sample text: {document.content[:200]}...")

docreader/parser/docx2_parser.py

@@ -0,0 +1,28 @@
+import logging
+
+from docreader.parser.chain_parser import FirstParser
+from docreader.parser.docx_parser import DocxParser
+from docreader.parser.markitdown_parser import MarkitdownParser
+
+logger = logging.getLogger(__name__)
+
+
+class Docx2Parser(FirstParser):
+    _parser_cls = (MarkitdownParser, DocxParser)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG)
+
+    your_file = "/path/to/your/file.docx"
+    parser = Docx2Parser(separators=[".", "?", "!", "。", "？", "！"])
+    with open(your_file, "rb") as f:
+        content = f.read()
+
+        document = parser.parse(content)
+        for cc in document.chunks:
+            logger.info(f"chunk: {cc}")
+
+        # document = parser.parse_into_text(content)
+        # logger.info(f"docx content: {document.content}")
+        # logger.info(f"find images {document.images.keys()}")

docreader/parser/docx_merge.py

@@ -0,0 +1,177 @@
+"""Expand vertically merged DOCX table cells before Markdown conversion."""
+
+from __future__ import annotations
+
+import copy
+import io
+import logging
+import zipfile
+
+from lxml import etree
+
+logger = logging.getLogger(__name__)
+
+_WORD_NAMESPACE = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+_W = f"{{{_WORD_NAMESPACE}}}"
+_TC_PR = f"{_W}tcPr"
+_VMERGE = f"{_W}vMerge"
+_GRID_SPAN = f"{_W}gridSpan"
+_TR_PR = f"{_W}trPr"
+_GRID_BEFORE = f"{_W}gridBefore"
+_VAL = f"{_W}val"
+
+
+def fill_vertical_merged_cells_docx(content: bytes) -> bytes:
+    """Repeat a vertical merge's master content in every covered table row.
+
+    DOCX stores vertical merges as a ``vMerge=restart`` cell followed by empty
+    ``vMerge=continue`` cells. Mammoth preserves that as an HTML ``rowspan``,
+    but Markdown tables cannot represent row spans and the later rows lose the
+    merged value. This rewrites only Word XML parts that contain ``vMerge``;
+    all other package parts and documents without vertical merges pass through
+    unchanged.
+    """
+    source = io.BytesIO(content)
+    if not zipfile.is_zipfile(source):
+        return content
+
+    with zipfile.ZipFile(source, "r") as archive:
+        normalized_names = {
+            info.filename.replace("\\", "/") for info in archive.infolist()
+        }
+        if "word/document.xml" not in normalized_names:
+            return content
+
+        rewritten_parts: dict[str, bytes] = {}
+        filled_count = 0
+        for info in archive.infolist():
+            name = info.filename.replace("\\", "/")
+            if not name.startswith("word/") or not name.endswith(".xml"):
+                continue
+
+            data = archive.read(info.filename)
+            if b"vMerge" not in data:
+                continue
+            rewritten, part_filled_count = _fill_vertical_merges_in_xml(data)
+            if rewritten is not None:
+                rewritten_parts[info.filename] = rewritten
+                filled_count += part_filled_count
+
+        if not rewritten_parts:
+            return content
+
+        output = io.BytesIO()
+        with zipfile.ZipFile(output, "w") as rewritten_archive:
+            for info in archive.infolist():
+                data = rewritten_parts.get(info.filename)
+                if data is None:
+                    data = archive.read(info.filename)
+                rewritten_archive.writestr(info, data)
+
+    logger.info(
+        "Expanded %d vertically merged DOCX continuation cells before parse",
+        filled_count,
+    )
+    return output.getvalue()
+
+
+def _fill_vertical_merges_in_xml(data: bytes) -> tuple[bytes | None, int]:
+    parser = etree.XMLParser(resolve_entities=False, no_network=True)
+    try:
+        root = etree.fromstring(data, parser=parser)
+    except etree.XMLSyntaxError:
+        return None, 0
+
+    changed = False
+    filled_count = 0
+    # Process nested tables first so copied cell content is already normalized.
+    for table in reversed(root.xpath(".//w:tbl", namespaces={"w": _WORD_NAMESPACE})):
+        table_changed, table_filled_count = _fill_table_vertical_merges(table)
+        changed = changed or table_changed
+        filled_count += table_filled_count
+
+    if not changed:
+        return None, 0
+    return etree.tostring(root, xml_declaration=True, encoding="UTF-8"), filled_count
+
+
+def _fill_table_vertical_merges(table: etree._Element) -> tuple[bool, int]:
+    active_merges: dict[int, tuple[etree._Element, ...]] = {}
+    changed = False
+    filled_count = 0
+
+    for row in table.findall(f"{_W}tr"):
+        next_active_merges: dict[int, tuple[etree._Element, ...]] = {}
+        column = _grid_before(row)
+
+        for cell in row.findall(f"{_W}tc"):
+            cell_properties = cell.find(_TC_PR)
+            span = _grid_span(cell_properties)
+            occupied_columns = range(column, column + span)
+            vertical_merge = (
+                cell_properties.find(_VMERGE) if cell_properties is not None else None
+            )
+
+            if vertical_merge is not None:
+                merge_type = vertical_merge.get(_VAL, "continue")
+                if merge_type == "restart":
+                    template = tuple(
+                        copy.deepcopy(child) for child in cell if child.tag != _TC_PR
+                    )
+                    for occupied_column in occupied_columns:
+                        next_active_merges[occupied_column] = template
+                    cell_properties.remove(vertical_merge)
+                    changed = True
+                else:
+                    template = next(
+                        (
+                            active_merges[occupied_column]
+                            for occupied_column in occupied_columns
+                            if occupied_column in active_merges
+                        ),
+                        None,
+                    )
+                    if template is not None:
+                        for child in list(cell):
+                            if child.tag != _TC_PR:
+                                cell.remove(child)
+                        for child in template:
+                            cell.append(copy.deepcopy(child))
+                        for occupied_column in occupied_columns:
+                            next_active_merges[occupied_column] = template
+                        cell_properties.remove(vertical_merge)
+                        changed = True
+                        filled_count += 1
+
+            column += span
+
+        active_merges = next_active_merges
+
+    return changed, filled_count
+
+
+def _grid_span(cell_properties: etree._Element | None) -> int:
+    if cell_properties is None:
+        return 1
+    grid_span = cell_properties.find(_GRID_SPAN)
+    if grid_span is None:
+        return 1
+    return _positive_int(grid_span.get(_VAL), default=1)
+
+
+def _grid_before(row: etree._Element) -> int:
+    row_properties = row.find(_TR_PR)
+    if row_properties is None:
+        return 0
+    grid_before = row_properties.find(_GRID_BEFORE)
+    if grid_before is None:
+        return 0
+    return _positive_int(grid_before.get(_VAL), default=0)
+
+
+def _positive_int(value: str | None, default: int) -> int:
+    try:
+        parsed = int(value or "")
+    except ValueError:
+        return default
+    return parsed if parsed >= 0 else default

docreader/parser/epub_parser.py

@@ -0,0 +1,332 @@
+"""EPUB parser.
+
+Parses EPUB files into markdown text and optional embedded images.
+"""
+
+import base64
+import logging
+import os
+import posixpath
+import tempfile
+from urllib.parse import unquote
+import uuid
+from typing import Dict
+
+from bs4 import BeautifulSoup
+import ebooklib
+from ebooklib import epub
+
+from docreader.models.document import Document
+from docreader.parser.base_parser import BaseParser
+
+logger = logging.getLogger(__name__)
+
+
+class EPUBParser(BaseParser):
+    """Parser for EPUB e-book files."""
+
+    def __init__(self, *args, extract_images: bool = True, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.extract_images = extract_images
+
+    def parse_into_text(self, content: bytes) -> Document:
+        logger.info(
+            "Parsing EPUB file: %s, size: %d bytes", self.file_name, len(content)
+        )
+        try:
+            with tempfile.NamedTemporaryFile(
+                suffix=".epub", delete=False, mode="wb"
+            ) as epub_file:
+                epub_file.write(content)
+                epub_path = epub_file.name
+            try:
+                book = epub.read_epub(epub_path)
+                metadata = self._extract_metadata(book)
+                markdown_content, images = self._extract_content(book)
+
+                metadata["source_format"] = "epub"
+                metadata["file_size"] = len(content)
+                metadata["chapter_count"] = len(
+                    [part for part in markdown_content.split("\n## ") if part.strip()]
+                )
+                metadata["image_count"] = len(images)
+                return Document(
+                    content=markdown_content, images=images, metadata=metadata
+                )
+            finally:
+                if os.path.exists(epub_path):
+                    os.unlink(epub_path)
+        except ImportError:
+            logger.error("ebooklib not installed")
+            raise
+        except Exception as e:
+            logger.warning(
+                "ebooklib failed to parse EPUB: %s, trying ZIP fallback", str(e)
+            )
+            return self._parse_epub_fallback(content)
+
+    def _parse_epub_fallback(self, content: bytes) -> Document:
+        """Parse EPUB directly as a ZIP when ebooklib cannot read it."""
+        import re
+        import zipfile
+        from io import BytesIO
+
+        metadata = {"source_format": "epub", "file_size": len(content)}
+        images: Dict[str, str] = {}
+        image_aliases: Dict[str, str] = {}
+        markdown_parts = []
+
+        with zipfile.ZipFile(BytesIO(content), "r") as epub_zip:
+            html_files = [
+                f
+                for f in epub_zip.namelist()
+                if f.endswith((".html", ".xhtml", ".htm"))
+            ]
+
+            def chapter_num(filename: str) -> int:
+                match = re.search(r"chapter(\d+)", filename, re.IGNORECASE)
+                return int(match.group(1)) if match else 999999
+
+            html_files.sort(key=chapter_num)
+
+            if self.extract_images:
+                img_exts = (".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg")
+                for img_file in epub_zip.namelist():
+                    if img_file.lower().endswith(img_exts):
+                        try:
+                            img_data = epub_zip.read(img_file)
+                            ext = os.path.splitext(img_file)[1]
+                            img_path = f"images/{uuid.uuid4().hex}{ext}"
+                            images[img_path] = base64.b64encode(img_data).decode(
+                                "utf-8"
+                            )
+                            self._add_image_aliases(image_aliases, img_file, img_path)
+                        except Exception as e:
+                            logger.warning(
+                                "Failed to extract image %s: %s", img_file, e
+                            )
+
+            for html_file in html_files:
+                try:
+                    html_content = epub_zip.read(html_file).decode(
+                        "utf-8", errors="ignore"
+                    )
+                    chapter_md = self._html_to_markdown(
+                        html_content,
+                        image_aliases=image_aliases,
+                        base_path=posixpath.dirname(html_file),
+                    )
+                    base = os.path.basename(html_file)
+                    title = base.replace(".html", "").replace(".xhtml", "")
+                    title = re.sub(
+                        r"chapter[_-]?", "Chapter ", title, flags=re.IGNORECASE
+                    )
+                    title = title.replace("_", " ").replace("-", " ").title()
+                    if chapter_md.strip():
+                        markdown_parts.append(f"## {title}\n\n{chapter_md}")
+                except Exception as e:
+                    logger.warning("Failed to process %s: %s", html_file, e)
+
+        metadata["chapter_count"] = len(markdown_parts)
+        metadata["image_count"] = len(images)
+        return Document(
+            content="\n\n".join(markdown_parts), images=images, metadata=metadata
+        )
+
+    def _extract_metadata(self, book) -> Dict[str, str]:
+        metadata: Dict[str, str] = {}
+        mapping = {
+            "title": "title",
+            "creator": "author",
+            "publisher": "publisher",
+            "language": "language",
+            "description": "description",
+            "date": "date",
+            "identifier": "isbn",
+        }
+        for dc_key, out_key in mapping.items():
+            try:
+                values = book.get_metadata("DC", dc_key)
+            except Exception:
+                values = None
+            if values:
+                if out_key == "author":
+                    metadata[out_key] = ", ".join(value[0] for value in values)
+                else:
+                    metadata[out_key] = values[0][0]
+        return metadata
+
+    def _extract_content(self, book) -> tuple[str, Dict[str, str]]:
+        markdown_parts = []
+        images: Dict[str, str] = {}
+        image_aliases: Dict[str, str] = {}
+
+        try:
+            toc = book.get_table_of_contents()
+        except Exception as e:
+            logger.debug("Failed to get TOC: %s, processing all HTML items", e)
+            toc = []
+
+        html_items = {}
+        for item in book.get_items():
+            if item.get_type() == ebooklib.ITEM_DOCUMENT:
+                html_items[item.get_name()] = item
+
+        if self.extract_images:
+            for item in book.get_items():
+                if item.get_type() == ebooklib.ITEM_IMAGE:
+                    img_data = item.get_content()
+                    ext = os.path.splitext(item.get_name())[1]
+                    img_path = f"images/{uuid.uuid4().hex}{ext}"
+                    images[img_path] = base64.b64encode(img_data).decode("utf-8")
+                    self._add_image_aliases(image_aliases, item.get_name(), img_path)
+
+        if toc:
+            for item in toc:
+                entries = item if isinstance(item, tuple) else (item,)
+                for sub in entries:
+                    if hasattr(sub, "get_name") and sub.get_name() in html_items:
+                        markdown_parts.append(
+                            self._process_chapter(
+                                html_items[sub.get_name()],
+                                toc_index=len(markdown_parts),
+                                image_aliases=image_aliases,
+                            )
+                        )
+
+        if not markdown_parts:
+            for _name, item in html_items.items():
+                markdown_parts.append(
+                    self._process_chapter(
+                        item,
+                        toc_index=len(markdown_parts),
+                        image_aliases=image_aliases,
+                    )
+                )
+
+        return "\n\n".join(part for part in markdown_parts if part.strip()), images
+
+    def _process_chapter(
+        self,
+        html_item,
+        toc_index: int = 0,
+        image_aliases: Dict[str, str] | None = None,
+    ) -> str:
+        try:
+            html_content = html_item.get_content()
+            soup = BeautifulSoup(html_content, "lxml")
+            title_tag = soup.find(["h1", "h2"])
+            if title_tag:
+                chapter_title = title_tag.get_text().strip()
+                title_tag.decompose()
+            else:
+                chapter_title = html_item.get_name().replace("/", "")
+                chapter_title = chapter_title.replace(".xhtml", "")
+                chapter_title = chapter_title.replace("-", " ").title()
+            body_html = str(soup.body) if soup.body else str(html_content)
+            chapter_md = self._html_to_markdown(
+                body_html,
+                image_aliases=image_aliases,
+                base_path=posixpath.dirname(html_item.get_name()),
+            )
+            return f"## {chapter_title}\n\n{chapter_md}"
+        except Exception as e:
+            logger.error(
+                "Failed to process chapter %s: %s", html_item.get_name(), e
+            )
+            return f"## Chapter {toc_index + 1}\n\n[Error processing chapter: {e}]"
+
+    def _html_to_markdown(
+        self,
+        html_content: str,
+        image_aliases: Dict[str, str] | None = None,
+        base_path: str = "",
+    ) -> str:
+        try:
+            from bs4 import Comment
+            from markdownify import markdownify as md
+
+            soup = BeautifulSoup(html_content, "lxml")
+            for element in soup(["script", "style"]):
+                element.decompose()
+            for comment in soup.find_all(
+                string=lambda text: isinstance(text, Comment)
+            ):
+                comment.extract()
+            self._strip_internal_links(soup)
+            if image_aliases:
+                self._rewrite_image_sources(soup, image_aliases, base_path)
+            markdown_text = md(str(soup), heading_style="ATX")
+            return "\n".join(
+                line.strip() for line in markdown_text.split("\n") if line.strip()
+            )
+        except ImportError:
+            logger.warning("markdownify not available, using HTML as-is")
+            return f"```html\n{html_content}\n```"
+        except Exception as e:
+            logger.error("HTML to Markdown conversion failed: %s", e)
+            return f"```html\n{html_content}\n```"
+
+    @staticmethod
+    def _strip_internal_links(soup: BeautifulSoup) -> None:
+        """Unwrap links that don't point to an external resource.
+
+        EPUB internal links (other chapter files, ``#fragment`` anchors, TOC
+        entries) become dead links after extraction. Keep only external links
+        and replace everything else with its text.
+        """
+        external = ("http://", "https://", "mailto:", "tel:")
+        for link in soup.find_all("a"):
+            href = (link.get("href") or "").strip().lower()
+            if not href or not href.startswith(external):
+                link.unwrap()
+
+    @staticmethod
+    def _add_image_aliases(
+        image_aliases: Dict[str, str],
+        original_path: str,
+        image_path: str,
+    ) -> None:
+        normalized = EPUBParser._normalize_epub_path(original_path)
+        aliases = {
+            original_path,
+            normalized,
+            unquote(original_path),
+            unquote(normalized),
+            posixpath.basename(normalized),
+        }
+        for alias in aliases:
+            if alias:
+                image_aliases[alias] = image_path
+
+    @staticmethod
+    def _rewrite_image_sources(
+        soup: BeautifulSoup,
+        image_aliases: Dict[str, str],
+        base_path: str = "",
+    ) -> None:
+        for img in soup.find_all("img"):
+            src = (img.get("src") or "").strip()
+            if not src:
+                continue
+            normalized_src = EPUBParser._normalize_epub_path(src)
+            candidates = [
+                src,
+                normalized_src,
+                unquote(src),
+                unquote(normalized_src),
+                posixpath.basename(normalized_src),
+            ]
+            if base_path:
+                joined = EPUBParser._normalize_epub_path(posixpath.join(base_path, src))
+                candidates.extend([joined, unquote(joined)])
+            for candidate in candidates:
+                if candidate in image_aliases:
+                    img["src"] = image_aliases[candidate]
+                    break
+
+    @staticmethod
+    def _normalize_epub_path(path: str) -> str:
+        path = unquote(path).split("#", 1)[0].split("?", 1)[0].replace("\\", "/")
+        normalized = posixpath.normpath(path)
+        return "" if normalized == "." else normalized.lstrip("/")

docreader/parser/excel_convert.py

@@ -0,0 +1,149 @@
+"""LibreOffice helpers for normalizing legacy or unusual Excel uploads."""
+
+from __future__ import annotations
+
+import logging
+import os
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+_XLS_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
+_ZIP_MAGIC = b"PK\x03\x04"
+
+
+def detect_excel_format(content: bytes) -> str | None:
+    """Return pandas/excel format id: xlsx, xls, xlsb, ods, or None."""
+    if not content:
+        return None
+
+    from pandas.io.excel._base import inspect_excel_format
+
+    ext = inspect_excel_format(content_or_path=content)
+    if ext in ("xlsx", "xls", "xlsb", "ods"):
+        return ext
+    if ext == "zip":
+        return "xlsx"
+
+    if content.startswith(_ZIP_MAGIC):
+        return "xlsx"
+    if len(content) >= len(_XLS_MAGIC) and content.startswith(_XLS_MAGIC):
+        return "xls"
+    return None
+
+
+def engine_for_format(ext: str | None) -> str:
+    if ext == "xls":
+        return "xlrd"
+    if ext in ("xlsx", "xlsb"):
+        return "openpyxl"
+    if ext == "ods":
+        return "odf"
+    return "openpyxl"
+
+
+def convert_excel_to_xlsx_bytes(content: bytes, suffix: str = ".xlsx") -> bytes | None:
+    """Convert arbitrary spreadsheet bytes to XLSX using LibreOffice, if available."""
+    soffice = find_soffice()
+    if not soffice:
+        return None
+
+    max_attempts = 3
+    for attempt in range(1, max_attempts + 1):
+        with tempfile.TemporaryDirectory() as temp_dir, tempfile.TemporaryDirectory() as profile_dir:
+            src = os.path.join(temp_dir, f"input{suffix}")
+            with open(src, "wb") as handle:
+                handle.write(content)
+
+            user_installation = Path(profile_dir).as_uri()
+            cmd = [
+                soffice,
+                "--headless",
+                f"-env:UserInstallation={user_installation}",
+                "--convert-to",
+                "xlsx",
+                "--outdir",
+                temp_dir,
+                src,
+            ]
+            try:
+                result = subprocess.run(cmd, capture_output=True, timeout=120)
+            except (OSError, subprocess.TimeoutExpired) as exc:
+                logger.warning("LibreOffice convert failed to start: %s", exc)
+                return None
+
+            if result.returncode != 0:
+                stderr = result.stderr.decode("utf-8", errors="ignore")
+                logger.warning(
+                    "LibreOffice convert failed (attempt %s/%s): %s",
+                    attempt,
+                    max_attempts,
+                    stderr,
+                )
+                if attempt < max_attempts:
+                    time.sleep(0.5 * attempt)
+                    continue
+                return None
+
+            for name in os.listdir(temp_dir):
+                if name.endswith(".xlsx"):
+                    with open(os.path.join(temp_dir, name), "rb") as handle:
+                        converted = handle.read()
+                    logger.info(
+                        "Converted spreadsheet via LibreOffice (%s -> xlsx, %d bytes)",
+                        suffix,
+                        len(converted),
+                    )
+                    return converted
+
+            if attempt < max_attempts:
+                time.sleep(0.5 * attempt)
+    return None
+
+
+def normalize_excel_bytes(content: bytes, file_type: str | None = None) -> bytes:
+    """Return bytes readable by pandas, converting via LibreOffice when needed."""
+    ext = detect_excel_format(content)
+    if ext is not None:
+        return content
+
+    suffixes = []
+    if file_type:
+        suffixes.append(f".{file_type.lstrip('.')}")
+    suffixes.extend([".xlsx", ".xls", ".et", ".csv"])
+    seen: set[str] = set()
+    for suffix in suffixes:
+        if suffix in seen:
+            continue
+        seen.add(suffix)
+        converted = convert_excel_to_xlsx_bytes(content, suffix=suffix)
+        if converted and detect_excel_format(converted) is not None:
+            return converted
+
+    raise ValueError(
+        "Unrecognized Excel file format; the file may be corrupt, encrypted, "
+        "or not a spreadsheet"
+    )
+
+
+def find_soffice() -> Optional[str]:
+    possible_paths = [
+        "/usr/bin/soffice",
+        "/usr/lib/libreoffice/program/soffice",
+        "/opt/libreoffice25.2/program/soffice",
+        "/Applications/LibreOffice.app/Contents/MacOS/soffice",
+        "C:\\Program Files\\LibreOffice\\program\\soffice.exe",
+        "C:\\Program Files (x86)\\LibreOffice\\program\\soffice.exe",
+    ]
+    for path in possible_paths:
+        if path and os.path.exists(path):
+            return path
+
+    result = subprocess.run(["which", "soffice"], capture_output=True, text=True)
+    if result.returncode == 0 and result.stdout.strip():
+        return result.stdout.strip()
+    return None

docreader/parser/excel_parser.py

@@ -0,0 +1,265 @@
+"""
+Excel Parser Module
+
+This module provides functionality to parse Excel files (.xlsx, .xls) into
+structured Document objects with text content and chunks. It supports multiple
+sheets and handles various Excel formats using pandas.
+"""
+import logging
+import re
+from io import BytesIO
+from typing import Any, List
+
+import pandas as pd
+
+from docreader.models.document import Chunk, Document
+from docreader.parser.base_parser import BaseParser
+from docreader.parser.excel_convert import (
+    convert_excel_to_xlsx_bytes,
+    detect_excel_format,
+    engine_for_format,
+    normalize_excel_bytes,
+)
+from docreader.parser.xlsx_merge import fill_merged_cells_xlsx
+from docreader.parser.xlsx_repair import repair_xlsx_bytes
+
+logger = logging.getLogger(__name__)
+
+# Pattern to detect Excel image function strings that should be excluded from
+# parsed text content.  WPS uses =DISPIMG("ID",mode) to embed images in cells;
+# when opened by other tools the formula may appear as plain text prefixed with
+# "_xlfn." or "=".  Office 365 uses =_xlfn.IMAGE(url, ...) similarly.
+# The _xlfn. prefix is optional — WPS may omit it (e.g. =DISPIMG("ID",1)).
+_IMAGE_FUNC_RE = re.compile(
+    r"^=?(_xlfn\.)?(DISPIMG|IMAGE)\(", re.IGNORECASE
+)
+
+
+def _is_image_function(value: object) -> bool:
+    """Return True if *value* looks like an embedded-image function string."""
+    if not isinstance(value, str):
+        return False
+    return _IMAGE_FUNC_RE.match(value) is not None
+
+
+class ExcelParser(BaseParser):
+    """Parser for Excel files (.xlsx, .xls).
+    
+    This parser extracts text content from Excel files by processing all sheets
+    and converting each row into a structured text format. Each row becomes a
+    separate chunk with key-value pairs.
+    
+    Features:
+        - Supports multiple sheets in a single Excel file
+        - Automatically removes completely empty rows
+        - Converts each row to "column: value" format
+        - Creates individual chunks for each row for better granularity
+        
+    Example:
+        >>> parser = ExcelParser()
+        >>> with open("data.xlsx", "rb") as f:
+        ...     content = f.read()
+        ...     document = parser.parse_into_text(content)
+        >>> print(document.content)
+        Name: John,Age: 30,City: NYC
+        Name: Jane,Age: 25,City: LA
+    """
+
+    def __init__(
+        self,
+        file_name: str = "",
+        file_type: str | None = None,
+        xlsx_first_row_as_header: Any = False,
+        **kwargs: Any,
+    ):
+        super().__init__(file_name=file_name, file_type=file_type, **kwargs)
+        self.xlsx_first_row_as_header = _parse_bool(xlsx_first_row_as_header)
+    
+    def parse_into_text(self, content: bytes) -> Document:
+        """Parse Excel file bytes into a Document object.
+        
+        Args:
+            content: Raw bytes of the Excel file
+            
+        Returns:
+            Document: Parsed document containing:
+                - content: Full text with all rows from all sheets
+                - chunks: List of Chunk objects, one per row
+                
+        Note:
+            - Empty rows (all NaN values) are automatically skipped
+            - Each row is formatted as: "col1: val1,col2: val2,..."
+            - Chunks maintain sequential ordering across all sheets
+        """
+        chunks: List[Chunk] = []
+        text: List[str] = []
+        start, end = 0, 0
+
+        excel_file = _open_excel_file(content, file_type=self.file_type)
+        
+        # Process each sheet in the Excel file
+        for excel_sheet_name in excel_file.sheet_names:
+            df = _read_sheet_dataframe(
+                excel_file,
+                excel_sheet_name,
+                xlsx_first_row_as_header=self.xlsx_first_row_as_header,
+            )
+            # Remove rows where all values are NaN (completely empty rows)
+            df.dropna(how="all", inplace=True)
+
+            # Process each row in the DataFrame
+            for _, row in df.iterrows():
+                page_content = []
+                # Build key-value pairs for non-null values
+                for k, v in row.items():
+                    if pd.notna(v) and not _is_image_function(v):
+                        page_content.append(f"{k}: {v}")
+                
+                # Skip rows with no valid content
+                if not page_content:
+                    continue
+                
+                # Format row as comma-separated key-value pairs
+                content_row = ",".join(page_content) + "\n"
+                end += len(content_row)
+                text.append(content_row)
+                
+                # Create a chunk for this row with position tracking
+                chunks.append(
+                    Chunk(content=content_row, seq=len(chunks), start=start, end=end)
+                )
+                start = end
+
+        # Combine all text and return as Document
+        return Document(content="".join(text), chunks=chunks)
+
+
+def _read_sheet_dataframe(
+    excel_file: pd.ExcelFile,
+    sheet_name: str,
+    xlsx_first_row_as_header: bool = False,
+) -> pd.DataFrame:
+    """Read a worksheet into a DataFrame with stable column labels."""
+    from openpyxl.utils import get_column_letter
+
+    # Keep row 1 as data by default for both XLSX and legacy XLS. Users can
+    # explicitly restore the historical behavior where row 1 supplies semantic
+    # labels for every row.
+    df = excel_file.parse(sheet_name=sheet_name, header=None)
+    if xlsx_first_row_as_header and len(df.index) >= 2:
+        df.columns = _stable_header_labels(df.iloc[0].tolist())
+        return df.iloc[1:].copy()
+
+    df.columns = [get_column_letter(idx + 1) for idx in range(len(df.columns))]
+    return df
+
+
+def _stable_header_labels(values: List[object]) -> List[str]:
+    """Build non-empty, unique labels from an explicitly selected header row."""
+    from openpyxl.utils import get_column_letter
+
+    labels: List[str] = []
+    counts: dict[str, int] = {}
+    for index, value in enumerate(values, start=1):
+        label = ""
+        if pd.notna(value) and not _is_image_function(value):
+            label = str(value).strip()
+        if not label:
+            label = get_column_letter(index)
+
+        count = counts.get(label, 0) + 1
+        counts[label] = count
+        labels.append(label if count == 1 else f"{label}_{count}")
+    return labels
+
+
+def _parse_bool(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    return str(value).strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _prepare_xlsx_bytes(data: bytes) -> bytes:
+    repaired = repair_xlsx_bytes(data)
+    if repaired is not None:
+        data = repaired
+    return fill_merged_cells_xlsx(data)
+
+
+def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFile:
+    """Open an Excel workbook with explicit engine selection and fallbacks."""
+    data = content
+    converted_via_soffice = False
+
+    while True:
+        ext = detect_excel_format(data)
+        if ext is None:
+            if converted_via_soffice:
+                raise ValueError(
+                    "Excel file format cannot be determined, you must specify an engine manually."
+                )
+            try:
+                data = normalize_excel_bytes(data, file_type=file_type)
+            except ValueError as exc:
+                raise ValueError(
+                    "Excel file format cannot be determined, you must specify an engine manually."
+                ) from exc
+            converted_via_soffice = True
+            continue
+
+        if ext == "ods":
+            converted = convert_excel_to_xlsx_bytes(data, suffix=".ods")
+            if converted:
+                data = converted
+                continue
+
+        engine = engine_for_format(ext)
+        if ext == "xlsx":
+            data = _prepare_xlsx_bytes(data)
+            engine = "openpyxl"
+        try:
+            return pd.ExcelFile(BytesIO(data), engine=engine)
+        except ImportError as exc:
+            raise ValueError(
+                f"Excel engine {engine!r} is not available for .{ext} files"
+            ) from exc
+        except KeyError as exc:
+            if "sharedStrings.xml" not in str(exc) or engine != "openpyxl":
+                raise
+            repaired = repair_xlsx_bytes(data)
+            if repaired is None:
+                raise
+            logger.info("Repaired XLSX sharedStrings packaging before parse")
+            data = _prepare_xlsx_bytes(repaired)
+            continue
+        except ValueError as exc:
+            if converted_via_soffice or "cannot be determined" not in str(exc):
+                raise
+            try:
+                data = normalize_excel_bytes(content, file_type=file_type)
+            except ValueError:
+                raise
+            converted_via_soffice = True
+            continue
+
+
+if __name__ == "__main__":
+    # Example usage: Parse an Excel file and display results
+    logging.basicConfig(level=logging.DEBUG)
+
+    # Specify the path to your Excel file
+    your_file = "/path/to/your/file.xlsx"
+    parser = ExcelParser()
+    
+    # Read and parse the Excel file
+    with open(your_file, "rb") as f:
+        content = f.read()
+        document = parser.parse_into_text(content)
+        
+        # Display the full document content
+        logger.error(document.content)
+
+        # Display the first chunk as an example
+        for chunk in document.chunks:
+            logger.error(chunk.content)
+            break  # Only show the first chunk

docreader/parser/html_parser.py

@@ -0,0 +1,37 @@
+"""Static HTML file parser."""
+
+from bs4 import BeautifulSoup
+
+from docreader.models.document import Document
+from docreader.parser.base_parser import BaseParser
+from docreader.parser.chain_parser import PipelineParser
+from docreader.parser.markdown_parser import MarkdownParser
+from docreader.parser.mhtml_parser import MHTMLParser
+
+
+class HTMLToMarkdownParser(BaseParser):
+    """Convert uploaded HTML bytes to Markdown without browser or network access."""
+
+    def parse_into_text(self, content: bytes) -> Document:
+        if not content.strip():
+            return Document()
+
+        # Inspect the original bytes so BOMs and HTML charset declarations are
+        # honored before the shared Markdown conversion runs.
+        html = BeautifulSoup(content, "lxml").decode()
+        markdown = MHTMLParser(
+            file_name=self.file_name,
+            file_type=self.file_type,
+            extract_images=False,
+        ).html_to_markdown(
+            html,
+            strip_internal_links=False,
+            fallback_to_raw_html=False,
+        )
+        return Document(content=markdown or "")
+
+
+class HTMLParser(PipelineParser):
+    """Extract static HTML content and normalize the resulting Markdown."""
+
+    _parser_cls = (HTMLToMarkdownParser, MarkdownParser)

docreader/parser/image_parser.py

@@ -0,0 +1,28 @@
+import base64
+import logging
+import os
+
+from docreader.models.document import Document
+from docreader.parser.base_parser import BaseParser
+
+logger = logging.getLogger(__name__)
+
+
+class ImageParser(BaseParser):
+    """Parser for standalone image files.
+
+    Returns the image as a markdown reference with the raw image data
+    in Document.images so that the Go-side ImageResolver (or main.py's
+    _resolve_images) can handle storage upload.
+    """
+
+    def parse_into_text(self, content: bytes) -> Document:
+        logger.info("Parsing image file=%s, size=%d bytes", self.file_name, len(content))
+
+        ext = os.path.splitext(self.file_name)[1].lower() or ".png"
+        ref_path = f"images/{self.file_name}"
+
+        text = f"![{self.file_name}]({ref_path})"
+        images = {ref_path: base64.b64encode(content).decode()}
+
+        return Document(content=text, images=images)

docreader/parser/markdown_parser.py

@@ -0,0 +1,489 @@
+"""
+Markdown Parser Module
+
+This module provides comprehensive Markdown parsing functionality including:
+- Table formatting and standardization
+- Base64 image extraction and conversion
+- Image path replacement and URL generation
+- Pipeline-based parsing with multiple stages
+
+The parser uses a pipeline approach to process Markdown content through
+multiple stages: table formatting -> image processing.
+"""
+
+import base64
+import logging
+import os
+import re
+import uuid
+from typing import Dict, List, Match, Optional, Tuple
+
+from docreader.models.document import Document
+from docreader.parser.base_parser import BaseParser
+from docreader.parser.chain_parser import PipelineParser
+from docreader.utils import endecode
+
+# Get logger object
+logger = logging.getLogger(__name__)
+
+_SEPARATOR_CELL = re.compile(r"^:?-{3,}:?$")
+# Cells of a GFM alignment row as matched by the previous ``align_pattern``.
+# Looser than ``_SEPARATOR_CELL``: any non-empty run of ``:``/``-`` qualifies.
+_ALIGNMENT_CELL = re.compile(r"^[:-]+$")
+
+
+class MarkdownTableUtil:
+    """Utility class for formatting Markdown tables.
+
+    This class standardizes Markdown table formatting by:
+    - Normalizing column alignment markers (e.g., :---, :---:, ---:)
+    - Adding consistent spacing around pipes (|)
+    - Preserving indentation levels
+    - Handling both header rows and data rows
+
+    Example:
+        Input:  |姓名|年龄|城市|
+                |:---|---:|:---:|
+                |张三|25|北京|
+
+        Output: | 姓名 | 年龄 | 城市 |
+                | :--- | ---: | :---: |
+                | 张三 | 25 | 北京 |
+    """
+
+    @staticmethod
+    def _split_row_cells(row_line: str) -> List[str]:
+        """Split a markdown table row into cells, preserving empty cells."""
+        inner = row_line.strip()
+        if not inner.startswith("|"):
+            return []
+        parts = inner.split("|")
+        if parts and parts[0].strip() == "":
+            parts = parts[1:]
+        if parts and parts[-1].strip() == "":
+            parts = parts[:-1]
+        return [part.strip() for part in parts]
+
+    @staticmethod
+    def _is_table_row(line: str) -> bool:
+        stripped = line.strip()
+        return stripped.startswith("|") and "|" in stripped[1:]
+
+    @classmethod
+    def _is_separator_row(cls, line: str) -> bool:
+        cells = cls._split_row_cells(line)
+        return bool(cells) and all(_SEPARATOR_CELL.match(cell) for cell in cells)
+
+    @classmethod
+    def _is_alignment_row(cls, line: str) -> bool:
+        """Detect a GFM alignment row, matching the previous ``align_pattern``.
+
+        Deliberately looser than :meth:`_is_separator_row` (which needs three
+        or more dashes per cell) so formatting output is unchanged for rows
+        like ``| - | - |``.
+        """
+        cells = cls._split_row_cells(line)
+        return bool(cells) and all(_ALIGNMENT_CELL.match(cell) for cell in cells)
+
+    @classmethod
+    def _is_empty_row(cls, line: str) -> bool:
+        cells = cls._split_row_cells(line)
+        return bool(cells) and all(cell == "" for cell in cells)
+
+    @classmethod
+    def _separator_row_for(cls, header_line: str) -> str:
+        cells = cls._split_row_cells(header_line)
+        return "| " + " | ".join("---" for _ in cells) + " |"
+
+    @classmethod
+    def _normalize_table_block(cls, block: List[str]) -> List[str]:
+        """Fix MarkItDown-style tables: drop bogus prefix rows, ensure GFM delimiter."""
+        while block and cls._is_empty_row(block[0]):
+            block.pop(0)
+        if block and cls._is_separator_row(block[0]):
+            block.pop(0)
+        # GFM/marked need "| --- |" after the first row. Headerless Word tables
+        # only have data rows after we strip the fake empty+separator prefix.
+        if len(block) >= 2 and not cls._is_separator_row(block[1]):
+            sep = cls._separator_row_for(block[0])
+            block = [block[0], sep] + block[1:]
+        return block
+
+    def normalize_spurious_table_prefixes(self, content: str) -> str:
+        """Remove bogus empty/separator prefix rows from MarkItDown table output."""
+        lines = content.split("\n")
+        out: List[str] = []
+        i = 0
+        while i < len(lines):
+            line = lines[i]
+            if not self._is_table_row(line):
+                out.append(line)
+                i += 1
+                continue
+            block: List[str] = []
+            while i < len(lines) and self._is_table_row(lines[i]):
+                block.append(lines[i])
+                i += 1
+            out.extend(self._normalize_table_block(block))
+        return "\n".join(out)
+
+    def format_table(self, content: str) -> str:
+        """Format all Markdown tables in the content.
+
+        Line-based and regex-free: only well-formed rows (leading *and*
+        trailing ``|``, with at least one cell) are treated as tables, so a
+        malformed row can never trigger the catastrophic backtracking
+        described in https://github.com/Tencent/WeKnora/issues/2768.
+        CRLF/CR input is normalized to LF before scanning.
+
+        Args:
+            content: Raw Markdown text containing tables
+
+        Returns:
+            Formatted Markdown text with standardized table formatting
+        """
+
+        def format_alignment_row(prefix: str, columns: List[str]) -> str:
+            """Standardize an alignment row, preserving ``:`` markers."""
+            processed = []
+            for col in columns:
+                # Preserve left alignment marker (:---)
+                left_colon = ":" if col.startswith(":") else ""
+                # Preserve right alignment marker (---:)
+                right_colon = ":" if col.endswith(":") else ""
+                processed.append(left_colon + "---" + right_colon)
+            return prefix + "| " + " | ".join(processed) + " |"
+
+        def format_data_row(prefix: str, columns: List[str]) -> str:
+            """Standardize a header or data row."""
+            return prefix + "| " + " | ".join(columns) + " |"
+
+        # Unify CRLF/CR first so a trailing ``\r`` left by split("\n") cannot
+        # be mistaken for "ends with |" via str.rstrip(), and so we do not
+        # emit mixed line endings. Trailing newlines are preserved.
+        unified = content.replace("\r\n", "\n").replace("\r", "\n")
+        lines = unified.split("\n")
+        out: List[str] = []
+        for line in lines:
+            stripped = line.lstrip(" \t")
+            # A row must both begin and end with a pipe to be a table row;
+            # anything else is passed through unchanged. This also keeps
+            # unclosed rows (the old regex's exponential worst case) cheap.
+            # rstrip only spaces/tabs (old ``\|[\t ]*$``), not all whitespace.
+            if stripped.startswith("|") and line.rstrip(" \t").endswith("|"):
+                prefix = line[: len(line) - len(stripped)]
+                columns = self._split_row_cells(line)
+                # Lone "|" is not a table row: the old line_pattern required
+                # opening *and* closing pipes. Formatting it to "|  |" would
+                # let normalize_spurious_table_prefixes drop the line.
+                if not columns:
+                    out.append(line)
+                elif self._is_alignment_row(line):
+                    out.append(format_alignment_row(prefix, columns))
+                else:
+                    out.append(format_data_row(prefix, columns))
+            else:
+                out.append(line)
+        return self.normalize_spurious_table_prefixes("\n".join(out))
+
+    @staticmethod
+    def _self_test():
+        test_content = """
+# 测试表格
+普通文本---不会被匹配
+
+## 表格1（无前置空格）
+
+| 姓名   | 年龄  | 城市          |
+|      :---------- | -------: | :------      |
+| 张三 | 25 | 北京 |
+
+## 表格3（前置4个空格+首尾|）
+    |   产品   |   价格   |   库存   |
+    | :-------------: | ----------- | :-----------: |
+    | 手机 | 5999       | 100 |
+"""
+        util = MarkdownTableUtil()
+        format_content = util.format_table(test_content)
+        print(format_content)
+
+
+class MarkdownTableFormatter(BaseParser):
+    """Parser for formatting Markdown tables.
+
+    This parser standardizes the formatting of all Markdown tables in the
+    document to ensure consistent spacing and alignment markers.
+
+    Example:
+        >>> formatter = MarkdownTableFormatter()
+        >>> content = b"|Name|Age|\n|---|---|\n|John|30|"
+        >>> doc = formatter.parse_into_text(content)
+        >>> print(doc.content)
+        | Name | Age |
+        | --- | --- |
+        | John | 30 |
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.table_helper = MarkdownTableUtil()
+
+    def parse_into_text(self, content: bytes) -> Document:
+        """Parse and format Markdown tables.
+
+        Args:
+            content: Raw Markdown content as bytes
+
+        Returns:
+            Document with formatted table content
+        """
+        # Decode bytes to string with automatic encoding detection
+        text = endecode.decode_bytes(content)
+        # Format all tables in the content
+        text = self.table_helper.format_table(text)
+        return Document(content=text)
+
+
+class MarkdownImageUtil:
+    """Utility class for handling images in Markdown.
+
+    This class provides functionality to:
+    - Extract base64-encoded images from Markdown
+    - Extract image paths from Markdown
+    - Replace image paths with new URLs
+    - Convert base64 images to binary format
+
+    Supported formats:
+    - Base64 embedded images: ![alt](data:image/png;base64,iVBORw0...)
+    - Regular image links: ![alt](path/to/image.png)
+    """
+
+    def __init__(self):
+        # Pattern to match base64 embedded images
+        # Captures: (1) alt text, (2) image format, (3) base64 data
+        # Alt text uses .*? (non-greedy) to allow literal ] (e.g. Windows paths).
+        # MIME subtype uses [^;]+ to handle types with hyphens like x-emf.
+        self.b64_pattern = re.compile(
+            r"!\[(.*?)\]\(data:image/([^;]+);base64,([^\)]+)\)"
+        )
+        # Pattern to match regular image syntax (alt text allows ])
+        self.image_pattern = re.compile(r"!\[(.*?)\]\(([^)]+)\)")
+        # Pattern for replacing image paths
+        self.replace_pattern = re.compile(r"!\[(.*?)\]\(([^)]+)\)")
+
+    def extract_image(
+        self,
+        content: str,
+        path_prefix: Optional[str] = None,
+        replace: bool = True,
+    ) -> Tuple[str, List[str]]:
+        """Extract image paths from Markdown content.
+
+        Args:
+            content: Markdown text containing images
+            path_prefix: Optional prefix to add to image paths
+            replace: Whether to replace image syntax in content
+
+        Returns:
+            Tuple of (processed_text, list_of_image_paths)
+
+        Example:
+            >>> util = MarkdownImageUtil()
+            >>> text, images = util.extract_image("![logo](img/logo.png)")
+            >>> print(images)
+            ['img/logo.png']
+        """
+        # List to store extracted image paths
+        images: List[str] = []
+
+        def repl(match: Match[str]) -> str:
+            """Replacement function for each image match."""
+            title = match.group(1)  # Alt text
+            image_path = match.group(2)  # Image path
+
+            # Add prefix if specified
+            if path_prefix:
+                image_path = f"{path_prefix}/{image_path}"
+
+            images.append(image_path)
+
+            # Keep original if replace is False
+            if not replace:
+                return match.group(0)
+
+            # Replace image path with potentially prefixed path
+            return f"![{title}]({image_path})"
+
+        text = self.image_pattern.sub(repl, content)
+        logger.debug(f"Extracted {len(images)} images from markdown")
+        return text, images
+
+    def extract_base64(
+        self,
+        content: str,
+        path_prefix: Optional[str] = None,
+        replace: bool = True,
+    ) -> Tuple[str, Dict[str, bytes]]:
+        """Extract and decode base64 embedded images from Markdown.
+
+        This method finds all base64-encoded images in the Markdown content,
+        decodes them to binary format, generates unique filenames, and
+        optionally replaces them with file path references.
+
+        Args:
+            content: Markdown text containing base64 images
+            path_prefix: Optional directory prefix for generated paths
+            replace: Whether to replace base64 syntax with file paths
+
+        Returns:
+            Tuple of (processed_text, dict_of_path_to_bytes)
+
+        Example:
+            >>> util = MarkdownImageUtil()
+            >>> text = "![logo](data:image/png;base64,iVBORw0KGg...)"
+            >>> new_text, images = util.extract_base64(text, "images")
+            >>> print(new_text)
+            ![logo](images/uuid.png)
+            >>> print(len(images))
+            1
+        """
+        # Dictionary mapping generated file paths to binary image data
+        images: Dict[str, bytes] = {}
+
+        def repl(match: Match[str]) -> str:
+            """Replacement function for each base64 image match."""
+            title = match.group(1)  # Alt text
+            img_ext = match.group(2)  # Image format (png, jpg, etc.)
+            img_b64 = match.group(3)  # Base64 encoded data
+
+            # Decode base64 string to bytes
+            image_byte = endecode.encode_image(img_b64, errors="ignore")
+            if not image_byte:
+                logger.error(f"Failed to decode base64 image skip it: {img_b64}")
+                return title  # Return just the alt text if decode fails
+
+            # Generate unique filename with original extension
+            image_path = f"{uuid.uuid4()}.{img_ext}"
+            if path_prefix:
+                image_path = f"{path_prefix}/{image_path}"
+            images[image_path] = image_byte
+
+            # Keep original base64 if replace is False
+            if not replace:
+                return match.group(0)
+
+            # Replace base64 data with file path reference
+            return f"![{title}]({image_path})"
+
+        text = self.b64_pattern.sub(repl, content)
+        logger.debug(f"Extracted {len(images)} base64 images from markdown")
+        return text, images
+
+    def replace_path(self, content: str, images: Dict[str, str]) -> str:
+        """Replace image paths in Markdown with new URLs.
+
+        This method is typically used to replace local file paths with
+        uploaded URLs after images have been stored.
+
+        Args:
+            content: Markdown text with image references
+            images: Mapping of old paths to new URLs
+
+        Returns:
+            Markdown text with updated image URLs
+
+        Example:
+            >>> util = MarkdownImageUtil()
+            >>> content = "![logo](temp/img.png)"
+            >>> mapping = {"temp/img.png": "https://cdn.com/img.png"}
+            >>> result = util.replace_path(content, mapping)
+            >>> print(result)
+            ![logo](https://cdn.com/img.png)
+        """
+        # Track which paths were actually replaced
+        content_replace: set = set()
+
+        def repl(match: Match[str]) -> str:
+            """Replacement function for each image match."""
+            title = match.group(1)  # Alt text
+            image_path = match.group(2)  # Current image path
+
+            # Only replace if path exists in mapping
+            if image_path not in images:
+                return match.group(0)  # Keep original
+
+            content_replace.add(image_path)
+            # Get new URL from mapping
+            image_path = images[image_path]
+            return f"![{title}]({image_path})" if image_path else title
+
+        text = self.replace_pattern.sub(repl, content)
+        logger.debug(f"Replaced {len(content_replace)} images in markdown")
+        return text
+
+    @staticmethod
+    def _self_test():
+        your_content = "test![](data:image/png;base64,iVBORw0KGgoAAAA)test"
+        image_handle = MarkdownImageUtil()
+        text, images = image_handle.extract_base64(your_content)
+        print(text)
+
+        for image_url, image_byte in images.items():
+            with open(image_url, "wb") as f:
+                f.write(image_byte)
+
+
+class MarkdownImageBase64(BaseParser):
+    """Parser for extracting base64 images from Markdown.
+
+    Extracts base64-encoded images, replaces them with path references,
+    and returns the raw image data in Document.images for the Go-side
+    ImageResolver (or main.py _resolve_images) to handle storage.
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.image_helper = MarkdownImageUtil()
+
+    def parse_into_text(self, content: bytes) -> Document:
+        text = endecode.decode_bytes(content)
+        text, img_b64 = self.image_helper.extract_base64(text, path_prefix="images")
+
+        images: Dict[str, str] = {}
+        for ipath, raw_bytes in img_b64.items():
+            images[ipath] = base64.b64encode(raw_bytes).decode()
+
+        logger.debug("Extracted %d base64 images from markdown", len(images))
+        return Document(content=text, images=images)
+
+
+class MarkdownParser(PipelineParser):
+    """Complete Markdown parser using pipeline approach.
+
+    This parser processes Markdown content through multiple stages:
+    1. MarkdownTableFormatter: Standardizes table formatting
+    2. MarkdownImageBase64: Extracts and uploads base64 images
+
+    The pipeline ensures that content flows through each parser in sequence,
+    with each stage's output becoming the next stage's input.
+    """
+
+    _parser_cls = (MarkdownTableFormatter, MarkdownImageBase64)
+
+
+if __name__ == "__main__":
+    # Example usage and testing
+    logging.basicConfig(level=logging.DEBUG)
+
+    # Test the complete MarkdownParser pipeline
+    your_content = "test![](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgA)test"
+    parser = MarkdownParser()
+
+    # Parse content and display results
+    document = parser.parse_into_text(your_content.encode())
+    logger.info(document.content)
+    logger.info(f"Images: {len(document.images)}, name: {document.images.keys()}")
+
+    # Run individual utility tests
+    MarkdownImageUtil._self_test()
+    MarkdownTableUtil._self_test()

docreader/parser/markitdown_parser.py

@@ -0,0 +1,83 @@
+import io
+import logging
+
+from markitdown import MarkItDown
+
+from docreader.config import CONFIG
+from docreader.models.document import Document
+from docreader.parser.base_parser import BaseParser
+from docreader.parser.chain_parser import PipelineParser
+from docreader.parser.concurrency import parser_worker_limit
+from docreader.parser.docx_merge import fill_vertical_merged_cells_docx
+from docreader.parser.markdown_parser import MarkdownParser
+from docreader.parser.ppt_convert import normalize_ppt_bytes
+from docreader.parser.pptx_media import (
+    attach_pptx_media_to_markdown,
+    markdown_needs_pptx_media_attach,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class StdMarkitdownParser(BaseParser):
+    """
+    Standard MarkItDown Parser Wrapper
+
+    This parser uses the markitdown library to convert various document formats
+    (docx, pptx, pdf, etc.) into text/markdown.
+    """
+
+    def __init__(self, *args, **kwargs):
+        # 这里的 super() 会调用 BaseParser 的初始化，确保 self.file_type 被正确赋值
+        super().__init__(*args, **kwargs)
+        self.markitdown = MarkItDown()
+
+    def parse_into_text(self, content: bytes) -> Document:
+        """
+        Parses content using MarkItDown.
+        Uses self.file_type (inherited from BaseParser) to hint the stream format.
+        """
+        ext = self.file_type
+        ft = (ext or "").lstrip(".").lower()
+        pptx_bytes: bytes | None = None
+        if ft in ("ppt", "pptx"):
+            content, ext = normalize_ppt_bytes(content, ft)
+            pptx_bytes = content
+            ft = "pptx"
+        elif ft == "docx":
+            content = fill_vertical_merged_cells_docx(content)
+        if ext and not ext.startswith("."):
+            ext = "." + ext
+
+        with parser_worker_limit("markitdown", CONFIG.markitdown_max_workers):
+            result = self._convert_markitdown(content, ext, keep_data_uris=True)
+            if result is None:
+                logger.warning(
+                    "MarkItDown failed with embedded images for %s; retrying without data URIs",
+                    ft or ext,
+                )
+                result = self._convert_markitdown(content, ext, keep_data_uris=False)
+
+        text = result.text_content
+        images: dict[str, str] = {}
+        if pptx_bytes is not None and markdown_needs_pptx_media_attach(text):
+            text, images = attach_pptx_media_to_markdown(text, pptx_bytes)
+        return Document(content=text, images=images)
+
+    def _convert_markitdown(
+        self, content: bytes, ext: str | None, *, keep_data_uris: bool
+    ):
+        try:
+            return self.markitdown.convert(
+                io.BytesIO(content),
+                file_extension=ext,
+                keep_data_uris=keep_data_uris,
+            )
+        except Exception:
+            if keep_data_uris:
+                return None
+            raise
+
+
+class MarkitdownParser(PipelineParser):
+    _parser_cls = (StdMarkitdownParser, MarkdownParser)

docreader/parser/mhtml_parser.py

@@ -0,0 +1,395 @@
+"""MHTML parser.
+
+Parses MIME HTML web archives into markdown text and optional embedded images.
+"""
+
+import base64
+import email
+import html
+import logging
+import os
+from email.header import decode_header
+from urllib.parse import unquote, urljoin, urlparse
+import uuid
+from typing import Dict
+
+from bs4 import BeautifulSoup
+
+from docreader.models.document import Document
+from docreader.parser.base_parser import BaseParser
+
+logger = logging.getLogger(__name__)
+
+_AD_DOMAINS = (
+    "googleads",
+    "doubleclick",
+    "googlesyndication",
+    "facebook.com/tr",
+    "analytics",
+    "pixel",
+)
+
+_UNKNOWN_8BIT = frozenset({"unknown-8bit", "unknown"})
+
+
+def _header_str(value) -> str:
+    """Normalize a MIME header to str, recovering 8-bit UTF-8 bytes.
+
+    ``email.message_from_bytes`` uses compat32 by default. Browser-saved
+    ``.mhtml`` files often store raw UTF-8 in headers such as Content-Location.
+    Those values come back as ``email.header.Header`` with charset
+    ``unknown-8bit``: ``.strip()`` / ``.lower()`` raise AttributeError, and
+    ``str(Header)`` replaces the bytes with U+FFFD so image aliases no longer
+    match the HTML body.
+    """
+    if value is None:
+        return ""
+    if isinstance(value, bytes):
+        return value.decode("utf-8", errors="replace")
+    if isinstance(value, str):
+        try:
+            return value.encode("ascii", "surrogateescape").decode("utf-8")
+        except UnicodeError:
+            return value
+    try:
+        chunks = decode_header(value)
+    except (TypeError, ValueError, LookupError, UnicodeError):
+        return str(value)
+
+    parts: list[str] = []
+    for chunk, charset in chunks:
+        if isinstance(chunk, bytes):
+            encoding = (charset or "utf-8").lower()
+            if encoding in _UNKNOWN_8BIT:
+                encoding = "utf-8"
+            try:
+                parts.append(chunk.decode(encoding, errors="replace"))
+            except LookupError:
+                parts.append(chunk.decode("utf-8", errors="replace"))
+        elif chunk:
+            try:
+                parts.append(
+                    chunk.encode("ascii", "surrogateescape").decode("utf-8")
+                )
+            except UnicodeError:
+                parts.append(chunk)
+    return "".join(parts)
+
+
+class MHTMLParser(BaseParser):
+    """Parser for MHTML web archives."""
+
+    def __init__(self, *args, extract_images: bool = True, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.extract_images = extract_images
+
+    def parse_into_text(self, content: bytes) -> Document:
+        logger.info(
+            "Parsing MHTML file: %s, size: %d bytes", self.file_name, len(content)
+        )
+        msg = email.message_from_bytes(content)
+
+        html_parts = []
+        images: Dict[str, str] = {}
+        image_aliases: Dict[str, str] = {}
+        metadata: Dict[str, object] = {}
+
+        for part in msg.walk():
+            content_type = part.get_content_type()
+            location = _header_str(part.get("Content-Location", ""))
+
+            if content_type == "text/html":
+                payload = part.get_payload(decode=True)
+                if not payload:
+                    continue
+                charset = part.get_content_charset() or "utf-8"
+                try:
+                    html_text = payload.decode(charset, errors="ignore")
+                except LookupError:
+                    html_text = payload.decode("utf-8", errors="ignore")
+                html_parts.append(
+                    {
+                        "content": html_text,
+                        "location": location,
+                        "size": len(html_text),
+                    }
+                )
+            elif content_type.startswith("image/") and self.extract_images:
+                image_data = part.get_payload(decode=True)
+                if image_data:
+                    image_path = self._image_path_for_part(part, content_type, images)
+                    images[image_path] = base64.b64encode(image_data).decode("utf-8")
+                    self._add_image_aliases(image_aliases, part, image_path)
+
+        main_html = self._select_main_html(html_parts)
+        if not main_html:
+            logger.warning("No HTML content found in MHTML file")
+            return Document(
+                content="", images=images, metadata={"source_format": "mhtml"}
+            )
+        html_content = main_html["content"]
+
+        try:
+            markdown_text = self.html_to_markdown(
+                html_content,
+                image_aliases=image_aliases,
+                base_location=main_html.get("location", ""),
+            )
+        except Exception as e:
+            logger.error("Failed to convert HTML to Markdown: %s", e)
+            markdown_text = f"```html\n{html_content}\n```"
+
+        metadata["source_format"] = "mhtml"
+        metadata["file_size"] = len(content)
+        metadata["image_count"] = len(images)
+        return Document(content=markdown_text, images=images, metadata=metadata)
+
+    def _select_main_html(self, html_parts) -> dict:
+        """Pick the largest non-ad HTML part as the main document body."""
+        if not html_parts:
+            return {}
+
+        def is_ad(location: str) -> bool:
+            loc = _header_str(location)
+            if not loc:
+                return False
+            loc = loc.lower()
+            return any(ad in loc for ad in _AD_DOMAINS)
+
+        non_ad = sorted(
+            (part for part in html_parts if not is_ad(part.get("location", ""))),
+            key=lambda part: part["size"],
+            reverse=True,
+        )
+        if non_ad:
+            logger.info("Selected main HTML: %d bytes", non_ad[0]["size"])
+            return non_ad[0]
+
+        largest = max(html_parts, key=lambda part: part["size"])
+        logger.warning(
+            "Only ad content found, using largest: %d bytes", largest["size"]
+        )
+        return largest
+
+    @staticmethod
+    def _add_image_aliases(
+        image_aliases: Dict[str, str], part, image_path: str
+    ) -> None:
+        """Register the refs an MHTML document may use for an image part."""
+        for raw in (
+            part.get("Content-Location", ""),
+            part.get("Content-ID", ""),
+            part.get("X-Attachment-Id", ""),
+        ):
+            raw = _header_str(raw).strip()
+            if not raw:
+                continue
+            values = {raw, html.unescape(raw), unquote(html.unescape(raw))}
+            cid = raw.strip("<>")
+            if cid:
+                values.add(f"cid:{cid}")
+                values.add(f"cid:{unquote(cid)}")
+            for value in values:
+                if value:
+                    image_aliases[value] = image_path
+
+    @staticmethod
+    def _image_extension(content_type: str) -> str:
+        return {
+            "image/png": ".png",
+            "image/jpeg": ".jpg",
+            "image/gif": ".gif",
+            "image/webp": ".webp",
+            "image/bmp": ".bmp",
+            "image/tiff": ".tiff",
+            "image/x-icon": ".ico",
+        }.get(content_type, ".png")
+
+    @classmethod
+    def _image_path_for_part(
+        cls, part, content_type: str, images: Dict[str, str]
+    ) -> str:
+        """Choose a stable image path when the MHTML part exposes a filename."""
+        ext = cls._image_extension(content_type)
+        location = _header_str(part.get("Content-Location", "")).strip()
+        filename = cls._filename_from_content_location(location)
+        if not filename:
+            return f"images/{uuid.uuid4().hex}{ext}"
+
+        stem, location_ext = os.path.splitext(filename)
+        if not location_ext:
+            filename = f"{filename}{ext}"
+        image_path = f"images/{filename}"
+        if image_path not in images:
+            return image_path
+
+        suffix = 2
+        stem, location_ext = os.path.splitext(filename)
+        while True:
+            candidate = f"images/{stem}_{suffix}{location_ext}"
+            if candidate not in images:
+                return candidate
+            suffix += 1
+
+    @staticmethod
+    def _filename_from_content_location(location: str) -> str:
+        decoded = unquote(html.unescape(_header_str(location).strip()))
+        if not decoded or decoded.lower().startswith("cid:"):
+            return ""
+        path = urlparse(decoded).path or decoded
+        filename = os.path.basename(path)
+        if not filename or filename in {".", ".."}:
+            return ""
+        if "/" in filename or "\\" in filename:
+            return ""
+        return filename
+
+    def html_to_markdown(
+        self,
+        html_content: str,
+        image_aliases: Dict[str, str] | None = None,
+        base_location: str = "",
+        *,
+        strip_internal_links: bool = True,
+        fallback_to_raw_html: bool = True,
+    ) -> str:
+        """Convert HTML to Markdown with explicit link and fallback policies."""
+
+        def raw_html_fallback() -> str:
+            if not fallback_to_raw_html:
+                return ""
+            return f"```html\n{html_content[:50000]}\n```"
+
+        try:
+            from markdownify import markdownify as md
+
+            soup = BeautifulSoup(html_content, "lxml")
+            for tag in soup(["script", "style", "noscript", "iframe"]):
+                tag.decompose()
+            if strip_internal_links:
+                self._strip_internal_links(soup)
+            if image_aliases:
+                self._rewrite_image_sources(soup, image_aliases, base_location)
+            text_fallback = soup.get_text(separator="\n", strip=True)
+            markdown_text = md(str(soup), heading_style="ATX")
+            result = self._normalize_markdown(markdown_text)
+            if not result and text_fallback:
+                logger.warning("Markdown empty, falling back to text extraction")
+                return text_fallback
+            if not result:
+                return raw_html_fallback()
+            return result
+        except ImportError:
+            logger.warning("markdownify not available, returning raw HTML")
+            return raw_html_fallback()
+        except Exception as e:
+            logger.error("HTML to Markdown conversion failed: %s", e)
+            return raw_html_fallback()
+
+    def _html_to_markdown(
+        self,
+        html_content: str,
+        image_aliases: Dict[str, str] | None = None,
+        base_location: str = "",
+    ) -> str:
+        """Backward-compatible wrapper for existing internal callers and tests."""
+        return self.html_to_markdown(html_content, image_aliases, base_location)
+
+    @staticmethod
+    def _normalize_markdown(markdown_text: str) -> str:
+        text = markdown_text.replace("\r\n", "\n").replace("\r", "\n")
+        output: list[str] = []
+        pending_blank = False
+        fence_char: str | None = None
+        fence_len = 0
+
+        for line in text.split("\n"):
+            if fence_char is not None:
+                output.append(line)
+                if MHTMLParser._is_closing_fence(line, fence_char, fence_len):
+                    fence_char = None
+                    fence_len = 0
+                continue
+
+            opening = MHTMLParser._opening_fence(line)
+            if opening is not None:
+                if pending_blank and output:
+                    output.append("")
+                pending_blank = False
+                output.append(line)
+                fence_char, fence_len = opening
+                continue
+
+            if not line.strip(" \t"):
+                pending_blank = True
+                continue
+
+            if pending_blank and output:
+                output.append("")
+            pending_blank = False
+
+            trailing_spaces = len(line) - len(line.rstrip(" "))
+            if trailing_spaces >= 2:
+                line = line.rstrip(" \t") + "  "
+            else:
+                line = line.rstrip(" \t")
+            output.append(line)
+
+        return "\n".join(output).strip("\n")
+
+    @staticmethod
+    def _opening_fence(line: str) -> tuple[str, int] | None:
+        stripped = line.lstrip(" ")
+        if len(line) - len(stripped) > 3 or not stripped:
+            return None
+        fence_char = stripped[0]
+        if fence_char not in {"`", "~"}:
+            return None
+        fence_len = len(stripped) - len(stripped.lstrip(fence_char))
+        if fence_len < 3:
+            return None
+        return fence_char, fence_len
+
+    @staticmethod
+    def _is_closing_fence(line: str, fence_char: str, fence_len: int) -> bool:
+        stripped = line.lstrip(" ")
+        if len(line) - len(stripped) > 3:
+            return False
+        closing_len = len(stripped) - len(stripped.lstrip(fence_char))
+        if closing_len < fence_len:
+            return False
+        return not stripped[closing_len:].strip(" \t")
+
+    @staticmethod
+    def _strip_internal_links(soup: BeautifulSoup) -> None:
+        """Unwrap links that don't point to an external resource."""
+        external = ("http://", "https://", "mailto:", "tel:")
+        for link in soup.find_all("a"):
+            href = (link.get("href") or "").strip().lower()
+            if not href or not href.startswith(external):
+                link.unwrap()
+
+    @staticmethod
+    def _rewrite_image_sources(
+        soup: BeautifulSoup,
+        image_aliases: Dict[str, str],
+        base_location: str = "",
+    ) -> None:
+        for img in soup.find_all("img"):
+            src = (img.get("src") or "").strip()
+            if not src:
+                continue
+            candidates = [
+                src,
+                html.unescape(src),
+                unquote(html.unescape(src)),
+            ]
+            if base_location:
+                candidates.append(urljoin(base_location, src))
+            base_name = os.path.basename(unquote(html.unescape(src)))
+            if base_name:
+                candidates.append(base_name)
+            for candidate in candidates:
+                if candidate in image_aliases:
+                    img["src"] = image_aliases[candidate]
+                    break

docreader/parser/opendataloader_parser.py

@@ -0,0 +1,378 @@
+"""PDF parser backed by OpenDataLoader PDF (Apache-2.0).
+
+Requires Java 11+ on PATH and the ``opendataloader-pdf`` Python package.
+Each ``convert()`` spawns a JVM; concurrency is limited via
+``DOCREADER_ODL_MAX_WORKERS``.
+
+Hybrid mode (``docling-fast``, etc.) needs a running
+``opendataloader-pdf-hybrid`` server — configure ``DOCREADER_ODL_HYBRID_URL``.
+"""
+
+from __future__ import annotations
+
+import base64
+import html
+import logging
+import os
+import re
+import shutil
+import tempfile
+import urllib.error
+import urllib.request
+from typing import Any, Dict, Mapping, Optional, Tuple
+
+from docreader.config import CONFIG
+from docreader.models.document import Document
+from docreader.parser.base_parser import BaseParser
+from docreader.parser.concurrency import parser_worker_limit
+from docreader.utils.ssrf import is_ssrf_safe_url
+
+logger = logging.getLogger(__name__)
+
+_MIN_CHARS_PER_PAGE = 20
+_IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")
+_MD_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
+_IMAGE_FILE_NUM_RE = re.compile(r"^imageFile(\d+)\.", re.I)
+
+
+class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
+    """Health probes never need redirects; refusing them closes redirect SSRF."""
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        return None
+
+
+def _override_str(overrides: Optional[Mapping[str, Any]], key: str, default: str = "") -> str:
+    if overrides:
+        v = overrides.get(key)
+        if v is not None and str(v).strip() != "":
+            return str(v).strip()
+    return default
+
+
+def _override_bool(overrides: Optional[Mapping[str, Any]], key: str, default: bool) -> bool:
+    if overrides:
+        v = overrides.get(key)
+        if v is not None and str(v).strip() != "":
+            return str(v).strip().lower() in {"1", "true", "yes", "y", "on"}
+    return default
+
+
+def _java_available() -> Tuple[bool, str]:
+    if not shutil.which("java"):
+        return False, "需要 Java 11+（JRE），请安装并在 PATH 中配置 java"
+    return True, ""
+
+
+def _package_available() -> Tuple[bool, str]:
+    try:
+        import opendataloader_pdf  # noqa: F401
+    except ImportError as e:
+        return False, f"opendataloader-pdf 未安装: {e}"
+    return True, ""
+
+
+def _ping_hybrid(
+    url: str,
+    *,
+    timeout_sec: float = 5.0,
+    retries: int = 3,
+    retry_delay_sec: float = 2.0,
+) -> Tuple[bool, str]:
+    import time
+
+    base = url.rstrip("/")
+    health_url = f"{base}/health"
+    safe, reason = is_ssrf_safe_url(health_url)
+    if not safe:
+        return False, f"OpenDataLoader hybrid URL 被 SSRF 防护拦截: {reason}"
+
+    opener = urllib.request.build_opener(_NoRedirectHandler())
+    last_err = ""
+    for attempt in range(max(1, retries)):
+        try:
+            req = urllib.request.Request(health_url, method="GET")
+            with opener.open(req, timeout=timeout_sec) as resp:
+                if 200 <= resp.status < 300:
+                    return True, ""
+                last_err = f"hybrid 健康检查 HTTP {resp.status}: {health_url}"
+        except urllib.error.URLError as e:
+            last_err = f"无法连接 OpenDataLoader hybrid 服务 ({health_url}): {e}"
+        except Exception as e:
+            last_err = f"hybrid 健康检查失败: {e}"
+        if attempt + 1 < retries:
+            time.sleep(retry_delay_sec)
+    hint = (
+        "；若刚执行 make dev-start --odl-hybrid，请等待镜像构建/服务就绪"
+        "（docker logs WeKnora-odl-hybrid）"
+    )
+    return False, last_err + hint
+
+
+def opendataloader_available(
+    overrides: Optional[Mapping[str, Any]] = None,
+    quick: bool = False,
+) -> Tuple[bool, str]:
+    """Registry / ListEngines availability probe.
+
+    When ``quick`` is set the hybrid health check uses a single short-timeout
+    attempt with no retries, so it can be used for fast status listing without
+    blocking on long retry/backoff loops. Parsing keeps the patient retry
+    behavior to tolerate a hybrid service that is still starting up.
+    """
+    ok, msg = _java_available()
+    if not ok:
+        return False, msg
+    ok, msg = _package_available()
+    if not ok:
+        return False, msg
+
+    hybrid = _resolve_hybrid(overrides)
+    if hybrid and hybrid.lower() not in ("off", ""):
+        url = _resolve_hybrid_url(overrides)
+        if url:
+            if quick:
+                return _ping_hybrid(url, retries=1, timeout_sec=2.0)
+            return _ping_hybrid(url, retries=6, retry_delay_sec=5.0, timeout_sec=5.0)
+    return True, ""
+
+
+def _resolve_hybrid(overrides: Optional[Mapping[str, Any]] = None) -> str:
+    return _override_str(overrides, "odl_hybrid", CONFIG.odl_hybrid)
+
+
+def _resolve_hybrid_url(overrides: Optional[Mapping[str, Any]] = None) -> str:
+    return _override_str(overrides, "odl_hybrid_url", CONFIG.odl_hybrid_url)
+
+
+def _find_markdown_file(output_dir: str, pdf_stem: str) -> str:
+    candidates = []
+    for root, _, files in os.walk(output_dir):
+        for name in files:
+            if name.lower().endswith(".md"):
+                path = os.path.join(root, name)
+                candidates.append(path)
+    if not candidates:
+        raise FileNotFoundError(f"OpenDataLoader 未在 {output_dir} 生成 markdown 文件")
+    for path in candidates:
+        base = os.path.splitext(os.path.basename(path))[0]
+        if base == pdf_stem or base.startswith(pdf_stem):
+            return path
+    candidates.sort(key=lambda p: os.path.getmtime(p), reverse=True)
+    return candidates[0]
+
+
+def _normalize_odl_image_url(raw: str) -> str:
+    """OpenDataLoader wraps paths as ``<images/foo.png>``; storage may HTML-escape them."""
+    s = html.unescape((raw or "").strip())
+    s = s.replace("&lt;", "<").replace("&gt;", ">").replace("&amp;", "&")
+    s = s.strip().strip("<>").strip().strip('"').strip("'")
+    if s.startswith("./"):
+        s = s[2:]
+    return s.replace("\\", "/")
+
+
+def _canonical_image_ref(abs_path: str, output_dir: str) -> str:
+    """Use ``images/<file>`` keys to match OpenDataLoader markdown conventions."""
+    rel = os.path.relpath(abs_path, output_dir).replace("\\", "/")
+    name = os.path.basename(abs_path)
+    if rel.startswith("images/"):
+        return rel
+    return f"images/{name}"
+
+
+def _collect_images_under_output(output_dir: str) -> Dict[str, str]:
+    """Collect every extracted image under the convert output tree."""
+    images: Dict[str, str] = {}
+    for root, _, files in os.walk(output_dir):
+        for name in files:
+            if not name.lower().endswith(_IMAGE_SUFFIXES):
+                continue
+            abs_path = os.path.join(root, name)
+            ref = _canonical_image_ref(abs_path, output_dir)
+            if ref in images:
+                continue
+            with open(abs_path, "rb") as f:
+                images[ref] = base64.b64encode(f.read()).decode("utf-8")
+    return images
+
+
+def _register_image_alias(aliases: Dict[str, str], alias: str, canonical: str) -> None:
+    key = _normalize_odl_image_url(alias)
+    if key:
+        aliases[key] = canonical
+
+
+def _build_path_alias_map(images: Dict[str, str]) -> Dict[str, str]:
+    """Map ODL markdown spellings (angle brackets, entities, basenames) to dict keys."""
+    aliases: Dict[str, str] = {}
+    for ref in images:
+        base = os.path.basename(ref)
+        variants = [
+            ref,
+            base,
+            f"images/{base}",
+            f"<{ref}>",
+            f"<images/{base}>",
+            f"&lt;{ref}&gt;",
+            f"&lt;images/{base}&gt;",
+        ]
+        for variant in variants:
+            _register_image_alias(aliases, variant, ref)
+    return aliases
+
+
+def _resolve_image_ref(url: str, aliases: Dict[str, str]) -> Optional[str]:
+    key = _normalize_odl_image_url(url)
+    if not key or key.startswith("data:"):
+        return None
+    if key in aliases:
+        return aliases[key]
+    base = os.path.basename(key)
+    for candidate in (base, f"images/{base}"):
+        if candidate in aliases:
+            return aliases[candidate]
+    m = _IMAGE_FILE_NUM_RE.match(base)
+    if m:
+        num = int(m.group(1))
+        numbered = []
+        for ref in {aliases[k] for k in aliases}:
+            bm = _IMAGE_FILE_NUM_RE.match(os.path.basename(ref))
+            if bm:
+                numbered.append((int(bm.group(1)), ref))
+        numbered.sort(key=lambda x: x[0])
+        for n, ref in numbered:
+            if n == num:
+                return ref
+        if numbered and 1 <= num <= len(numbered):
+            return numbered[num - 1][1]
+    return None
+
+
+def _rewrite_markdown_image_refs(
+    markdown: str, images: Dict[str, str]
+) -> str:
+    if not images:
+        return markdown
+    aliases = _build_path_alias_map(images)
+
+    def repl(match: re.Match[str]) -> str:
+        alt, raw_url = match.group(1), match.group(2)
+        url = raw_url.strip().split()[0] if raw_url else ""
+        canonical = _resolve_image_ref(url, aliases)
+        if canonical is None:
+            return match.group(0)
+        return f"![{alt}]({canonical})"
+
+    return _MD_IMAGE_RE.sub(repl, markdown)
+
+
+def _run_convert(
+    pdf_path: str,
+    output_dir: str,
+    image_dir: str,
+    overrides: Optional[Mapping[str, Any]] = None,
+) -> None:
+    import opendataloader_pdf
+
+    kwargs: Dict[str, Any] = {
+        "input_path": pdf_path,
+        "output_dir": output_dir,
+        "format": "markdown",
+        "image_output": "external",
+        "image_dir": image_dir,
+        "quiet": True,
+        "markdown_with_html": _override_bool(
+            overrides, "odl_markdown_with_html", CONFIG.odl_markdown_with_html
+        ),
+    }
+    hybrid = _resolve_hybrid(overrides)
+    if hybrid and hybrid.lower() not in ("off", ""):
+        kwargs["hybrid"] = hybrid
+        hybrid_url = _resolve_hybrid_url(overrides)
+        if hybrid_url:
+            safe, reason = is_ssrf_safe_url(hybrid_url)
+            if not safe:
+                raise RuntimeError(
+                    f"OpenDataLoader hybrid URL 被 SSRF 防护拦截: {reason}"
+                )
+            kwargs["hybrid_url"] = hybrid_url
+        hybrid_mode = _override_str(overrides, "odl_hybrid_mode", CONFIG.odl_hybrid_mode)
+        if hybrid_mode:
+            kwargs["hybrid_mode"] = hybrid_mode
+        if _override_bool(overrides, "odl_hybrid_fallback", CONFIG.odl_hybrid_fallback):
+            kwargs["hybrid_fallback"] = True
+
+    opendataloader_pdf.convert(**kwargs)
+
+
+class OpenDataLoaderParser(BaseParser):
+    """Parse PDFs with OpenDataLoader (layout-aware markdown + external images)."""
+
+    def __init__(self, *args: Any, **kwargs: Any):
+        self._engine_overrides: Dict[str, Any] = {
+            k: v
+            for k, v in kwargs.items()
+            if k.startswith("odl_") or k in ("mineru_endpoint", "mineru_api_key")
+        }
+        super().__init__(*args, **kwargs)
+
+    def parse_into_text(self, content: bytes) -> Document:
+        ok, msg = opendataloader_available(self._engine_overrides)
+        if not ok:
+            raise RuntimeError(msg)
+
+        safe_name = os.path.basename(self.file_name) or "document.pdf"
+        if not safe_name.lower().endswith(".pdf"):
+            safe_name = f"{os.path.splitext(safe_name)[0] or 'document'}.pdf"
+        pdf_stem = os.path.splitext(safe_name)[0]
+
+        max_workers = CONFIG.odl_max_workers
+        with parser_worker_limit("opendataloader", max_workers):
+            with tempfile.TemporaryDirectory(prefix="weknora-odl-") as tmp_dir:
+                pdf_path = os.path.join(tmp_dir, safe_name)
+                with open(pdf_path, "wb") as f:
+                    f.write(content)
+                image_dir = os.path.join(tmp_dir, "images")
+                os.makedirs(image_dir, exist_ok=True)
+
+                _run_convert(
+                    pdf_path,
+                    tmp_dir,
+                    image_dir,
+                    overrides=self._engine_overrides,
+                )
+
+                md_path = _find_markdown_file(tmp_dir, pdf_stem)
+                with open(md_path, encoding="utf-8", errors="replace") as f:
+                    text = f.read()
+
+                images = _collect_images_under_output(tmp_dir)
+                text = _rewrite_markdown_image_refs(text, images)
+
+        if len(text.strip()) < _MIN_CHARS_PER_PAGE:
+            logger.info(
+                "OpenDataLoaderParser: %s yielded little text; "
+                "falling back to builtin scanned renderer",
+                self.file_name,
+            )
+            from docreader.parser.pdf_parser import PDFScannedParser
+
+            return PDFScannedParser(
+                file_name=self.file_name, file_type=self.file_type
+            ).parse_into_text(content)
+
+        logger.info(
+            "OpenDataLoaderParser: %s -> content_len=%d images=%d",
+            self.file_name,
+            len(text),
+            len(images),
+        )
+        return Document(
+            content=text,
+            images=images,
+            metadata={
+                "parser_engine": "opendataloader",
+                "odl_hybrid": _resolve_hybrid(self._engine_overrides) or "off",
+            },
+        )

docreader/parser/parser.py

@@ -0,0 +1,103 @@
+import logging
+from typing import Any, Optional
+
+from docreader.models.document import Document
+from docreader.parser.registry import registry
+from docreader.parser.web_parser import WebParser
+
+logger = logging.getLogger(__name__)
+
+
+# OLE Compound File magic used by legacy binary Microsoft Office files.
+# Some WPS/Word documents keep this payload while being renamed to .docx.
+_OLE_COMPOUND_FILE_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
+
+
+def detect_effective_file_type(file_type: str, content: bytes) -> str:
+    """Return the parser file type after checking trustworthy file magic.
+
+    OOXML ``.docx`` files are ZIP containers, while legacy ``.doc`` files
+    use the OLE Compound File format.  Word and WPS tolerate a legacy file
+    renamed to ``.docx``, so route that well-known mismatch through the DOC
+    parser instead of feeding binary OLE data to the DOCX parser.
+    """
+    normalized = file_type.lower().lstrip(".")
+    if normalized == "docx" and content.startswith(_OLE_COMPOUND_FILE_MAGIC):
+        logger.warning(
+            "Detected legacy DOC content with a DOCX extension; using DOC parser"
+        )
+        return "doc"
+    return normalized
+
+
+class Parser:
+    """Document parser facade (lightweight version).
+
+    Converts files/URLs to markdown + image references.
+    No chunking, no storage, no OCR, no VLM.
+    """
+
+    def __init__(self):
+        self.registry = registry
+        logger.info(
+            "Parser initialized with engines: %s",
+            ", ".join(self.registry.get_engine_names()),
+        )
+
+    def parse_file(
+        self,
+        file_name: str,
+        file_type: str,
+        content: bytes,
+        parser_engine: Optional[str] = None,
+        engine_overrides: Optional[dict[str, Any]] = None,
+    ) -> Document:
+        """Parse file content to markdown."""
+        engine = parser_engine or ""
+        overrides = engine_overrides or {}
+        logger.info(
+            "Parsing file: %s, type: %s, engine: %s",
+            file_name,
+            file_type,
+            engine or "builtin",
+        )
+
+        effective_file_type = detect_effective_file_type(file_type, content)
+        cls = self.registry.get_parser_class(engine, effective_file_type)
+        logger.info(
+            "Creating %s parser instance for %s file",
+            cls.__name__,
+            effective_file_type,
+        )
+        parser = cls(
+            file_name=file_name,
+            file_type=effective_file_type,
+            **overrides,
+        )
+
+        logger.info("Starting to parse file content, size: %d bytes", len(content))
+        result = parser.parse(content)
+
+        if not result.content:
+            logger.warning("Parser returned empty content for file: %s", file_name)
+        logger.info("Parsed file %s, content length=%d", file_name, len(result.content))
+        return result
+
+    def parse_url(
+        self,
+        url: str,
+        title: str,
+        parser_engine: Optional[str] = None,
+        engine_overrides: Optional[dict[str, Any]] = None,
+    ) -> Document:
+        """Parse content from a URL to markdown."""
+        logger.info("Parsing URL: %s, title: %s", url, title)
+
+        parser = WebParser(title=title)
+        logger.info("Starting to parse URL content")
+        result = parser.parse(url.encode())
+
+        if not result.content:
+            logger.warning("Parser returned empty content for url: %s", url)
+        logger.info("Parsed url %s, content length=%d", url, len(result.content))
+        return result

docreader/parser/ppt_convert.py

@@ -0,0 +1,116 @@
+"""LibreOffice helpers for legacy binary PowerPoint (.ppt) uploads."""
+
+from __future__ import annotations
+
+import logging
+import os
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+from docreader.parser.excel_convert import find_soffice
+
+logger = logging.getLogger(__name__)
+
+_OLE_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
+_ZIP_MAGIC = b"PK\x03\x04"
+
+
+def is_ole_compound(content: bytes) -> bool:
+    return len(content) >= len(_OLE_MAGIC) and content.startswith(_OLE_MAGIC)
+
+
+def is_zip_openxml(content: bytes) -> bool:
+    return len(content) >= len(_ZIP_MAGIC) and content.startswith(_ZIP_MAGIC)
+
+
+def needs_ppt_to_pptx_conversion(content: bytes, file_type: str | None) -> bool:
+    """True when content is legacy .ppt (OLE), not modern .pptx (ZIP)."""
+    ext = (file_type or "").lstrip(".").lower()
+    if ext == "pptx" or is_zip_openxml(content):
+        return False
+    if ext == "ppt" or is_ole_compound(content):
+        return is_ole_compound(content) or ext == "ppt"
+    return False
+
+
+def convert_ppt_to_pptx_bytes(content: bytes, suffix: str = ".ppt") -> bytes | None:
+    """Convert legacy PowerPoint bytes to PPTX using LibreOffice, if available."""
+    soffice = find_soffice()
+    if not soffice:
+        return None
+
+    max_attempts = 3
+    for attempt in range(1, max_attempts + 1):
+        with tempfile.TemporaryDirectory() as temp_dir, tempfile.TemporaryDirectory() as profile_dir:
+            src = os.path.join(temp_dir, f"input{suffix}")
+            with open(src, "wb") as handle:
+                handle.write(content)
+
+            user_installation = Path(profile_dir).as_uri()
+            cmd = [
+                soffice,
+                "--headless",
+                f"-env:UserInstallation={user_installation}",
+                "--convert-to",
+                "pptx",
+                "--outdir",
+                temp_dir,
+                src,
+            ]
+            try:
+                result = subprocess.run(cmd, capture_output=True, timeout=120)
+            except (OSError, subprocess.TimeoutExpired) as exc:
+                logger.warning("LibreOffice PPT convert failed to start: %s", exc)
+                return None
+
+            if result.returncode != 0:
+                stderr = result.stderr.decode("utf-8", errors="ignore")
+                logger.warning(
+                    "LibreOffice PPT convert failed (attempt %s/%s): %s",
+                    attempt,
+                    max_attempts,
+                    stderr,
+                )
+                if attempt < max_attempts:
+                    time.sleep(0.5 * attempt)
+                    continue
+                return None
+
+            for name in os.listdir(temp_dir):
+                if name.endswith(".pptx"):
+                    with open(os.path.join(temp_dir, name), "rb") as handle:
+                        converted = handle.read()
+                    logger.info(
+                        "Converted presentation via LibreOffice (%s -> pptx, %d bytes)",
+                        suffix,
+                        len(converted),
+                    )
+                    return converted
+
+            if attempt < max_attempts:
+                time.sleep(0.5 * attempt)
+    return None
+
+
+def normalize_ppt_bytes(content: bytes, file_type: str | None) -> tuple[bytes, str]:
+    """Return (bytes, extension) suitable for MarkItDown (pptx when converted)."""
+    ext = (file_type or "").lstrip(".").lower()
+
+    if is_zip_openxml(content):
+        return content, ".pptx"
+
+    if not needs_ppt_to_pptx_conversion(content, ext):
+        dotted = f".{ext}" if ext else ".pptx"
+        return content, dotted
+
+    suffix = ".ppt" if ext in ("", "ppt") else f".{ext}"
+    converted = convert_ppt_to_pptx_bytes(content, suffix=suffix)
+    if converted:
+        return converted, ".pptx"
+
+    raise ValueError(
+        "Legacy PowerPoint (.ppt) is not supported by MarkItDown directly; "
+        "LibreOffice is required to convert it to .pptx. Install LibreOffice "
+        "(soffice) in the docreader environment or upload .pptx instead."
+    )

docreader/parser/pptx_media.py

@@ -0,0 +1,154 @@
+"""Extract and rasterize images embedded in PPTX (e.g. WMF) when MarkItDown cannot inline them."""
+
+from __future__ import annotations
+
+import base64
+import io
+import logging
+import os
+import re
+import subprocess
+import tempfile
+import uuid
+import zipfile
+from typing import Dict, List, Tuple
+
+logger = logging.getLogger(__name__)
+
+_MARKDOWN_IMAGE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
+_RASTER_EXT = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"}
+_VECTOR_EXT = {".wmf", ".emf", ".svg"}
+
+
+def _find_convert() -> str | None:
+    for path in ("/usr/bin/convert", "/usr/local/bin/convert"):
+        if os.path.isfile(path):
+            return path
+    try:
+        result = subprocess.run(
+            ["which", "convert"], capture_output=True, text=True, check=False
+        )
+        if result.returncode == 0 and result.stdout.strip():
+            return result.stdout.strip()
+    except OSError:
+        pass
+    return None
+
+
+def _rasterize_with_imagemagick(data: bytes, suffix: str) -> bytes | None:
+    convert = _find_convert()
+    if not convert:
+        return None
+    with tempfile.TemporaryDirectory() as temp_dir:
+        src = os.path.join(temp_dir, f"input{suffix}")
+        dst = os.path.join(temp_dir, "output.png")
+        with open(src, "wb") as handle:
+            handle.write(data)
+        try:
+            result = subprocess.run(
+                [convert, src, dst],
+                capture_output=True,
+                timeout=60,
+            )
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            logger.warning("ImageMagick convert failed: %s", exc)
+            return None
+        if result.returncode != 0 or not os.path.isfile(dst):
+            stderr = (result.stderr or b"").decode("utf-8", errors="ignore")
+            logger.warning("ImageMagick convert exit %s: %s", result.returncode, stderr)
+            return None
+        with open(dst, "rb") as handle:
+            return handle.read()
+
+
+def _rasterize_with_pillow(data: bytes) -> bytes | None:
+    try:
+        from PIL import Image
+    except ImportError:
+        return None
+    try:
+        img = Image.open(io.BytesIO(data))
+        if img.mode not in ("RGB", "L"):
+            img = img.convert("RGB")
+        out = io.BytesIO()
+        img.save(out, format="PNG")
+        return out.getvalue()
+    except Exception as exc:
+        logger.debug("Pillow could not open media bytes: %s", exc)
+        return None
+
+
+def rasterize_media_bytes(name: str, data: bytes) -> bytes | None:
+    ext = os.path.splitext(name)[1].lower()
+    if ext in _RASTER_EXT:
+        png = _rasterize_with_pillow(data)
+        if png:
+            return png
+    if ext in _VECTOR_EXT or ext in _RASTER_EXT:
+        return _rasterize_with_imagemagick(data, ext or ".bin")
+    return _rasterize_with_imagemagick(data, ext or ".bin")
+
+
+def list_pptx_media(pptx_bytes: bytes) -> List[Tuple[str, bytes]]:
+    """Return (zip path, raw bytes) for each file under ppt/media/, in archive order."""
+    items: List[Tuple[str, bytes]] = []
+    with zipfile.ZipFile(io.BytesIO(pptx_bytes)) as archive:
+        for name in archive.namelist():
+            if not name.startswith("ppt/media/"):
+                continue
+            base = os.path.basename(name)
+            if not base or base.startswith("."):
+                continue
+            items.append((name, archive.read(name)))
+    return items
+
+
+def extract_pptx_media_rasterized(pptx_bytes: bytes) -> List[bytes]:
+    """Rasterize all ppt/media assets to PNG bytes, skipping failures."""
+    rasterized: List[bytes] = []
+    for path, raw in list_pptx_media(pptx_bytes):
+        png = rasterize_media_bytes(os.path.basename(path), raw)
+        if png:
+            rasterized.append(png)
+            logger.info("Rasterized pptx media %s (%d -> %d bytes)", path, len(raw), len(png))
+        else:
+            logger.warning("Failed to rasterize pptx media %s", path)
+    return rasterized
+
+
+def _is_unresolved_image_ref(url: str) -> bool:
+    if not url or url.startswith("data:") or url.startswith("images/"):
+        return False
+    if url.startswith(("http://", "https://")):
+        return False
+    return True
+
+
+def attach_pptx_media_to_markdown(
+    markdown: str, pptx_bytes: bytes
+) -> Tuple[str, Dict[str, str]]:
+    """Replace unresolved ![](...) refs with images/ paths and inline image payloads."""
+    media = extract_pptx_media_rasterized(pptx_bytes)
+    if not media:
+        return markdown, {}
+
+    images: Dict[str, str] = {}
+    media_iter = iter(media)
+
+    def repl(match: re.Match[str]) -> str:
+        alt, url = match.group(1), match.group(2)
+        if not _is_unresolved_image_ref(url):
+            return match.group(0)
+        try:
+            png = next(media_iter)
+        except StopIteration:
+            return match.group(0)
+        ref = f"images/{uuid.uuid4()}.png"
+        images[ref] = base64.b64encode(png).decode()
+        return f"![{alt}]({ref})"
+
+    return _MARKDOWN_IMAGE.sub(repl, markdown), images
+
+
+def markdown_needs_pptx_media_attach(markdown: str) -> bool:
+    return any(_is_unresolved_image_ref(m.group(2)) for m in _MARKDOWN_IMAGE.finditer(markdown))

docreader/parser/registry.py

@@ -0,0 +1,216 @@
+import logging
+from typing import Callable, Dict, List, Optional, Tuple, Type
+
+from docreader.parser.base_parser import BaseParser
+from docreader.parser.doc_parser import DocParser
+from docreader.parser.docx2_parser import Docx2Parser
+from docreader.parser.epub_parser import EPUBParser
+from docreader.parser.excel_parser import ExcelParser
+from docreader.parser.html_parser import HTMLParser
+from docreader.parser.image_parser import ImageParser
+from docreader.parser.markdown_parser import MarkdownParser
+from docreader.parser.markitdown_parser import MarkitdownParser
+from docreader.parser.mhtml_parser import MHTMLParser
+from docreader.parser.opendataloader_parser import (
+    OpenDataLoaderParser,
+    opendataloader_available,
+)
+from docreader.parser.pdf_parser import PDFParser
+from docreader.parser.xmind_parser import XMindParser
+
+logger = logging.getLogger(__name__)
+
+BUILTIN_ENGINE = "builtin"
+
+# File types that builtin does not implement natively. When the requested
+# engine (including empty / builtin) cannot parse them, route to this engine
+# instead of raising ValueError — PPT/PPTX/CSV are markitdown-only.
+_DEFAULT_ENGINE_BY_TYPE = {
+    "ppt": "markitdown",
+    "pptx": "markitdown",
+    "csv": "markitdown",
+}
+
+
+class ParserEngineRegistry:
+    """Registry for parser engines.
+
+    Each engine maps file extensions to parser classes.
+    When a requested engine doesn't support a file type, the registry
+    tries a type-level default engine, then builtin.
+    """
+
+    def __init__(self):
+        self._engines: Dict[str, Dict[str, Type[BaseParser]]] = {}
+        self._descriptions: Dict[str, str] = {}
+        self._check_available: Dict[str, Callable[..., Tuple[bool, str]]] = {}
+        self._unavailable_hint: Dict[str, str] = {}
+
+    def register(
+        self,
+        name: str,
+        file_types: Dict[str, Type[BaseParser]],
+        description: str = "",
+        check_available: Callable[..., Tuple[bool, str]] | None = None,
+        unavailable_hint: str = "",
+    ):
+        self._engines[name] = file_types
+        self._descriptions[name] = description
+        if check_available is not None:
+            self._check_available[name] = check_available
+            self._unavailable_hint[name] = unavailable_hint
+        logger.info(
+            "Registered parser engine '%s' with file types: %s",
+            name,
+            ", ".join(file_types.keys()),
+        )
+
+    def get_parser_class(self, engine: str, file_type: str) -> Type[BaseParser]:
+        """Resolve parser class for the given engine and file type.
+
+        Fallback order when the requested engine is empty or does not
+        support the type: type-level default (e.g. ppt → markitdown),
+        then builtin. Raises an actionable error listing engines that
+        do support the type.
+        """
+        ft = file_type.lower().lstrip(".")
+
+        if engine and engine in self._engines:
+            cls = self._engines[engine].get(ft)
+            if cls:
+                logger.info("Using engine '%s' for file type '%s'", engine, ft)
+                return cls
+            logger.info(
+                "Engine '%s' does not support '%s', falling back",
+                engine,
+                ft,
+            )
+
+        default_engine = _DEFAULT_ENGINE_BY_TYPE.get(ft)
+        if default_engine:
+            cls = self._engines.get(default_engine, {}).get(ft)
+            if cls:
+                logger.info(
+                    "Using default engine '%s' for file type '%s'",
+                    default_engine,
+                    ft,
+                )
+                return cls
+
+        builtin = self._engines.get(BUILTIN_ENGINE, {})
+        cls = builtin.get(ft)
+        if cls:
+            return cls
+
+        supported = sorted(
+            name for name, parsers in self._engines.items() if ft in parsers
+        )
+        engine_label = engine or BUILTIN_ENGINE
+        if supported:
+            raise ValueError(
+                f"Unsupported file type {file_type!r} for engine {engine_label!r}. "
+                f"Configure one of: {', '.join(supported)}"
+            )
+        raise ValueError(f"Unsupported file type: {file_type}")
+
+    def list_engines(self, overrides: Optional[Dict[str, str]] = None) -> List[Dict]:
+        """Return metadata for all registered engines, including availability.
+
+        Args:
+            overrides: tenant-level config overrides (e.g. mineru_endpoint, mineru_api_key)
+                       forwarded to each engine's check_available function.
+        """
+        result = []
+        for name, parsers in self._engines.items():
+            available = True
+            unavailable_reason = ""
+            check = self._check_available.get(name)
+            if check is not None:
+                try:
+                    available, unavailable_reason = check(overrides)
+                except Exception as e:
+                    available = False
+                    unavailable_reason = str(e) or self._unavailable_hint.get(name, "")
+            if not available and not unavailable_reason:
+                unavailable_reason = self._unavailable_hint.get(name, "不可用")
+            result.append(
+                {
+                    "name": name,
+                    "description": self._descriptions.get(name, ""),
+                    "file_types": sorted(parsers.keys()),
+                    "available": available,
+                    "unavailable_reason": unavailable_reason,
+                }
+            )
+        return result
+
+    def get_engine_names(self) -> List[str]:
+        return list(self._engines.keys())
+
+
+def _build_default_registry() -> ParserEngineRegistry:
+    """Create and populate the default registry with all known engines."""
+    reg = ParserEngineRegistry()
+
+    _image_types = {
+        ext: ImageParser for ext in ("jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp")
+    }
+
+    reg.register(
+        BUILTIN_ENGINE,
+        {
+            "docx": Docx2Parser,
+            "doc": DocParser,
+            "pdf": PDFParser,
+            "md": MarkdownParser,
+            "markdown": MarkdownParser,
+            "xlsx": ExcelParser,
+            "xls": ExcelParser,
+            "pptx": MarkitdownParser,
+            "ppt": MarkitdownParser,
+            "epub": EPUBParser,
+            "html": HTMLParser,
+            "htm": HTMLParser,
+            "mhtml": MHTMLParser,
+            "xmind": XMindParser,
+            **_image_types,
+        },
+        description="内置解析引擎",
+    )
+
+    reg.register(
+        "markitdown",
+        {
+            "md": MarkitdownParser,
+            "markdown": MarkitdownParser,
+            "pdf": MarkitdownParser,
+            "docx": MarkitdownParser,
+            "doc": MarkitdownParser,
+            "pptx": MarkitdownParser,
+            "ppt": MarkitdownParser,
+            "xlsx": MarkitdownParser,
+            "xls": MarkitdownParser,
+            "csv": MarkitdownParser,
+        },
+        description="MarkItDown 解析引擎（微软 MarkItDown 库）",
+    )
+
+    reg.register(
+        "opendataloader",
+        {"pdf": OpenDataLoaderParser},
+        description="OpenDataLoader PDF（版面分析，需 Java 11+）",
+        check_available=lambda overrides: opendataloader_available(
+            overrides, quick=True
+        ),
+        unavailable_hint="请安装 opendataloader-pdf 与 Java 11+",
+    )
+
+    # NOTE: Engine listing is managed by Go-side engine registry
+    # (docparser.ListAllEngines). The Python list_engines method is kept for
+    # backward compatibility with the gRPC ListEngines RPC but the Go app
+    # no longer calls it. MinerU engines are handled natively by Go.
+
+    return reg
+
+
+registry = _build_default_registry()

docreader/parser/web_parser.py

@@ -0,0 +1,353 @@
+import asyncio
+import logging
+import re
+from dataclasses import dataclass
+from typing import Optional
+
+from lxml.etree import XPath
+from playwright.async_api import Page, async_playwright
+from trafilatura import extract, utils, xpaths
+
+from docreader.config import CONFIG
+from docreader.models.document import Document
+from docreader.parser.base_parser import BaseParser
+from docreader.parser.chain_parser import PipelineParser
+from docreader.parser.markdown_parser import MarkdownParser
+from docreader.utils import endecode
+from docreader.utils.ssrf import is_ssrf_safe_url
+
+logger = logging.getLogger(__name__)
+
+
+class WebParseError(RuntimeError):
+    """Raised when a URL cannot be scraped or has no extractable content.
+
+    This must propagate as a parse failure. Returning the error string as
+    document body would index it as knowledge.
+    """
+
+
+_GOTO_TIMEOUT_MS = 30_000
+_NETWORK_IDLE_TIMEOUT_MS = 10_000
+_SPA_WAIT_TIMEOUT_MS = 15_000
+# Minimum visible characters before treating an SPA shell as "rendered".
+_SPA_MIN_TEXT_LEN = 80
+# Minimum visible characters for Playwright text fallback when trafilatura fails.
+_MIN_FALLBACK_TEXT_LEN = 50
+
+# Monkey-patch trafilatura internals to better support WeChat Official Account
+# articles, whose images live on `mmbiz.qpic.cn` without a standard file
+# extension and whose main content sits inside `#js_content` /
+# `.rich_media_content`. Trafilatura's `utils.IMAGE_EXTENSION` and
+# `xpaths.BODY_XPATH` are internal APIs, so we guard the patch and skip
+# silently if they are renamed/removed in a future release.
+try:
+    _WECHAT_IMAGE_EXTENSION = re.compile(
+        r"[^\s]+\.(avif|bmp|gif|hei[cf]|jpe?g|png|webp)(\b|$)|"  # Standard extensions
+        r"mmbiz\.qpic\.cn/[^\s]*wx_fmt=(jpeg|jpg|png|gif|webp)"  # WeChat query format
+    )
+    utils.IMAGE_EXTENSION = _WECHAT_IMAGE_EXTENSION
+
+    _WECHAT_BODY_XPATH = XPath(
+        '(.//*[@id="js_content" or contains(@class, "rich_media_content")])[1]'
+    )
+    _wechat_xpath_str = str(_WECHAT_BODY_XPATH)
+    if not any(str(x) == _wechat_xpath_str for x in xpaths.BODY_XPATH):
+        xpaths.BODY_XPATH.insert(0, _WECHAT_BODY_XPATH)
+except (AttributeError, ImportError) as e:
+    logger.warning(
+        "Failed to patch trafilatura internals for WeChat support: %s", e
+    )
+
+
+@dataclass(frozen=True)
+class _ScrapeResult:
+    html: str
+    visible_text: str
+    page_title: str
+    error: str = ""
+
+
+def extract_markdown_from_html(html: str) -> Optional[str]:
+    """Run trafilatura on HTML; return markdown or None if nothing extracted."""
+    if not html or not html.strip():
+        return None
+    md_text = extract(
+        html,
+        output_format="markdown",
+        with_metadata=True,
+        include_images=True,
+        include_tables=True,
+        include_links=True,
+    )
+    if not md_text or not md_text.strip():
+        return None
+    return md_text
+
+
+def build_visible_text_fallback(visible_text: str, page_title: str = "") -> Optional[str]:
+    """Build markdown from Playwright-visible text when trafilatura finds no article body."""
+    text = (visible_text or "").strip()
+    if len(text) < _MIN_FALLBACK_TEXT_LEN:
+        return None
+    title = (page_title or "").strip()
+    if title and not text.startswith(title):
+        return f"# {title}\n\n{text}"
+    return text
+
+
+async def wait_for_rendered_content(page: Page) -> None:
+    """Wait for SPA/JS pages beyond the initial HTML shell."""
+    try:
+        await page.wait_for_load_state("networkidle", timeout=_NETWORK_IDLE_TIMEOUT_MS)
+        logger.info("Network idle after navigation")
+    except Exception:
+        logger.info("Network idle wait timed out, continuing")
+
+    try:
+        await page.wait_for_function(
+            """(minLen) => {
+                const root = document.querySelector('#app')
+                    || document.querySelector('main')
+                    || document.body;
+                return ((root?.innerText || '').trim().length >= minLen);
+            }""",
+            arg=_SPA_MIN_TEXT_LEN,
+            timeout=_SPA_WAIT_TIMEOUT_MS,
+        )
+        logger.info("SPA/root visible text reached minimum length")
+    except Exception:
+        logger.info("SPA text wait timed out, using current DOM")
+
+
+async def read_visible_text(page: Page) -> str:
+    """Prefer #app/main innerText, then fall back to body."""
+    return await page.evaluate(
+        """() => {
+            const root = document.querySelector('#app')
+                || document.querySelector('main')
+                || document.querySelector('[role="main"]')
+                || document.body;
+            return (root?.innerText || '').trim();
+        }"""
+    )
+
+
+async def install_ssrf_route_guard(page: Page) -> None:
+    """Block navigation/subresource requests to SSRF-restricted targets (incl. redirects)."""
+
+    async def handle_route(route) -> None:
+        safe, reason = is_ssrf_safe_url(route.request.url)
+        if not safe:
+            logger.warning(
+                "SSRF guard blocked request to %s: %s", route.request.url, reason
+            )
+            await route.abort("blockedbyclient")
+            return
+        await route.continue_()
+
+    await page.route("**/*", handle_route)
+
+
+class StdWebParser(BaseParser):
+    """Standard web page parser using Playwright and Trafilatura.
+
+    This parser scrapes web pages using Playwright's WebKit browser and extracts
+    clean content using Trafilatura library. It supports proxy configuration and
+    converts HTML content to markdown format.
+    """
+
+    def __init__(self, title: str, **kwargs):
+        """Initialize the web parser.
+
+        Args:
+            title: Title of the web page to be used as file name
+            **kwargs: Additional arguments passed to BaseParser
+        """
+        self.title = title
+        # Get proxy configuration from config if available
+        self.proxy = CONFIG.external_https_proxy
+        super().__init__(file_name=title, **kwargs)
+        logger.info(f"Initialized WebParser with title: {title}")
+
+    async def scrape(self, url: str) -> _ScrapeResult:
+        """Scrape web page content using Playwright.
+
+        Args:
+            url: The URL of the web page to scrape
+
+        Returns:
+            HTML, visible text, and document title. On hard failure the
+            content fields are empty and ``error`` explains why.
+        """
+        logger.info(f"Starting web page scraping for URL: {url}")
+        safe, reason = is_ssrf_safe_url(url)
+        if not safe:
+            logger.error("URL blocked by SSRF guard before navigation: %s", reason)
+            return _ScrapeResult(
+                html="", visible_text="", page_title="",
+                error=f"URL blocked by SSRF guard: {reason}",
+            )
+        try:
+            async with async_playwright() as p:
+                kwargs = {}
+                # Configure proxy if available
+                if self.proxy:
+                    kwargs["proxy"] = {"server": self.proxy}
+                logger.info("Launching WebKit browser")
+                browser = await p.webkit.launch(**kwargs)
+                page = await browser.new_page()
+                await install_ssrf_route_guard(page)
+
+                logger.info(f"Navigating to URL: {url}")
+                try:
+                    await page.goto(
+                        url,
+                        timeout=_GOTO_TIMEOUT_MS,
+                        wait_until="domcontentloaded",
+                    )
+                    logger.info("Initial page load complete")
+                except Exception as e:
+                    logger.error(f"Error navigating to URL: {str(e)}")
+                    await browser.close()
+                    return _ScrapeResult(
+                        html="", visible_text="", page_title="",
+                        error=f"navigation failed: {e}",
+                    )
+
+                await wait_for_rendered_content(page)
+
+                page_title = await page.title()
+                visible_text = await read_visible_text(page)
+                content = await page.content()
+                logger.info(
+                    "Retrieved %d bytes HTML, %d chars visible text, title=%r",
+                    len(content),
+                    len(visible_text),
+                    page_title[:80] if page_title else "",
+                )
+
+                await browser.close()
+                logger.info("Browser closed")
+
+            logger.info("Successfully retrieved HTML content")
+            return _ScrapeResult(
+                html=content,
+                visible_text=visible_text,
+                page_title=page_title or "",
+            )
+
+        except Exception as e:
+            logger.error(f"Failed to scrape web page: {str(e)}")
+            return _ScrapeResult(
+                html="", visible_text="", page_title="",
+                error=f"scrape failed: {e}",
+            )
+
+    def parse_into_text(self, content: bytes) -> Document:
+        """Parse web page content into a Document object.
+
+        Args:
+            content: URL encoded as bytes
+
+        Returns:
+            Document object containing the parsed markdown content
+
+        Raises:
+            WebParseError: If scraping fails or the page has no extractable
+                content. Callers must surface this as a parse failure rather
+                than indexing the error string as document body.
+        """
+        url = endecode.decode_bytes(content)
+
+        logger.info(f"Scraping web page: {url}")
+        scrape_result = asyncio.run(self.scrape(url))
+        if scrape_result.error or (
+            not scrape_result.html and not scrape_result.visible_text
+        ):
+            detail = scrape_result.error or "no HTML or visible text"
+            logger.error("Failed to scrape web page %s: %s", url, detail)
+            raise WebParseError(f"Failed to scrape web page: {url} ({detail})")
+
+        md_text = extract_markdown_from_html(scrape_result.html)
+        if not md_text:
+            md_text = build_visible_text_fallback(
+                scrape_result.visible_text,
+                scrape_result.page_title,
+            )
+            if md_text:
+                logger.info(
+                    "Trafilatura empty; using Playwright visible-text fallback (%d chars)",
+                    len(md_text),
+                )
+
+        if not md_text:
+            logger.error("Failed to parse web page: %s", url)
+            raise WebParseError(f"Failed to parse web page: {url}")
+
+        metadata = {}
+        title_match = re.search(r"^title:\s*(.+)", md_text, re.MULTILINE)
+        if title_match:
+            extracted_title = title_match.group(1).strip()
+            if extracted_title:
+                metadata["title"] = extracted_title
+                logger.info(
+                    f"Extracted article title from trafilatura: {extracted_title}"
+                )
+        elif scrape_result.page_title:
+            metadata["title"] = scrape_result.page_title.strip()
+            logger.info(
+                "Using page title from Playwright: %s", metadata["title"]
+            )
+        else:
+            logger.info(
+                "No title found in trafilatura output, first 200 chars: %r",
+                md_text[:200],
+            )
+        return Document(content=md_text, metadata=metadata)
+
+
+class WebParser(PipelineParser):
+    """Web parser using pipeline pattern.
+
+    This parser chains StdWebParser (for web scraping and HTML to markdown conversion)
+    with MarkdownParser (for markdown processing). The pipeline processes content
+    sequentially through both parsers.
+    """
+
+    # Parser classes to be executed in sequence
+    _parser_cls = (StdWebParser, MarkdownParser)
+
+
+if __name__ == "__main__":
+    import sys
+
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
+
+    url = sys.argv[1] if len(sys.argv) > 1 else "https://cloud.tencent.com/document/product/457/6759"
+    print(f"\n{'='*60}")
+    print(f"URL: {url}")
+    print(f"{'='*60}\n")
+
+    parser = WebParser(title="")
+    doc = parser.parse_into_text(url.encode())
+
+    print(f"--- metadata ---")
+    for k, v in doc.metadata.items():
+        print(f"  {k}: {v}")
+
+    print(f"\n--- images ({len(doc.images)}) ---")
+    for path in list(doc.images.keys())[:10]:
+        print(f"  {path}  ({len(doc.images[path])} chars base64)")
+
+    print(f"\n--- content ({len(doc.content)} chars) ---")
+    print(doc.content[:300000])
+    if len(doc.content) > 300000:
+        print(f"\n... (truncated, total {len(doc.content)} chars)")
+
+    print(f"\n--- chunks ({len(doc.chunks)}) ---")
+    for i, chunk in enumerate(doc.chunks[:5]):
+        print(f"  [{i}] seq={chunk.seq} range=[{chunk.start}:{chunk.end}] len={len(chunk.content)}")
+        print(f"      {chunk.content[:120]}{'...' if len(chunk.content) > 120 else ''}")
+    if len(doc.chunks) > 5:
+        print(f"  ... ({len(doc.chunks) - 5} more chunks)")

docreader/parser/xlsx_merge.py

@@ -0,0 +1,56 @@
+"""Fill merged cell values before pandas reads an XLSX workbook."""
+
+from __future__ import annotations
+
+import logging
+import zipfile
+from io import BytesIO
+
+logger = logging.getLogger(__name__)
+
+
+def fill_merged_cells_xlsx(content: bytes) -> bytes:
+    """Unmerge ranges and copy the master cell value into every covered cell.
+
+    openpyxl only stores values on the top-left cell of a merge; pandas then
+    sees NaN in the rest. Filling makes row-wise RAG chunks retain context.
+    """
+    if not zipfile.is_zipfile(BytesIO(content)):
+        return content
+
+    from openpyxl import load_workbook
+
+    wb = load_workbook(BytesIO(content), data_only=True)
+    changed = False
+    for ws in wb.worksheets:
+        if not ws.merged_cells.ranges:
+            continue
+        for merge_range in list(ws.merged_cells.ranges):
+            master_value = ws.cell(merge_range.min_row, merge_range.min_col).value
+            ws.unmerge_cells(str(merge_range))
+            for row in range(merge_range.min_row, merge_range.max_row + 1):
+                for col in range(merge_range.min_col, merge_range.max_col + 1):
+                    ws.cell(row, col).value = master_value
+            changed = True
+
+    if not changed:
+        return content
+
+    # Charts are not part of the cell text extracted by DocReader. Remove
+    # dedicated chart sheets through the public workbook API, then clear
+    # embedded worksheet charts (openpyxl has no public removal API for them)
+    # before saving this temporary workbook.
+    active_sheet = wb.active
+    for chart_sheet in list(wb.chartsheets):
+        wb.remove(chart_sheet)
+    if active_sheet in wb.worksheets:
+        wb.active = active_sheet
+    elif wb.worksheets:
+        wb.active = wb.worksheets[0]
+    for ws in wb.worksheets:
+        ws._charts.clear()
+
+    out = BytesIO()
+    wb.save(out)
+    logger.info("Filled merged cells in XLSX before parse")
+    return out.getvalue()

docreader/parser/xlsx_repair.py

@@ -0,0 +1,126 @@
+"""Repair common XLSX packaging issues before openpyxl/pandas read."""
+
+from __future__ import annotations
+
+import io
+import re
+import zipfile
+from typing import Callable, Dict, Iterable, Set
+
+SST_PART = "xl/sharedStrings.xml"
+_SST_OVERRIDE_RE = re.compile(
+    r'<Override[^>]*PartName="[^"]*sharedStrings\.xml"[^>]*/>',
+    re.IGNORECASE,
+)
+_SST_REL_RE = re.compile(
+    r'<Relationship[^>]*Type="[^"]*sharedStrings"[^>]*/>',
+    re.IGNORECASE,
+)
+
+
+def repair_xlsx_bytes(content: bytes) -> bytes | None:
+    """Return repaired XLSX bytes, or None if no repair was applied.
+
+    Handles workbooks that reference ``xl/sharedStrings.xml`` in package
+    metadata but omit the part (common with some exporters). When worksheets
+    only use inline strings, manifest references are stripped so openpyxl can
+    read the file.
+    """
+    if not zipfile.is_zipfile(io.BytesIO(content)):
+        return None
+
+    with zipfile.ZipFile(io.BytesIO(content), "r") as zin:
+        names = _normalized_names(zin.namelist())
+        sst_path = _find_shared_strings_path(names)
+        if sst_path:
+            if sst_path == SST_PART:
+                return None
+            return _rewrite_zip(
+                zin, lambda files: _rename_shared_strings_part(files, sst_path)
+            )
+        if not _package_references_shared_strings(zin, names):
+            return None
+        if _worksheets_use_shared_string_cells(zin, names):
+            return None
+        return _rewrite_zip(zin, _strip_shared_strings_manifest)
+
+
+def _normalized_names(namelist: Iterable[str]) -> Set[str]:
+    return {name.replace("\\", "/") for name in namelist}
+
+
+def _find_shared_strings_path(names: Set[str]) -> str | None:
+    for name in names:
+        if name.lower().endswith("sharedstrings.xml"):
+            return name
+    return None
+
+
+def _package_references_shared_strings(
+    zin: zipfile.ZipFile, names: Set[str]
+) -> bool:
+    content_types = "[Content_Types].xml"
+    if content_types in names:
+        ct = zin.read(content_types).decode("utf-8", errors="replace")
+        if "sharedstrings.xml" in ct.lower():
+            return True
+
+    rels_path = "xl/_rels/workbook.xml.rels"
+    if rels_path in names:
+        rels = zin.read(rels_path).decode("utf-8", errors="replace")
+        if "sharedstrings" in rels.lower():
+            return True
+    return False
+
+
+def _worksheets_use_shared_string_cells(
+    zin: zipfile.ZipFile, names: Set[str]
+) -> bool:
+    for name in names:
+        if not name.startswith("xl/worksheets/") or not name.endswith(".xml"):
+            continue
+        sheet = zin.read(name).decode("utf-8", errors="replace")
+        if re.search(r'\bt="s"', sheet):
+            return True
+    return False
+
+
+def _rename_shared_strings_part(
+    files: Dict[str, bytes], source_path: str
+) -> Dict[str, bytes]:
+    updated = dict(files)
+    updated[SST_PART] = updated.pop(source_path)
+    return updated
+
+
+def _strip_shared_strings_manifest(files: Dict[str, bytes]) -> Dict[str, bytes]:
+    updated = dict(files)
+    ct_path = "[Content_Types].xml"
+    if ct_path in updated:
+        ct = updated[ct_path].decode("utf-8")
+        ct = _SST_OVERRIDE_RE.sub("", ct)
+        updated[ct_path] = ct.encode("utf-8")
+
+    rels_path = "xl/_rels/workbook.xml.rels"
+    if rels_path in updated:
+        rels = updated[rels_path].decode("utf-8")
+        rels = _SST_REL_RE.sub("", rels)
+        updated[rels_path] = rels.encode("utf-8")
+    return updated
+
+
+def _rewrite_zip(
+    zin: zipfile.ZipFile,
+    transform: Callable[[Dict[str, bytes]], Dict[str, bytes]],
+) -> bytes:
+    files: Dict[str, bytes] = {}
+    for info in zin.infolist():
+        name = info.filename.replace("\\", "/")
+        files[name] = zin.read(info.filename)
+    files = transform(files)
+
+    out = io.BytesIO()
+    with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zout:
+        for name, data in files.items():
+            zout.writestr(name, data)
+    return out.getvalue()

docreader/parser/xmind_parser.py

@@ -0,0 +1,246 @@
+"""Parse XMind archives into Markdown outlines."""
+
+import io
+import json
+import zipfile
+from dataclasses import dataclass, field
+from xml.etree import ElementTree
+from xml.etree.ElementTree import Element
+
+from docreader.models.document import Document
+from docreader.parser.base_parser import BaseParser
+
+
+MAX_CONTENT_BYTES = 32 * 1024 * 1024
+
+
+@dataclass
+class _Topic:
+    title: str = ""
+    note: str = ""
+    children: list["_Topic"] = field(default_factory=list)
+
+
+@dataclass
+class _Sheet:
+    title: str = ""
+    root_topic: _Topic | None = None
+
+
+def _clean_text(value: object) -> str:
+    return value.strip() if isinstance(value, str) else ""
+
+
+def _topic_from_json(value: object) -> _Topic | None:
+    if not isinstance(value, dict):
+        return None
+
+    note = ""
+    notes = value.get("notes")
+    if isinstance(notes, dict):
+        plain = notes.get("plain")
+        if isinstance(plain, dict):
+            note = _clean_text(plain.get("content"))
+
+    topics: list[_Topic] = []
+    children = value.get("children")
+    if isinstance(children, dict):
+        attached = children.get("attached")
+        if isinstance(attached, list):
+            for child in attached:
+                topic = _topic_from_json(child)
+                if topic is not None:
+                    topics.append(topic)
+
+    return _Topic(
+        title=_clean_text(value.get("title")),
+        note=note,
+        children=topics,
+    )
+
+
+def _parse_json_sheets(payload: bytes) -> list[_Sheet]:
+    values = json.loads(payload)
+    if not isinstance(values, list):
+        raise ValueError("invalid XMind content.json: expected a sheet list")
+
+    sheets: list[_Sheet] = []
+    for value in values:
+        if not isinstance(value, dict):
+            continue
+        sheets.append(
+            _Sheet(
+                title=_clean_text(value.get("title")),
+                root_topic=_topic_from_json(value.get("rootTopic")),
+            )
+        )
+    return sheets
+
+
+def _local_name(tag: str) -> str:
+    return tag.rsplit("}", 1)[-1].split(":", 1)[-1]
+
+
+def _direct_children(element: Element, name: str) -> list[Element]:
+    return [child for child in element if _local_name(child.tag) == name]
+
+
+def _first_child(element: Element, name: str) -> Element | None:
+    children = _direct_children(element, name)
+    return children[0] if children else None
+
+
+def _element_text(element: Element | None) -> str:
+    if element is None:
+        return ""
+    return "".join(element.itertext()).strip()
+
+
+def _topic_from_xml(element: Element) -> _Topic:
+    notes = _first_child(element, "notes")
+    plain = _first_child(notes, "plain") if notes is not None else None
+
+    topics: list[_Topic] = []
+    for children in _direct_children(element, "children"):
+        for topic_group in _direct_children(children, "topics"):
+            topics.extend(
+                _topic_from_xml(topic)
+                for topic in _direct_children(topic_group, "topic")
+            )
+
+    return _Topic(
+        title=_element_text(_first_child(element, "title")),
+        note=_element_text(plain),
+        children=topics,
+    )
+
+
+def _parse_xml_sheets(payload: bytes) -> list[_Sheet]:
+    try:
+        root = ElementTree.fromstring(payload)
+    except ElementTree.ParseError as exc:
+        raise ValueError("invalid XMind content.xml") from exc
+
+    sheets: list[_Sheet] = []
+    for sheet in _direct_children(root, "sheet"):
+        root_topic = _first_child(sheet, "topic")
+        sheets.append(
+            _Sheet(
+                title=_element_text(_first_child(sheet, "title")),
+                root_topic=(
+                    _topic_from_xml(root_topic) if root_topic is not None else None
+                ),
+            )
+        )
+    return sheets
+
+
+def _read_content_entry(content: bytes) -> tuple[str, bytes]:
+    try:
+        with zipfile.ZipFile(io.BytesIO(content)) as archive:
+            try:
+                info = archive.getinfo("content.json")
+                content_format = "json"
+            except KeyError:
+                try:
+                    info = archive.getinfo("content.xml")
+                except KeyError as exc:
+                    raise ValueError(
+                        "XMind archive is missing content.json or content.xml"
+                    ) from exc
+                content_format = "xml"
+
+            if info.flag_bits & 0x1:
+                raise ValueError("encrypted XMind content is not supported")
+            if info.file_size > MAX_CONTENT_BYTES:
+                raise ValueError("XMind content entry exceeds the 32 MiB limit")
+
+            with archive.open(info) as entry:
+                payload = entry.read(MAX_CONTENT_BYTES + 1)
+            if len(payload) > MAX_CONTENT_BYTES:
+                raise ValueError("XMind content entry exceeds the 32 MiB limit")
+            return content_format, payload
+    except zipfile.BadZipFile as exc:
+        raise ValueError("invalid XMind archive") from exc
+
+
+def _render_topic(topic: _Topic, depth: int) -> tuple[list[str], int, int]:
+    lines: list[str] = []
+    topic_count = 0
+    note_count = 0
+    child_depth = depth
+
+    if topic.title:
+        lines.append(f"{'  ' * depth}- {topic.title}")
+        topic_count = 1
+        child_depth += 1
+        if topic.note:
+            for note_line in topic.note.splitlines():
+                normalized = note_line.strip()
+                quote = f"> {normalized}" if normalized else ">"
+                lines.append(f"{'  ' * child_depth}{quote}")
+            note_count = 1
+
+    for child in topic.children:
+        child_lines, child_topics, child_notes = _render_topic(child, child_depth)
+        lines.extend(child_lines)
+        topic_count += child_topics
+        note_count += child_notes
+
+    return lines, topic_count, note_count
+
+
+def _render_sheets(sheets: list[_Sheet]) -> tuple[str, int, int, int]:
+    rendered_sheets: list[str] = []
+    topic_count = 0
+    note_count = 0
+
+    for index, sheet in enumerate(sheets, start=1):
+        if sheet.root_topic is None:
+            continue
+        lines, sheet_topics, sheet_notes = _render_topic(sheet.root_topic, 0)
+        if not lines:
+            continue
+        title = sheet.title or f"Sheet {index}"
+        rendered_sheets.append(f"# {title}\n\n" + "\n".join(lines))
+        topic_count += sheet_topics
+        note_count += sheet_notes
+
+    return (
+        "\n\n---\n\n".join(rendered_sheets),
+        len(rendered_sheets),
+        topic_count,
+        note_count,
+    )
+
+
+class XMindParser(BaseParser):
+    """Extract topic hierarchy and plain-text notes from XMind files."""
+
+    def parse_into_text(self, content: bytes) -> Document:
+        content_format, payload = _read_content_entry(content)
+
+        if content_format == "json":
+            try:
+                sheets = _parse_json_sheets(payload)
+            except (json.JSONDecodeError, UnicodeDecodeError) as exc:
+                raise ValueError("invalid XMind content.json") from exc
+        else:
+            sheets = _parse_xml_sheets(payload)
+
+        markdown, sheet_count, topic_count, note_count = _render_sheets(
+            sheets
+        )
+        if not markdown:
+            raise ValueError("XMind archive contains no renderable topics")
+        return Document(
+            content=markdown,
+            metadata={
+                "source_format": "xmind",
+                "xmind_content_format": content_format,
+                "file_size": len(content),
+                "sheet_count": sheet_count,
+                "topic_count": topic_count,
+                "note_count": note_count,
+            },
+        )