WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

docreader/tests/fixtures/generate_issue_2634_fixture.py

@@ -0,0 +1,57 @@
+"""Generate the real DOCX regression fixture for GitHub issue #2634."""
+
+from pathlib import Path
+
+from docx import Document
+from docx.enum.table import WD_CELL_VERTICAL_ALIGNMENT
+
+OUTPUT = Path(__file__).with_name("issue_2634_vertical_merge.docx")
+
+
+def main() -> None:
+    document = Document()
+    document.add_heading("遗传病检测项目表", level=1)
+    document.add_paragraph(
+        "回归场景：检测方法纵向合并，但每条检测项目在转换后都必须保留该方法。"
+    )
+
+    rows = [
+        ("Q0101", "遗传性乳腺癌", "BRCA1", "15个工作日"),
+        ("Q0102", "遗传性卵巢癌", "BRCA2", "15个工作日"),
+        ("Q0103", "林奇综合征", "MLH1", "15个工作日"),
+        ("Q0104", "家族性腺瘤性息肉病", "APC", "15个工作日"),
+    ]
+    method = (
+        "检测方法：提取外周血基因组 DNA，采用高通量测序，并对候选位点进行"
+        "Sanger 测序验证；检测结果需结合临床表现综合判断。"
+    )
+
+    table = document.add_table(rows=len(rows) + 1, cols=5)
+    table.style = "Table Grid"
+    headers = ("项目编号", "检测项目", "相关基因", "周期", "检测方法")
+    for column, value in enumerate(headers):
+        table.cell(0, column).text = value
+
+    for row_index, row in enumerate(rows, start=1):
+        for column, value in enumerate(row):
+            table.cell(row_index, column).text = value
+
+    merged_method = table.cell(1, 4).merge(table.cell(len(rows), 4))
+    merged_method.text = method
+    merged_method.vertical_alignment = WD_CELL_VERTICAL_ALIGNMENT.CENTER
+
+    document.add_paragraph("非合并对照表：相邻单元格内容允许相同，但不应被当作合并。")
+    control = document.add_table(rows=2, cols=3)
+    control.style = "Table Grid"
+    for column, value in enumerate(("列A", "列B", "列C")):
+        control.cell(0, column).text = value
+    for column, value in enumerate(("相同值", "相同值", "独立值")):
+        control.cell(1, column).text = value
+
+    document.save(OUTPUT)
+    print(f"generated={OUTPUT}")
+    print(f"size={OUTPUT.stat().st_size}")
+
+
+if __name__ == "__main__":
+    main()

docreader/tests/test_config.py

@@ -0,0 +1,43 @@
+import os
+import unittest
+from unittest.mock import patch
+
+from docreader import config
+
+
+class DocReaderConfigTest(unittest.TestCase):
+    def test_parser_concurrency_defaults_are_conservative(self):
+        with patch.dict(os.environ, {}, clear=True):
+            cfg = config.load_config()
+
+        self.assertEqual(cfg.markitdown_max_workers, 1)
+        self.assertEqual(cfg.pdf_render_max_workers, 1)
+        self.assertEqual(cfg.pdf_render_dpi, 200)
+        self.assertEqual(cfg.pdf_jpeg_quality, 85)
+
+    def test_loads_parser_concurrency_env(self):
+        env = {
+            "DOCREADER_MARKITDOWN_MAX_WORKERS": "3",
+            "DOCREADER_PDF_RENDER_MAX_WORKERS": "2",
+            "DOCREADER_PDF_RENDER_DPI": "180",
+            "DOCREADER_PDF_JPEG_QUALITY": "85",
+        }
+        with patch.dict(os.environ, env):
+            cfg = config.load_config()
+
+        self.assertEqual(cfg.markitdown_max_workers, 3)
+        self.assertEqual(cfg.pdf_render_max_workers, 2)
+        self.assertEqual(cfg.pdf_render_dpi, 180)
+        self.assertEqual(cfg.pdf_jpeg_quality, 85)
+
+    def test_dump_config_includes_parser_limits(self):
+        dumped = config.dump_config()
+
+        self.assertIn("DOCREADER_MARKITDOWN_MAX_WORKERS", dumped)
+        self.assertIn("DOCREADER_PDF_RENDER_MAX_WORKERS", dumped)
+        self.assertIn("DOCREADER_PDF_RENDER_DPI", dumped)
+        self.assertIn("DOCREADER_PDF_JPEG_QUALITY", dumped)
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_docx_merge.py

@@ -0,0 +1,187 @@
+import io
+import unittest
+import zipfile
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from docx import Document as WordDocument
+from lxml import etree
+
+from docreader.parser.docx2_parser import Docx2Parser
+from docreader.parser.docx_merge import fill_vertical_merged_cells_docx
+from docreader.parser.markitdown_parser import StdMarkitdownParser
+
+FIXTURE = Path(__file__).with_name("fixtures") / "issue_2634_vertical_merge.docx"
+METHOD_PREFIX = "检测方法：提取外周血基因组 DNA"
+PROJECT_CODES = ("Q0101", "Q0102", "Q0103", "Q0104")
+WORD_NAMESPACE = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+
+
+class DocxVerticalMergeFillTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.fixture_bytes = FIXTURE.read_bytes()
+
+    def test_fixture_is_a_real_docx_with_vertical_merge(self):
+        self.assertTrue(zipfile.is_zipfile(io.BytesIO(self.fixture_bytes)))
+        with zipfile.ZipFile(io.BytesIO(self.fixture_bytes)) as archive:
+            self.assertIsNone(archive.testzip())
+            root = etree.fromstring(archive.read("word/document.xml"))
+
+        vertical_merges = root.xpath(
+            ".//w:tbl[1]//w:vMerge", namespaces={"w": WORD_NAMESPACE}
+        )
+        merge_values = [
+            node.get(f"{{{WORD_NAMESPACE}}}val", "continue") for node in vertical_merges
+        ]
+        self.assertEqual(
+            merge_values,
+            ["restart", "continue", "continue", "continue"],
+        )
+
+    def test_fill_repeats_master_content_in_every_covered_row(self):
+        filled = fill_vertical_merged_cells_docx(self.fixture_bytes)
+        document = WordDocument(io.BytesIO(filled))
+        table = document.tables[0]
+
+        for row_index, project_code in enumerate(PROJECT_CODES, start=1):
+            self.assertEqual(table.cell(row_index, 0).text, project_code)
+            self.assertIn(METHOD_PREFIX, table.cell(row_index, 4).text)
+
+        with zipfile.ZipFile(io.BytesIO(filled)) as archive:
+            root = etree.fromstring(archive.read("word/document.xml"))
+        self.assertEqual(
+            root.xpath("count(.//w:vMerge)", namespaces={"w": WORD_NAMESPACE}),
+            0.0,
+        )
+
+    def test_non_merged_equal_cells_are_not_collapsed(self):
+        filled = fill_vertical_merged_cells_docx(self.fixture_bytes)
+        document = WordDocument(io.BytesIO(filled))
+        control_row = document.tables[1].rows[1]
+        self.assertEqual(len(control_row.cells), 3)
+        self.assertEqual(
+            [cell.text for cell in control_row.cells],
+            ["相同值", "相同值", "独立值"],
+        )
+
+    def test_non_docx_bytes_pass_through_unchanged(self):
+        content = b"not a docx package"
+        self.assertIs(fill_vertical_merged_cells_docx(content), content)
+
+    def test_docx_without_vertical_merges_passes_through_unchanged(self):
+        document = WordDocument()
+        table = document.add_table(rows=1, cols=2)
+        table.cell(0, 0).text = "left"
+        table.cell(0, 1).text = "right"
+        output = io.BytesIO()
+        document.save(output)
+        content = output.getvalue()
+
+        self.assertIs(fill_vertical_merged_cells_docx(content), content)
+
+    def test_horizontal_merge_only_passes_through_unchanged(self):
+        document = WordDocument()
+        table = document.add_table(rows=1, cols=3)
+        table.cell(0, 0).merge(table.cell(0, 1)).text = "horizontal merge"
+        table.cell(0, 2).text = "independent"
+        output = io.BytesIO()
+        document.save(output)
+        content = output.getvalue()
+
+        self.assertIs(fill_vertical_merged_cells_docx(content), content)
+
+    def test_separate_merge_groups_keep_distinct_values_and_other_columns(self):
+        document = WordDocument()
+        table = document.add_table(rows=6, cols=2)
+        table.cell(0, 0).text = "group"
+        table.cell(0, 1).text = "item"
+        for row_index in range(1, 6):
+            table.cell(row_index, 1).text = f"item-{row_index}"
+
+        table.cell(1, 0).merge(table.cell(2, 0)).text = "group-a"
+        table.cell(3, 0).text = "standalone"
+        table.cell(4, 0).merge(table.cell(5, 0)).text = "group-b"
+
+        output = io.BytesIO()
+        document.save(output)
+        filled = fill_vertical_merged_cells_docx(output.getvalue())
+        parsed_table = WordDocument(io.BytesIO(filled)).tables[0]
+
+        self.assertEqual(
+            [parsed_table.cell(row_index, 0).text for row_index in range(1, 6)],
+            ["group-a", "group-a", "standalone", "group-b", "group-b"],
+        )
+        self.assertEqual(
+            [parsed_table.cell(row_index, 1).text for row_index in range(1, 6)],
+            [f"item-{row_index}" for row_index in range(1, 6)],
+        )
+
+
+class DocxMarkitdownDispatchTest(unittest.TestCase):
+    def test_non_docx_types_do_not_run_docx_preprocessor(self):
+        for file_type in ("doc", "txt", "md"):
+            with self.subTest(file_type=file_type):
+                parser = StdMarkitdownParser(file_type=file_type)
+                result = SimpleNamespace(text_content="parsed")
+                with (
+                    patch(
+                        "docreader.parser.markitdown_parser.fill_vertical_merged_cells_docx"
+                    ) as preprocess,
+                    patch.object(parser, "_convert_markitdown", return_value=result),
+                ):
+                    document = parser.parse_into_text(b"payload")
+
+                self.assertEqual(document.content, "parsed")
+                preprocess.assert_not_called()
+
+    def test_uppercase_docx_type_runs_docx_preprocessor(self):
+        parser = StdMarkitdownParser(file_type=".DOCX")
+        result = SimpleNamespace(text_content="parsed")
+        with (
+            patch(
+                "docreader.parser.markitdown_parser.fill_vertical_merged_cells_docx",
+                return_value=b"preprocessed",
+            ) as preprocess,
+            patch.object(parser, "_convert_markitdown", return_value=result) as convert,
+        ):
+            document = parser.parse_into_text(b"payload")
+
+        self.assertEqual(document.content, "parsed")
+        preprocess.assert_called_once_with(b"payload")
+        convert.assert_called_once_with(b"preprocessed", ".DOCX", keep_data_uris=True)
+
+
+class Issue2634ParserRegressionTest(unittest.TestCase):
+    def test_markitdown_output_repeats_merged_value_for_every_data_row(self):
+        document = StdMarkitdownParser(
+            file_name=FIXTURE.name, file_type="docx"
+        ).parse_into_text(FIXTURE.read_bytes())
+
+        data_lines = {
+            project_code: next(
+                line for line in document.content.splitlines() if project_code in line
+            )
+            for project_code in PROJECT_CODES
+        }
+        for line in data_lines.values():
+            self.assertIn(METHOD_PREFIX, line)
+        self.assertEqual(document.content.count(METHOD_PREFIX), len(PROJECT_CODES))
+        self.assertIn("| 相同值 | 相同值 | 独立值 |", document.content)
+
+    def test_full_docx_entry_keeps_context_on_all_four_rows(self):
+        document = Docx2Parser(
+            file_name=FIXTURE.name, file_type="docx"
+        ).parse_into_text(FIXTURE.read_bytes())
+
+        for project_code in PROJECT_CODES:
+            data_line = next(
+                line for line in document.content.splitlines() if project_code in line
+            )
+            self.assertIn(METHOD_PREFIX, data_line)
+        self.assertEqual(document.content.count(METHOD_PREFIX), len(PROJECT_CODES))
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_docx_tables.py

@@ -0,0 +1,213 @@
+import importlib.util
+import io
+import sys
+import types
+import unittest
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from unittest.mock import patch
+
+from docx import Document as WordDocument
+
+
+def _load_docx_parser():
+    """Load docx_parser without triggering the heavy package __init__.
+
+    ``docreader/parser/__init__.py`` imports doc_parser -> textract, a heavy
+    dependency unrelated to DOCX parsing. Registering the packages as bare
+    namespaces lets us import only the modules docx_parser actually needs.
+    """
+    root = Path(__file__).resolve().parents[2]
+    docreader_pkg = types.ModuleType("docreader")
+    docreader_pkg.__path__ = [str(root / "docreader")]
+    sys.modules.setdefault("docreader", docreader_pkg)
+    parser_pkg = types.ModuleType("docreader.parser")
+    parser_pkg.__path__ = [str(root / "docreader" / "parser")]
+    sys.modules["docreader.parser"] = parser_pkg
+
+    spec = importlib.util.spec_from_file_location(
+        "docreader.parser.docx_parser", root / "docreader" / "parser" / "docx_parser.py"
+    )
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["docreader.parser.docx_parser"] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+docx_parser = _load_docx_parser()
+DocxParser = docx_parser.DocxParser
+table_to_gfm_markdown = docx_parser.table_to_gfm_markdown
+
+
+def _parse(content):
+    """Parse a DOCX through the real Docx processor.
+
+    The production path uses a ProcessPoolExecutor backed by a multiprocessing
+    Manager. Some CI sandboxes forbid POSIX semaphores, so run the same page
+    task pool on threads and swap the manager for a plain list - parse
+    behavior is identical, only the execution backend changes.
+    """
+
+    class _FakeManager:
+        def __enter__(self):
+            self._items = []
+            return self
+
+        def __exit__(self, *exc):
+            return False
+
+        def list(self):
+            return self._items
+
+    with patch.object(docx_parser, "Manager", _FakeManager), patch.object(
+        docx_parser, "ProcessPoolExecutor", ThreadPoolExecutor
+    ):
+        return DocxParser(max_pages=100).parse_into_text(content)
+
+
+def _docx_bytes(build):
+    doc = WordDocument()
+    build(doc)
+    buf = io.BytesIO()
+    doc.save(buf)
+    return buf.getvalue()
+
+
+class DocxTableContentTest(unittest.TestCase):
+    """Regression test: DOCX tables must be kept in the parsed text.
+
+    Docx.__call__ used to return tables separately from the text lines, and
+    parse_into_text dropped them, silently losing all table content.
+    """
+
+    def _docx_with_table(self, cells):
+        def build(doc):
+            doc.add_paragraph("Introduction paragraph")
+            table = doc.add_table(rows=len(cells), cols=len(cells[0]))
+            for r, row in enumerate(cells):
+                for c, value in enumerate(row):
+                    table.cell(r, c).text = value
+
+        return _docx_bytes(build)
+
+    def test_table_content_is_kept_in_parsed_text(self):
+        content = self._docx_with_table(
+            [["City", "Population"], ["Beijing", "21.5M"]]
+        )
+        document = _parse(content)
+        self.assertIn("Introduction paragraph", document.content)
+        for cell in ("City", "Population", "Beijing", "21.5M"):
+            self.assertIn(cell, document.content)
+        self.assertIn("| City | Population |", document.content)
+        self.assertIn("| --- | --- |", document.content)
+
+    def test_table_only_document_is_not_empty(self):
+        def build(doc):
+            table = doc.add_table(rows=2, cols=1)
+            table.cell(0, 0).text = "Header"
+            table.cell(1, 0).text = "Value"
+
+        document = _parse(_docx_bytes(build))
+        self.assertIn("Header", document.content)
+        self.assertIn("Value", document.content)
+
+    def test_pipe_in_cell_does_not_break_table(self):
+        def build(doc):
+            table = doc.add_table(rows=2, cols=1)
+            table.cell(0, 0).text = "A|B"
+            table.cell(1, 0).text = "C"
+
+        document = _parse(_docx_bytes(build))
+        self.assertIn(r"A\|B", document.content)
+
+    def test_empty_adjacent_cells_keep_columns(self):
+        def build(doc):
+            table = doc.add_table(rows=2, cols=3)
+            table.cell(0, 0).text = "A"
+            table.cell(0, 1).text = "B"
+            table.cell(0, 2).text = "C"
+            table.cell(1, 0).text = "1"
+            table.cell(1, 1).text = ""
+            table.cell(1, 2).text = ""
+
+        document = _parse(_docx_bytes(build))
+        self.assertIn("| A | B | C |", document.content)
+        self.assertIn("| 1 |  |  |", document.content)
+
+    def test_equal_adjacent_cells_are_not_collapsed(self):
+        """Adjacent independent cells with the same text must stay separate.
+
+        Regression for the #2634 control row: ``相同值 | 相同值 | 独立值``.
+        """
+
+        def build(doc):
+            table = doc.add_table(rows=2, cols=3)
+            table.cell(0, 0).text = "相同值"
+            table.cell(0, 1).text = "相同值"
+            table.cell(0, 2).text = "独立值"
+            table.cell(1, 0).text = "x"
+            table.cell(1, 1).text = "y"
+            table.cell(1, 2).text = "z"
+
+        document = _parse(_docx_bytes(build))
+        self.assertIn("| 相同值 | 相同值 | 独立值 |", document.content)
+        self.assertIn("| x | y | z |", document.content)
+
+    def test_horizontal_merge_keeps_grid_width(self):
+        def build(doc):
+            table = doc.add_table(rows=2, cols=3)
+            table.cell(0, 0).merge(table.cell(0, 1)).text = "merged"
+            table.cell(0, 2).text = "right"
+            table.cell(1, 0).text = "a"
+            table.cell(1, 1).text = "b"
+            table.cell(1, 2).text = "c"
+
+        document = _parse(_docx_bytes(build))
+        self.assertIn("| merged | merged | right |", document.content)
+        self.assertIn("| a | b | c |", document.content)
+
+    def test_table_stays_between_paragraphs(self):
+        def build(doc):
+            doc.add_paragraph("before table")
+            table = doc.add_table(rows=2, cols=1)
+            table.cell(0, 0).text = "H"
+            table.cell(1, 0).text = "V"
+            doc.add_paragraph("after table")
+
+        document = _parse(_docx_bytes(build))
+        before = document.content.index("before table")
+        header = document.content.index("| H |")
+        after = document.content.index("after table")
+        self.assertLess(before, header)
+        self.assertLess(header, after)
+
+    def test_html_like_cell_text_is_literal(self):
+        def build(doc):
+            table = doc.add_table(rows=1, cols=2)
+            table.cell(0, 0).text = "a < b"
+            table.cell(0, 1).text = "</td><td>injected"
+
+        document = _parse(_docx_bytes(build))
+        self.assertIn("| a < b | </td><td>injected |", document.content)
+
+    def test_table_to_gfm_markdown_helper(self):
+        doc = WordDocument()
+        table = doc.add_table(rows=2, cols=2)
+        table.cell(0, 0).text = "A"
+        table.cell(0, 1).text = "B"
+        table.cell(1, 0).text = "C"
+        table.cell(1, 1).text = "D"
+        self.assertEqual(
+            table_to_gfm_markdown(table),
+            "| A | B |\n| --- | --- |\n| C | D |",
+        )
+
+    def test_table_to_gfm_markdown_skips_empty(self):
+        class _Empty:
+            rows = []
+
+        self.assertEqual(table_to_gfm_markdown(_Empty()), "")
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_epub_parser.py

@@ -0,0 +1,92 @@
+import os
+import tempfile
+import unittest
+
+from ebooklib import epub
+
+from docreader.parser.epub_parser import EPUBParser
+from docreader.parser.registry import registry
+
+
+def _minimal_epub_bytes() -> bytes:
+    book = epub.EpubBook()
+    book.set_identifier("test-epub")
+    book.set_title("Tiny EPUB")
+    book.set_language("en")
+    book.add_author("WeKnora")
+
+    chapter = epub.EpubHtml(
+        title="Chapter One", file_name="text/chapter_01.xhtml", lang="en"
+    )
+    chapter.content = (
+        "<html><body><h1>Chapter One</h1>"
+        "<p>Hello EPUB world.</p>"
+        '<p><a href="chapter_02.xhtml#sec2">Chapter 2</a> '
+        '<a href="#footnote1">note</a> '
+        '<a href="https://example.com">the site</a></p>'
+        '<img alt="cover" src="../images/pic.png">'
+        "</body></html>"
+    )
+    book.add_item(chapter)
+    book.add_item(
+        epub.EpubItem(
+            uid="pic",
+            file_name="images/pic.png",
+            media_type="image/png",
+            content=b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR",
+        )
+    )
+    book.toc = (epub.Link("text/chapter_01.xhtml", "Chapter One", "chapter-one"),)
+    book.spine = ["nav", chapter]
+    book.add_item(epub.EpubNcx())
+    book.add_item(epub.EpubNav())
+
+    with tempfile.NamedTemporaryFile(suffix=".epub", delete=False) as handle:
+        path = handle.name
+    try:
+        epub.write_epub(path, book)
+        with open(path, "rb") as handle:
+            return handle.read()
+    finally:
+        if os.path.exists(path):
+            os.unlink(path)
+
+
+class EPUBParserTest(unittest.TestCase):
+    def test_parse_minimal_epub(self):
+        document = EPUBParser(
+            file_name="tiny.epub", file_type="epub"
+        ).parse_into_text(_minimal_epub_bytes())
+
+        self.assertIn("Hello EPUB world", document.content)
+        self.assertEqual(document.metadata["source_format"], "epub")
+        self.assertEqual(len(document.images), 1)
+        image_ref = next(iter(document.images))
+        self.assertTrue(image_ref.startswith("images/"))
+        self.assertIn(image_ref, document.content)
+        self.assertNotIn("../images/pic.png", document.content)
+
+    def test_internal_links_are_unwrapped_but_external_links_remain(self):
+        document = EPUBParser(
+            file_name="tiny.epub", file_type="epub"
+        ).parse_into_text(_minimal_epub_bytes())
+
+        self.assertIn("Chapter 2", document.content)
+        self.assertIn("note", document.content)
+        self.assertNotIn("chapter_02.xhtml#sec2", document.content)
+        self.assertNotIn("#footnote1", document.content)
+        self.assertIn("[the site](https://example.com)", document.content)
+
+    def test_parse_without_images(self):
+        document = EPUBParser(
+            file_name="tiny.epub", file_type="epub", extract_images=False
+        ).parse_into_text(_minimal_epub_bytes())
+
+        self.assertEqual(document.images, {})
+
+    def test_registry_resolves_epub(self):
+        self.assertIs(registry.get_parser_class("", "epub"), EPUBParser)
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_excel_parser.py

@@ -0,0 +1,513 @@
+import io
+import os
+import shutil
+import subprocess
+import tempfile
+import unittest
+import zipfile
+from unittest.mock import patch
+
+import openpyxl
+import pandas as pd
+from openpyxl.chart import BarChart, Reference
+
+from docreader.parser.excel_convert import detect_excel_format, engine_for_format
+from docreader.parser.excel_parser import ExcelParser
+from docreader.parser.xlsx_merge import fill_merged_cells_xlsx
+from docreader.parser.xlsx_repair import repair_xlsx_bytes
+
+
+def _xlsx_with_phantom_shared_strings() -> bytes:
+    """Workbook with inline strings but a dangling sharedStrings manifest entry."""
+    wb = openpyxl.Workbook()
+    ws = wb.active
+    ws["A1"] = "hello"
+    ws["B1"] = 42
+    bio = io.BytesIO()
+    wb.save(bio)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        with zipfile.ZipFile(io.BytesIO(bio.getvalue()), "r") as zin:
+            zin.extractall(tmpdir)
+
+        ct_path = f"{tmpdir}/[Content_Types].xml"
+        with open(ct_path, encoding="utf-8") as f:
+            ct = f.read()
+        override = (
+            '<Override PartName="/xl/sharedStrings.xml" '
+            'ContentType="application/vnd.openxmlformats-officedocument.'
+            'spreadsheetml.sharedStrings+xml"/>'
+        )
+        with open(ct_path, "w", encoding="utf-8") as f:
+            f.write(ct.replace("</Types>", override + "</Types>"))
+
+        out = io.BytesIO()
+        with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zout:
+            for root, _, files in os.walk(tmpdir):
+                for name in files:
+                    path = os.path.join(root, name)
+                    arc = os.path.relpath(path, tmpdir)
+                    zout.write(path, arc)
+        return out.getvalue()
+
+
+class ExcelFormatDetectionTest(unittest.TestCase):
+    def test_detect_xlsx_and_engine(self):
+        wb = openpyxl.Workbook()
+        bio = io.BytesIO()
+        wb.save(bio)
+        content = bio.getvalue()
+        self.assertEqual(detect_excel_format(content), "xlsx")
+        self.assertEqual(engine_for_format("xlsx"), "openpyxl")
+
+    def test_detect_xls_magic(self):
+        content = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 512
+        self.assertEqual(detect_excel_format(content), "xls")
+        self.assertEqual(engine_for_format("xls"), "xlrd")
+
+    def test_open_legacy_xls_bytes_with_xlsx_extension(self):
+        if not shutil.which("soffice"):
+            self.skipTest("LibreOffice not available")
+        wb = openpyxl.Workbook()
+        ws = wb.active
+        ws["A1"] = "legacy"
+        xlsx_bio = io.BytesIO()
+        wb.save(xlsx_bio)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            src = os.path.join(tmpdir, "sheet.xlsx")
+            with open(src, "wb") as handle:
+                handle.write(xlsx_bio.getvalue())
+            subprocess.run(
+                [
+                    "soffice",
+                    "--headless",
+                    "--convert-to",
+                    "xls",
+                    "--outdir",
+                    tmpdir,
+                    src,
+                ],
+                check=True,
+                capture_output=True,
+            )
+            xls_path = os.path.join(tmpdir, "sheet.xls")
+            with open(xls_path, "rb") as handle:
+                xls_bytes = handle.read()
+
+        document = ExcelParser(file_name="fake.xlsx", file_type="xlsx").parse_into_text(
+            xls_bytes
+        )
+        self.assertIn("legacy", document.content)
+
+
+class XlsxRepairTest(unittest.TestCase):
+    def test_repair_removes_phantom_shared_strings_reference(self):
+        broken = _xlsx_with_phantom_shared_strings()
+        with self.assertRaises(KeyError):
+            pd.read_excel(io.BytesIO(broken))
+
+        repaired = repair_xlsx_bytes(broken)
+        self.assertIsNotNone(repaired)
+        df = pd.read_excel(io.BytesIO(repaired), header=None)
+        self.assertEqual(df.values.tolist(), [["hello", 42]])
+
+    def test_repair_skips_when_shared_string_cells_need_table(self):
+        import xlsxwriter
+
+        bio = io.BytesIO()
+        wb = xlsxwriter.Workbook(bio, {"in_memory": True})
+        ws = wb.add_worksheet()
+        ws.write(0, 0, "hello")
+        wb.close()
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            with zipfile.ZipFile(io.BytesIO(bio.getvalue()), "r") as zin:
+                zin.extractall(tmpdir)
+            os.remove(f"{tmpdir}/xl/sharedStrings.xml")
+
+            out = io.BytesIO()
+            with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zout:
+                for root, _, files in os.walk(tmpdir):
+                    for name in files:
+                        path = os.path.join(root, name)
+                        arc = os.path.relpath(path, tmpdir)
+                        zout.write(path, arc)
+            broken = out.getvalue()
+
+        self.assertIsNone(repair_xlsx_bytes(broken))
+
+
+class XlsxMergeFillTest(unittest.TestCase):
+    @staticmethod
+    def _workbook_with_merged_cells_and_chart() -> bytes:
+        wb = openpyxl.Workbook()
+        ws = wb.active
+        ws["A1"] = "title"
+        ws.merge_cells("A1:B1")
+        ws.append(["category", "value"])
+        ws.append(["one", 1])
+        ws.append(["two", 2])
+
+        chart = BarChart()
+        chart.add_data(
+            Reference(ws, min_col=2, min_row=2, max_row=4),
+            titles_from_data=True,
+        )
+        chart.set_categories(Reference(ws, min_col=1, min_row=3, max_row=4))
+        ws.add_chart(chart, "D2")
+
+        bio = io.BytesIO()
+        wb.save(bio)
+        return bio.getvalue()
+
+    def test_fill_merged_cells_propagates_master_value(self):
+        wb = openpyxl.Workbook()
+        ws = wb.active
+        ws["A1"] = "title"
+        ws.merge_cells("A1:B1")
+        ws["A2"] = "left"
+        ws["B2"] = "right"
+        ws.merge_cells("A2:A3")
+        ws["B3"] = "only-b"
+        bio = io.BytesIO()
+        wb.save(bio)
+
+        filled = fill_merged_cells_xlsx(bio.getvalue())
+        out_wb = openpyxl.load_workbook(io.BytesIO(filled), data_only=True)
+        out_ws = out_wb.active
+        self.assertEqual(out_ws["B1"].value, "title")
+        self.assertEqual(out_ws["A3"].value, "left")
+        self.assertEqual(out_ws["B3"].value, "only-b")
+
+    def test_fill_merged_cells_does_not_serialize_charts(self):
+        content = self._workbook_with_merged_cells_and_chart()
+
+        with patch(
+            "openpyxl.chart._chart.ChartBase._write",
+            side_effect=AttributeError("'Typed' object has no attribute 'to_tree'"),
+        ):
+            filled = fill_merged_cells_xlsx(content)
+
+        out_wb = openpyxl.load_workbook(io.BytesIO(filled), data_only=True)
+        out_ws = out_wb.active
+        self.assertEqual(out_ws["B1"].value, "title")
+        self.assertEqual(out_ws["A3"].value, "one")
+        self.assertEqual(out_ws["B4"].value, 2)
+        self.assertEqual(out_ws._charts, [])
+
+    def test_excel_parser_reads_merged_workbook_with_chart(self):
+        document = ExcelParser().parse_into_text(
+            self._workbook_with_merged_cells_and_chart()
+        )
+
+        self.assertIn("A: title,B: title", document.content)
+        self.assertIn("A: one,B: 1", document.content)
+        self.assertIn("A: two,B: 2", document.content)
+
+    def test_chart_workbook_without_merges_passes_through_unchanged(self):
+        content = self._workbook_with_merged_cells_and_chart()
+        wb = openpyxl.load_workbook(io.BytesIO(content))
+        wb.active.unmerge_cells("A1:B1")
+        bio = io.BytesIO()
+        wb.save(bio)
+        content = bio.getvalue()
+
+        self.assertEqual(fill_merged_cells_xlsx(content), content)
+
+    def test_fill_does_not_serialize_dedicated_chart_sheets(self):
+        content = self._workbook_with_merged_cells_and_chart()
+        wb = openpyxl.load_workbook(io.BytesIO(content))
+        ws = wb.active
+        chart = BarChart()
+        chart.add_data(
+            Reference(ws, min_col=2, min_row=2, max_row=4),
+            titles_from_data=True,
+        )
+        chart.set_categories(Reference(ws, min_col=1, min_row=3, max_row=4))
+        chart_sheet = wb.create_chartsheet("Summary")
+        chart_sheet.add_chart(chart)
+        wb.active = chart_sheet
+        bio = io.BytesIO()
+        wb.save(bio)
+
+        with patch(
+            "openpyxl.chart._chart.ChartBase._write",
+            side_effect=AttributeError("'Typed' object has no attribute 'to_tree'"),
+        ):
+            filled = fill_merged_cells_xlsx(bio.getvalue())
+
+        out_wb = openpyxl.load_workbook(io.BytesIO(filled), data_only=True)
+        self.assertEqual(out_wb.active._charts, [])
+        self.assertEqual(out_wb.active.title, "Sheet")
+        self.assertEqual(out_wb.chartsheets, [])
+
+    def test_fill_keeps_active_worksheet_after_removing_earlier_chart_sheet(self):
+        content = self._workbook_with_merged_cells_and_chart()
+        wb = openpyxl.load_workbook(io.BytesIO(content))
+        data_sheet = wb.active
+        chart = BarChart()
+        chart.add_data(
+            Reference(data_sheet, min_col=2, min_row=2, max_row=4),
+            titles_from_data=True,
+        )
+        chart_sheet = wb.create_chartsheet("Summary", 0)
+        chart_sheet.add_chart(chart)
+        wb.active = data_sheet
+        bio = io.BytesIO()
+        wb.save(bio)
+
+        filled = fill_merged_cells_xlsx(bio.getvalue())
+
+        out_wb = openpyxl.load_workbook(io.BytesIO(filled), data_only=True)
+        self.assertEqual(out_wb.active.title, "Sheet")
+        self.assertEqual(out_wb.chartsheets, [])
+
+    def test_parse_en_mergecell_workbook(self):
+        path = os.path.join(
+            os.path.dirname(__file__),
+            "..",
+            "..",
+            "testdata",
+            "rag_test",
+            "xlsx",
+            "en_mergecell.xlsx",
+        )
+        if not os.path.isfile(path):
+            self.skipTest("en_mergecell.xlsx fixture not available")
+        with open(path, "rb") as handle:
+            document = ExcelParser().parse_into_text(handle.read())
+
+        chunks = [chunk.content.strip() for chunk in document.chunks]
+        self.assertEqual(len(chunks), 12)
+        self.assertIn("A: A1", chunks[0])
+        self.assertIn("A: A2", chunks[1])
+        self.assertIn("B: B3", chunks[2])
+        self.assertNotIn("Unnamed:", document.content)
+        self.assertIn("A: A7", chunks[6])
+        self.assertIn("A: A7", chunks[7])
+        self.assertIn("D: D10", chunks[9])
+
+
+class ExcelImageFilterTest(unittest.TestCase):
+    """Tests for filtering embedded image function strings (#1779)."""
+
+    def _xlsx_with_image_functions(self) -> bytes:
+        """Create an XLSX where image functions are stored as text values.
+
+        WPS embeds images using =DISPIMG("ID",1) which may appear as plain
+        text (not a formula) in some export scenarios.
+        """
+        wb = openpyxl.Workbook()
+        ws = wb.active
+        ws["A1"] = "Name"
+        ws["B1"] = "Photo"
+        ws["A2"] = "Alice"
+        ws["B2"] = '_xlfn.DISPIMG("ID_ABCDEF123",1)'
+        ws["A3"] = "Bob"
+        ws["B3"] = '_xlfn.DISPIMG("ID_GHIJKL456",1)'
+        ws["A4"] = "Charlie"
+        ws["B4"] = "real data"
+        bio = io.BytesIO()
+        wb.save(bio)
+        return bio.getvalue()
+
+    def test_dispimg_text_values_are_excluded(self):
+        """Image function strings stored as text must not appear in output."""
+        document = ExcelParser().parse_into_text(self._xlsx_with_image_functions())
+        self.assertNotIn("DISPIMG", document.content)
+        self.assertNotIn("_xlfn", document.content)
+        # Real data must still be present
+        self.assertIn("Alice", document.content)
+        self.assertIn("Bob", document.content)
+        self.assertIn("Charlie", document.content)
+        self.assertIn("real data", document.content)
+
+    def test_dispimg_with_equals_prefix(self):
+        """=_xlfn.DISPIMG(...) as text (not formula) should also be filtered."""
+        wb = openpyxl.Workbook()
+        ws = wb.active
+        ws["A1"] = "Name"
+        ws["B1"] = "Photo"
+        ws["A2"] = "Alice"
+        # Stored as text with = prefix (some exporters do this)
+        ws["B2"] = '=_xlfn.DISPIMG("ID_123",1)'
+        bio = io.BytesIO()
+        wb.save(bio)
+        # Note: openpyxl treats strings starting with = as formulas,
+        # so data_only=True in fill_merged_cells_xlsx will turn them to None.
+        # This test verifies the formula path also works correctly.
+        document = ExcelParser().parse_into_text(bio.getvalue())
+        self.assertNotIn("DISPIMG", document.content)
+        self.assertIn("Alice", document.content)
+
+    def test_image_function_variations(self):
+        """Various image function patterns should all be filtered."""
+        wb = openpyxl.Workbook()
+        ws = wb.active
+        ws["A1"] = '_xlfn.DISPIMG("ID_001",1)'
+        ws["A2"] = 'DISPIMG("ID_002",1)'  # no prefix at all
+        ws["A3"] = '_xlfn.IMAGE("https://example.com/img.png")'
+        ws["A4"] = 'IMAGE("https://example.com/img.png",1)'
+        ws["A5"] = "Normal text"
+        bio = io.BytesIO()
+        wb.save(bio)
+        document = ExcelParser().parse_into_text(bio.getvalue())
+        self.assertNotIn("DISPIMG", document.content)
+        self.assertNotIn("IMAGE(", document.content)
+        self.assertIn("Normal text", document.content)
+
+    def test_real_world_wps_dispimg(self):
+        """Exact pattern from issue #1779 screenshot: =DISPIMG("ID_...",1)."""
+        wb = openpyxl.Workbook()
+        ws = wb.active
+        ws["A1"] = "Product"
+        ws["B1"] = "Image"
+        ws["C1"] = "Price"
+        ws["A2"] = "Basic"
+        # This is the exact format shown in the issue screenshot
+        ws["B2"] = 'DISPIMG("ID_5A60F9ED501E48A38EBEE5D326E18235",1)'
+        ws["C2"] = "39.9"
+        ws["A3"] = "Pro"
+        ws["B3"] = 'DISPIMG("ID_AABBCCDD",1)'
+        ws["C3"] = "99.9"
+        bio = io.BytesIO()
+        wb.save(bio)
+        document = ExcelParser().parse_into_text(bio.getvalue())
+        self.assertNotIn("DISPIMG", document.content)
+        self.assertNotIn("ID_5A60F9ED", document.content)
+        self.assertIn("Basic", document.content)
+        self.assertIn("39.9", document.content)
+        self.assertIn("Pro", document.content)
+        self.assertIn("99.9", document.content)
+
+
+class ExcelParserTest(unittest.TestCase):
+    @staticmethod
+    def _workbook_bytes(rows):
+        wb = openpyxl.Workbook()
+        ws = wb.active
+        for row in rows:
+            ws.append(row)
+        bio = io.BytesIO()
+        wb.save(bio)
+        return bio.getvalue()
+
+    def test_xlsx_first_row_header_mode_repeats_column_context(self):
+        content = self._workbook_bytes(
+            [
+                ["Name", "Age", "City"],
+                ["Alice", 30, "Shenzhen"],
+                ["Bob", 28, "Shanghai"],
+            ]
+        )
+
+        document = ExcelParser(
+            file_name="people.xlsx",
+            file_type="xlsx",
+            xlsx_first_row_as_header="true",
+        ).parse_into_text(content)
+
+        chunks = [chunk.content.strip() for chunk in document.chunks]
+        self.assertEqual(
+            chunks,
+            [
+                "Name: Alice,Age: 30,City: Shenzhen",
+                "Name: Bob,Age: 28,City: Shanghai",
+            ],
+        )
+        self.assertNotIn("A: Name", document.content)
+
+    def test_xlsx_keeps_first_row_as_data_by_default(self):
+        content = self._workbook_bytes(
+            [
+                ["Name", "City"],
+                ["Alice", "Shenzhen"],
+            ]
+        )
+
+        document = ExcelParser(file_name="people.xlsx", file_type="xlsx").parse_into_text(
+            content
+        )
+
+        chunks = [chunk.content.strip() for chunk in document.chunks]
+        self.assertEqual(len(chunks), 2)
+        self.assertEqual(chunks[0], "A: Name,B: City")
+        self.assertEqual(chunks[1], "A: Alice,B: Shenzhen")
+
+    def test_single_row_xlsx_is_not_consumed_in_header_mode(self):
+        content = self._workbook_bytes([["Name", "Age", "City"]])
+
+        document = ExcelParser(
+            file_name="single.xlsx",
+            file_type="xlsx",
+            xlsx_first_row_as_header=True,
+        ).parse_into_text(content)
+
+        self.assertEqual(
+            [chunk.content.strip() for chunk in document.chunks],
+            ["A: Name,B: Age,C: City"],
+        )
+
+    def test_header_mode_generates_stable_labels_for_duplicate_and_empty_cells(self):
+        content = self._workbook_bytes(
+            [
+                ["Name", "Name", None],
+                ["Alice", "Alias", "Shenzhen"],
+            ]
+        )
+
+        document = ExcelParser(
+            file_name="people.xlsx",
+            file_type="xlsx",
+            xlsx_first_row_as_header="yes",
+        ).parse_into_text(content)
+
+        self.assertEqual(
+            [chunk.content.strip() for chunk in document.chunks],
+            ["Name: Alice,Name_2: Alias,C: Shenzhen"],
+        )
+
+    def test_xlsx_explicit_false_override_keeps_first_row_as_data(self):
+        content = self._workbook_bytes(
+            [
+                ["Name", "Age"],
+                ["Alice", 30],
+            ]
+        )
+
+        document = ExcelParser(
+            file_name="people.xlsx",
+            file_type="xlsx",
+            xlsx_first_row_as_header="false",
+        ).parse_into_text(content)
+
+        chunks = [chunk.content.strip() for chunk in document.chunks]
+        self.assertEqual(chunks[0], "A: Name,B: Age")
+        self.assertEqual(chunks[1], "A: Alice,B: 30")
+
+    def test_parse_phantom_shared_strings_workbook(self):
+        document = ExcelParser().parse_into_text(_xlsx_with_phantom_shared_strings())
+        self.assertIn("hello", document.content)
+        self.assertIn("42", document.content)
+        self.assertGreater(len(document.chunks), 0)
+
+    def test_parse_en_calcchain_shared_strings_case(self):
+        path = os.path.join(
+            os.path.dirname(__file__),
+            "..",
+            "..",
+            "testdata",
+            "rag_test",
+            "xlsx",
+            "en_calcchain.xlsx",
+        )
+        if not os.path.isfile(path):
+            self.skipTest("en_calcchain.xlsx fixture not available")
+        with open(path, "rb") as f:
+            document = ExcelParser().parse_into_text(f.read())
+        self.assertGreater(len(document.content), 0)
+        self.assertGreater(len(document.chunks), 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_html_parser.py

@@ -0,0 +1,71 @@
+import unittest
+from pathlib import Path
+
+from docreader.parser.html_parser import HTMLParser
+from docreader.parser.registry import BUILTIN_ENGINE, registry
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+class HTMLParserTest(unittest.TestCase):
+    def test_parse_static_html_fixture_into_markdown(self):
+        content = (REPO_ROOT / "docreader" / "testdata" / "test.html").read_bytes()
+
+        document = HTMLParser(file_name="test.html", file_type="html").parse(content)
+
+        self.assertIn("# 测试 HTML 文档", document.content)
+        self.assertIn("[测试链接](https://example.com)", document.content)
+        self.assertIn("| 表头1 | 表头2 |", document.content)
+        self.assertIn("内容4", document.content)
+
+    def test_parse_empty_html_returns_empty_document(self):
+        document = HTMLParser(file_name="empty.htm", file_type="htm").parse(b"")
+
+        self.assertEqual("", document.content)
+
+    def test_parse_honors_declared_windows_1252_charset(self):
+        html = (
+            '<meta charset="windows-1252">'
+            "<h1>R\u00e9sum\u00e9 \u201cquoted\u201d \u20ac</h1>"
+        ).encode("windows-1252")
+
+        document = HTMLParser(file_name="legacy.html", file_type="html").parse(html)
+
+        self.assertIn("# R\u00e9sum\u00e9 \u201cquoted\u201d \u20ac", document.content)
+
+    def test_parse_honors_utf16_bom(self):
+        html = "<html><body><h1>UTF-16 title</h1></body></html>".encode("utf-16")
+
+        document = HTMLParser(file_name="utf16.html", file_type="html").parse(html)
+
+        self.assertIn("# UTF-16 title", document.content)
+        self.assertNotIn("\x00", document.content)
+
+    def test_parse_script_only_html_returns_empty_document(self):
+        html = (
+            b"<html><body><script>alert('x')</script><style>p{}</style></body></html>"
+        )
+
+        document = HTMLParser(file_name="script.html", file_type="html").parse(html)
+
+        self.assertEqual("", document.content)
+
+    def test_parse_preserves_relative_and_fragment_links(self):
+        html = b'<a href="./guide.html">Guide</a><a href="#section">Section</a>'
+
+        document = HTMLParser(file_name="links.html", file_type="html").parse(html)
+
+        self.assertIn("[Guide](./guide.html)", document.content)
+        self.assertIn("[Section](#section)", document.content)
+
+    def test_registry_routes_html_extensions_to_html_parser(self):
+        self.assertIs(registry.get_parser_class("", "html"), HTMLParser)
+        self.assertIs(registry.get_parser_class("", "htm"), HTMLParser)
+
+        builtin_types = registry._engines[BUILTIN_ENGINE]
+        self.assertIn("html", builtin_types)
+        self.assertIn("htm", builtin_types)
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_markdown_table_util.py

@@ -0,0 +1,109 @@
+import io
+import unittest
+from pathlib import Path
+
+from markitdown import MarkItDown
+
+from docreader.parser.markdown_parser import MarkdownTableUtil
+
+
+class TestMarkdownTableUtil(unittest.TestCase):
+    def test_preserves_empty_cells(self):
+        """Interior empty cells must not be dropped during formatting."""
+        raw = "| a |  | c |\n| --- | --- | --- |\n| 1 | 2 | 3 |"
+        formatted = MarkdownTableUtil().format_table(raw)
+        self.assertIn("| a |  | c |", formatted)
+        self.assertEqual(formatted.count("|"), raw.count("|"))
+
+    def test_format_nonempty_table(self):
+        raw = "|Name|Age|\n|---|---|\n|John|30|"
+        formatted = MarkdownTableUtil().format_table(raw)
+        self.assertIn("| Name | Age |", formatted)
+        self.assertIn("| --- | --- |", formatted)
+        self.assertIn("| John | 30 |", formatted)
+
+    def test_normalize_markitdown_en_tables(self):
+        docx = (
+            Path(__file__).resolve().parents[2]
+            / "testdata"
+            / "rag_test"
+            / "docx"
+            / "en_tables.docx"
+        )
+        if not docx.is_file():
+            docx = Path(__file__).resolve().parents[2].parent / "testdata/rag_test/docx/en_tables.docx"
+        if not docx.is_file():
+            self.skipTest("en_tables.docx fixture not available")
+        raw = MarkItDown().convert(io.BytesIO(docx.read_bytes()), file_extension=".docx").text_content
+        normalized = MarkdownTableUtil().format_table(raw)
+
+        self.assertNotIn("|  |  |  |  |", normalized)
+        self.assertIn("| Name | Game | Fame | Blame |", normalized)
+        idx_name = normalized.index("| Name | Game | Fame | Blame |")
+        idx_sep = normalized.index("| --- | --- | --- | --- |", idx_name)
+        self.assertLess(idx_name, idx_sep)
+        self.assertIn("| Lebron James | Basketball |", normalized)
+
+        # Headerless 2-row tables: delimiter inserted so GFM renderers show a table
+        self.assertIn(
+            "| Sinple | Table |\n| --- | --- |\n| Without | Header |", normalized
+        )
+        self.assertIn(
+            "| Simple  Multiparagraph | Table  Full |\n| --- | --- |\n"
+            "| Of  Paragraphs | In each  Cell. |",
+            normalized,
+        )
+
+    def test_malformed_unclosed_row_is_passthrough(self):
+        """An unclosed ``|``-prefixed row must not hang the process.
+
+        Regression for Tencent/WeKnora#2768: the old line_pattern explored an
+        exponential number of backtracking paths on such rows and stalled the
+        single-threaded docreader event loop.
+        """
+        line = "| " + " | ".join(f"col{i}" for i in range(100)) + "  unclosed"
+        formatted = MarkdownTableUtil().format_table(line)
+        self.assertEqual(formatted, line)
+
+    def test_long_valid_table_formats(self):
+        """A long, well-formed table still formats in linear time."""
+        header = "| " + " | ".join(f"c{i}" for i in range(200)) + " |"
+        sep = "| " + " | ".join("---" for _ in range(200)) + " |"
+        row = "| " + " | ".join(f"v{i}" for i in range(200)) + " |"
+        formatted = MarkdownTableUtil().format_table("\n".join([header, sep, row]))
+        self.assertIn("| c0 | c1 |", formatted)
+        self.assertIn("| c199 |", formatted)
+        self.assertIn("| --- |", formatted)
+        self.assertIn("| v0 | v1 |", formatted)
+        self.assertIn("| v199 |", formatted)
+
+    def test_lone_pipe_is_passthrough(self):
+        """A single-pipe line is not a table row and must not be deleted."""
+        self.assertEqual(MarkdownTableUtil().format_table("|"), "|")
+        self.assertEqual(MarkdownTableUtil().format_table("  |  "), "  |  ")
+
+    def test_lone_pipe_above_table_is_kept(self):
+        raw = "|\n| a | b |\n| --- | --- |\n| 1 | 2 |"
+        formatted = MarkdownTableUtil().format_table(raw)
+        self.assertEqual(formatted.split("\n")[0], "|")
+        self.assertIn("| a | b |", formatted)
+        self.assertIn("| 1 | 2 |", formatted)
+
+    def test_alignment_colons_preserved(self):
+        raw = "| a | b | c |\n| :---------- | -------: | :------: |\n| 1 | 2 | 3 |"
+        formatted = MarkdownTableUtil().format_table(raw)
+        self.assertIn("| :--- | ---: | :---: |", formatted)
+
+    def test_crlf_table_formats_without_mixed_endings(self):
+        raw = "# Title\r\n\r\n|Name|Age|\r\n|---|---|\r\n|John|30|\r\n\r\nparagraph\r\n"
+        formatted = MarkdownTableUtil().format_table(raw)
+        self.assertIn("| Name | Age |", formatted)
+        self.assertIn("| --- | --- |", formatted)
+        self.assertIn("| John | 30 |", formatted)
+        self.assertIn("paragraph", formatted)
+        self.assertNotIn("\r", formatted)
+        self.assertTrue(formatted.endswith("\n"))
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_mhtml_parser.py

@@ -0,0 +1,323 @@
+import base64
+import email
+import json
+import unittest
+from email.header import Header
+from email.message import EmailMessage
+from pathlib import Path
+
+from docreader.parser.mhtml_parser import MHTMLParser, _header_str
+from docreader.parser.registry import registry
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+def _minimal_mhtml_bytes() -> bytes:
+    root = EmailMessage()
+    root["Subject"] = "Tiny MHTML"
+    root.make_related()
+
+    main = EmailMessage()
+    main.set_content(
+        "<html><body><h1>Main Article</h1>"
+        "<p>Hello MHTML world.</p>"
+        '<p><a href="chapter03.xhtml#sec2">Chapter 3</a> '
+        '<a href="#footnote1">note</a> '
+        '<a href="https://example.com">the site</a></p>'
+        '<img alt="tiny" src="cid:tiny-image">'
+        "<script>window.noise = true</script>"
+        "</body></html>",
+        subtype="html",
+    )
+    main["Content-Location"] = "https://example.com/article"
+    root.attach(main)
+
+    ad = EmailMessage()
+    ad.set_content(
+        "<html><body><h1>Advertisement</h1>"
+        "<p>Buy this unrelated thing.</p></body></html>",
+        subtype="html",
+    )
+    ad["Content-Location"] = "https://googleads.example/frame.html"
+    root.attach(ad)
+
+    image = EmailMessage()
+    image.set_content(
+        b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR",
+        maintype="image",
+        subtype="png",
+    )
+    image["Content-Location"] = "cid:tiny-image"
+    root.attach(image)
+
+    return root.as_bytes()
+
+
+def _mhtml_with_table_image_and_caption() -> bytes:
+    root = EmailMessage()
+    root["Subject"] = "MHTML with table image"
+    root.make_related()
+
+    main = EmailMessage()
+    main.set_content(
+        "<html><body>"
+        "<table>"
+        "<tr><th>体验方向</th><th>代表内容</th></tr>"
+        "<tr><td>赛季制建立</td><td>BP、Rank</td></tr>"
+        "</table>"
+        '<img src="cid:test-image" alt="图片" title="图片">'
+        "<p>高机动性身法与独特枪械反馈</p>"
+        "</body></html>",
+        subtype="html",
+    )
+    main["Content-Location"] = "https://example.com/article"
+    root.attach(main)
+
+    image = EmailMessage()
+    image.set_content(
+        b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff,\x00\x00"
+        b"\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;",
+        maintype="image",
+        subtype="gif",
+    )
+    image["Content-ID"] = "<test-image>"
+    root.attach(image)
+
+    return root.as_bytes()
+
+
+def _mhtml_with_8bit_utf8_headers() -> bytes:
+    """Browser-style MHTML: raw UTF-8 bytes in MIME headers, not RFC 2047."""
+    html_loc = "https://example.com/测试页面.html".encode("utf-8")
+    ad_loc = "https://googleads.example/广告.html".encode("utf-8")
+    image_loc = "https://example.com/图片.png".encode("utf-8")
+    content_id = "<图片@local>".encode("utf-8")
+    attachment_id = "图片".encode("utf-8")
+    main_html = (
+        "<html><body><h1>中文主文</h1>"
+        '<img alt="绝对" src="https://example.com/图片.png">'
+        '<img alt="相对" src="图片.png">'
+        '<img alt="cid" src="cid:图片@local">'
+        "</body></html>"
+    ).encode("utf-8")
+    ad_html = "<html><body><h1>广告正文</h1></body></html>".encode("utf-8")
+    png = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"
+    return b"".join(
+        [
+            b"MIME-Version: 1.0\r\n",
+            b'Content-Type: multipart/related; boundary="----=_Next"\r\n',
+            b"\r\n",
+            b"------=_Next\r\n",
+            b'Content-Type: text/html; charset="utf-8"\r\n',
+            b"Content-Transfer-Encoding: 8bit\r\n",
+            b"Content-Location: ",
+            html_loc,
+            b"\r\n\r\n",
+            main_html,
+            b"\r\n",
+            b"------=_Next\r\n",
+            b'Content-Type: text/html; charset="utf-8"\r\n',
+            b"Content-Transfer-Encoding: 8bit\r\n",
+            b"Content-Location: ",
+            ad_loc,
+            b"\r\n\r\n",
+            ad_html,
+            b"\r\n",
+            b"------=_Next\r\n",
+            b"Content-Type: image/png\r\n",
+            b"Content-Transfer-Encoding: base64\r\n",
+            b"Content-Location: ",
+            image_loc,
+            b"\r\nContent-ID: ",
+            content_id,
+            b"\r\nX-Attachment-Id: ",
+            attachment_id,
+            b"\r\n\r\n",
+            base64.b64encode(png),
+            b"\r\n",
+            b"------=_Next--\r\n",
+        ]
+    )
+
+
+class MHTMLParserTest(unittest.TestCase):
+    def test_parse_selects_main_html_and_filters_noise(self):
+        document = MHTMLParser(
+            file_name="article.mhtml", file_type="mhtml"
+        ).parse_into_text(_minimal_mhtml_bytes())
+
+        self.assertIn("Main Article", document.content)
+        self.assertIn("Hello MHTML world", document.content)
+        self.assertNotIn("Advertisement", document.content)
+        self.assertNotIn("window.noise", document.content)
+        self.assertEqual(document.metadata["source_format"], "mhtml")
+
+    def test_internal_links_are_unwrapped_but_external_links_remain(self):
+        document = MHTMLParser(
+            file_name="article.mhtml", file_type="mhtml"
+        ).parse_into_text(_minimal_mhtml_bytes())
+
+        self.assertIn("Chapter 3", document.content)
+        self.assertIn("note", document.content)
+        self.assertNotIn("chapter03.xhtml#sec2", document.content)
+        self.assertNotIn("#footnote1", document.content)
+        self.assertIn("[the site](https://example.com)", document.content)
+
+    def test_image_extraction_toggle(self):
+        with_images = MHTMLParser(
+            file_name="article.mhtml", file_type="mhtml", extract_images=True
+        ).parse_into_text(_minimal_mhtml_bytes())
+        without_images = MHTMLParser(
+            file_name="article.mhtml", file_type="mhtml", extract_images=False
+        ).parse_into_text(_minimal_mhtml_bytes())
+
+        self.assertEqual(len(with_images.images), 1)
+        image_ref = next(iter(with_images.images))
+        self.assertTrue(image_ref.startswith("images/"))
+        self.assertIn(image_ref, with_images.content)
+        self.assertNotIn("cid:tiny-image", with_images.content)
+        self.assertEqual(without_images.images, {})
+
+    def test_table_image_and_caption_keep_markdown_block_boundaries(self):
+        document = MHTMLParser(
+            file_name="article.mhtml", file_type="mhtml"
+        ).parse_into_text(_mhtml_with_table_image_and_caption())
+
+        self.assertEqual(len(document.images), 1)
+        image_ref = next(iter(document.images))
+        self.assertIn(f'![图片]({image_ref} "图片")', document.content)
+        self.assertIn(
+            f"| 赛季制建立 | BP、Rank |\n\n![图片]({image_ref} \"图片\")",
+            document.content,
+        )
+        self.assertIn(
+            f'![图片]({image_ref} "图片")\n\n高机动性身法与独特枪械反馈',
+            document.content,
+        )
+
+    def test_html_to_markdown_preserves_indentation_and_code_block_blanks(self):
+        markdown = MHTMLParser(
+            file_name="article.mhtml", file_type="mhtml"
+        )._html_to_markdown(
+            "<ul><li>parent<ul><li>child</li></ul></li></ul>"
+            "<blockquote><p>quoted</p></blockquote>"
+            "<pre><code>line1\n\n  indented\n</code></pre>"
+        )
+
+        self.assertIn("* parent\n  + child", markdown)
+        self.assertIn("\n\n> quoted\n\n", markdown)
+        self.assertIn("```\nline1\n\n  indented\n```", markdown)
+
+    def test_html_to_markdown_preserves_nested_list_indentation(self):
+        markdown = MHTMLParser(
+            file_name="article.mhtml", file_type="mhtml"
+        )._html_to_markdown("<ul><li>parent<ul><li>child</li></ul></li></ul>")
+
+        self.assertIn("* parent\n  + child", markdown)
+
+    def test_html_to_markdown_preserves_blockquote_boundaries(self):
+        markdown = MHTMLParser(
+            file_name="article.mhtml", file_type="mhtml"
+        )._html_to_markdown("<p>before</p><blockquote><p>quoted</p></blockquote><p>after</p>")
+
+        self.assertIn("before\n\n> quoted\n\nafter", markdown)
+
+    def test_html_to_markdown_preserves_fenced_code_blank_lines(self):
+        markdown = MHTMLParser(
+            file_name="article.mhtml", file_type="mhtml"
+        )._html_to_markdown("<pre><code>line1\n\n\nline2\n</code></pre>")
+
+        self.assertIn("```\nline1\n\n\nline2\n```", markdown)
+
+    def test_html_to_markdown_collapses_excess_blank_lines_outside_code(self):
+        markdown = MHTMLParser._normalize_markdown("alpha\n\n  \n\t\n\nbeta")
+
+        self.assertEqual(markdown, "alpha\n\nbeta")
+        self.assertNotIn("\n\n\n", markdown)
+
+    def test_html_to_markdown_preserves_hard_break_spaces(self):
+        markdown = MHTMLParser(
+            file_name="article.mhtml", file_type="mhtml"
+        )._html_to_markdown("<p>alpha<br>beta</p>")
+
+        self.assertEqual(markdown, "alpha  \nbeta")
+
+    def test_normalize_markdown_preserves_two_space_hard_break(self):
+        markdown = MHTMLParser._normalize_markdown("alpha  \nbeta")
+
+        self.assertEqual(markdown, "alpha  \nbeta")
+
+    def test_html_to_markdown_normalizes_crlf(self):
+        markdown = MHTMLParser._normalize_markdown("alpha\r\n\r\nbeta\rgamma")
+
+        self.assertEqual(markdown, "alpha\n\nbeta\ngamma")
+
+    def test_html_to_markdown_does_not_strip_leading_indentation_at_document_start(self):
+        markdown = MHTMLParser._normalize_markdown("  indented start\n")
+
+        self.assertEqual(markdown, "  indented start")
+
+    def test_mhtml_shared_contract_fixture(self):
+        fixture = REPO_ROOT / "testdata" / "mhtml" / "titled-image.mhtml"
+        contract_path = REPO_ROOT / "testdata" / "mhtml" / "titled-image-contract.json"
+        contract = json.loads(contract_path.read_text(encoding="utf-8"))
+
+        document = MHTMLParser(
+            file_name="titled-image.mhtml", file_type="mhtml"
+        ).parse_into_text(fixture.read_bytes())
+
+        self.assertEqual(document.content, contract["markdown_content"])
+        self.assertEqual(len(document.images), 1)
+        image_contract = contract["images"][0]
+        self.assertIn(image_contract["original_ref"], document.images)
+        self.assertEqual(
+            base64.b64decode(document.images[image_contract["original_ref"]]),
+            base64.b64decode(image_contract["image_data_base64"]),
+        )
+        self.assertIn(
+            '| 赛季制建立 | BP、Rank |\n\n![图片](images/第 1 页 (测试).gif "阶段 1) 图片")',
+            document.content,
+        )
+        self.assertIn(
+            '![图片](images/第 1 页 (测试).gif "阶段 1) 图片")\n\n高机动性身法与独特枪械反馈',
+            document.content,
+        )
+        self.assertNotIn("\n\n\n", document.content.split("```", 1)[0])
+        self.assertIn("```\nline1\n\n\nline2\n```", document.content)
+        self.assertIn("* parent\n  + child", document.content)
+        self.assertIn("alpha  \nbeta", document.content)
+
+    def test_header_str_recovers_unknown_8bit_utf8(self):
+        raw = (
+            b"Content-Type: text/plain\r\n"
+            b"Content-Location: https://example.com/"
+            + "图片.png".encode("utf-8")
+            + b"\r\n\r\nbody\r\n"
+        )
+        value = email.message_from_bytes(raw).get("Content-Location")
+        self.assertIsInstance(value, Header)
+        self.assertEqual(_header_str(value), "https://example.com/图片.png")
+        self.assertEqual(_header_str(None), "")
+        self.assertEqual(_header_str("ascii-location"), "ascii-location")
+        self.assertEqual(_header_str("已是unicode"), "已是unicode")
+
+    def test_parse_8bit_utf8_headers_rewrites_images_without_crashing(self):
+        document = MHTMLParser(
+            file_name="article.mhtml", file_type="mhtml"
+        ).parse_into_text(_mhtml_with_8bit_utf8_headers())
+
+        self.assertIn("中文主文", document.content)
+        self.assertNotIn("广告正文", document.content)
+        self.assertIn("images/图片.png", document.images)
+        self.assertIn("images/图片.png", document.content)
+        self.assertEqual(document.content.count("images/图片.png"), 3)
+        self.assertNotIn("cid:图片@local", document.content)
+        self.assertNotIn("\ufffd", document.content)
+        self.assertTrue(all("\ufffd" not in key for key in document.images))
+
+    def test_registry_resolves_mhtml(self):
+        self.assertIs(registry.get_parser_class("", "mhtml"), MHTMLParser)
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_opendataloader_parser.py

@@ -0,0 +1,133 @@
+"""Unit tests for OpenDataLoader parser helpers (no JVM required)."""
+
+import os
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+from docreader.parser.opendataloader_parser import (
+    OpenDataLoaderParser,
+    _collect_images_under_output,
+    _find_markdown_file,
+    _normalize_odl_image_url,
+    _ping_hybrid,
+    _run_convert,
+    _rewrite_markdown_image_refs,
+    opendataloader_available,
+)
+
+
+class OpenDataLoaderHelpersTest(unittest.TestCase):
+    def test_hybrid_health_probe_blocks_private_url_before_request(self):
+        with mock.patch(
+            "docreader.parser.opendataloader_parser.is_ssrf_safe_url",
+            return_value=(False, "restricted test address"),
+        ), mock.patch("urllib.request.build_opener") as build_opener:
+            ok, msg = _ping_hybrid("http://127.0.0.1:8080", retries=1)
+
+        self.assertFalse(ok)
+        self.assertIn("SSRF", msg)
+        build_opener.assert_not_called()
+
+    def test_convert_blocks_private_hybrid_url_at_final_sink(self):
+        fake_module = mock.Mock()
+        with mock.patch.dict(sys.modules, {"opendataloader_pdf": fake_module}):
+            with self.assertRaisesRegex(RuntimeError, "SSRF"):
+                _run_convert(
+                    "/tmp/input.pdf",
+                    "/tmp/output",
+                    "/tmp/output/images",
+                    overrides={
+                        "odl_hybrid": "docling-fast",
+                        "odl_hybrid_url": "http://169.254.169.254/latest/meta-data",
+                    },
+                )
+
+        fake_module.convert.assert_not_called()
+
+    def test_find_markdown_prefers_stem_match(self):
+        with tempfile.TemporaryDirectory() as d:
+            other = os.path.join(d, "other.md")
+            target = os.path.join(d, "paper.md")
+            with open(other, "w") as f:
+                f.write("x")
+            with open(target, "w") as f:
+                f.write("# Title")
+            self.assertEqual(_find_markdown_file(d, "paper"), target)
+
+    def test_collect_and_rewrite_images(self):
+        with tempfile.TemporaryDirectory() as d:
+            img_dir = os.path.join(d, "images")
+            os.makedirs(img_dir)
+            png = os.path.join(img_dir, "fig1.png")
+            with open(png, "wb") as f:
+                f.write(b"\x89PNG\r\n\x1a\n")
+            images = _collect_images_under_output(d)
+            self.assertIn("images/fig1.png", images)
+            md = "See ![fig](images/fig1.png) and ![alt](./fig1.png)."
+            out = _rewrite_markdown_image_refs(md, images)
+            self.assertIn("![fig](images/fig1.png)", out)
+            self.assertIn("![alt](images/fig1.png)", out)
+
+    def test_rewrite_odl_angle_bracket_and_entity_urls(self):
+        images = {"images/imageFile1.png": "e30="}
+        for md_in in (
+            "![image 1](<images/imageFile1.png>)",
+            "![image 1](&lt;images/imageFile1.png&gt;)",
+        ):
+            out = _rewrite_markdown_image_refs(md_in, images)
+            self.assertEqual("![image 1](images/imageFile1.png)", out)
+
+    def test_normalize_odl_image_url(self):
+        self.assertEqual(
+            _normalize_odl_image_url("&lt;images/imageFile2.png&gt;"),
+            "images/imageFile2.png",
+        )
+        self.assertEqual(
+            _normalize_odl_image_url("<images/imageFile2.png>"),
+            "images/imageFile2.png",
+        )
+
+    def test_rewrite_skips_data_uris(self):
+        md = "![x](data:image/png;base64,abc)"
+        self.assertEqual(_rewrite_markdown_image_refs(md, {"images/a.png": "e30="}), md)
+
+
+class OpenDataLoaderParserTest(unittest.TestCase):
+    @mock.patch("docreader.parser.opendataloader_parser.opendataloader_available")
+    @mock.patch("docreader.parser.opendataloader_parser._run_convert")
+    def test_parse_reads_markdown_and_images(self, mock_convert, mock_avail):
+        mock_avail.return_value = (True, "")
+
+        def fake_convert(pdf_path, output_dir, image_dir, overrides=None):
+            stem = os.path.splitext(os.path.basename(pdf_path))[0]
+            md_path = os.path.join(output_dir, f"{stem}.md")
+            with open(md_path, "w") as f:
+                f.write("# Hello\n\n![pic](images/pic.png)\n")
+            os.makedirs(image_dir, exist_ok=True)
+            with open(os.path.join(image_dir, "pic.png"), "wb") as f:
+                f.write(b"png")
+
+        mock_convert.side_effect = fake_convert
+
+        parser = OpenDataLoaderParser(file_name="doc.pdf", file_type="pdf")
+        doc = parser.parse_into_text(b"%PDF-1.4 fake")
+        self.assertIn("# Hello", doc.content)
+        self.assertIn("images/pic.png", doc.content)
+        self.assertIn("images/pic.png", doc.images)
+        self.assertEqual(doc.metadata.get("parser_engine"), "opendataloader")
+
+    @mock.patch("docreader.parser.opendataloader_parser.shutil.which", return_value=None)
+    def test_availability_requires_java(self, _which):
+        with mock.patch(
+            "docreader.parser.opendataloader_parser._package_available",
+            return_value=(True, ""),
+        ):
+            ok, msg = opendataloader_available()
+        self.assertFalse(ok)
+        self.assertIn("Java", msg)
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_parser_concurrency.py

@@ -0,0 +1,88 @@
+import base64
+import io
+import threading
+import time
+import unittest
+import uuid
+
+from PIL import Image
+
+from docreader.parser.concurrency import parser_worker_limit
+from docreader.parser.pdf_parser import PDFScannedParser, _normalize_image_quality
+
+
+class ParserConcurrencyTest(unittest.TestCase):
+    def test_parser_worker_limit_serializes_work(self):
+        limiter_name = f"test-{uuid.uuid4()}"
+        active_workers = 0
+        max_active_workers = 0
+        state_lock = threading.Lock()
+        start = threading.Barrier(3)
+
+        def worker():
+            nonlocal active_workers, max_active_workers
+            start.wait()
+            with parser_worker_limit(limiter_name, 1):
+                with state_lock:
+                    active_workers += 1
+                    max_active_workers = max(max_active_workers, active_workers)
+                time.sleep(0.02)
+                with state_lock:
+                    active_workers -= 1
+
+        threads = [threading.Thread(target=worker) for _ in range(2)]
+        for thread in threads:
+            thread.start()
+
+        start.wait()
+        for thread in threads:
+            thread.join()
+
+        self.assertEqual(max_active_workers, 1)
+
+    def test_scanned_pdf_parser_outputs_jpeg_images(self):
+        pdf_bytes = io.BytesIO()
+        pages = [
+            Image.new("RGB", (64, 64), "white"),
+            Image.new("RGB", (64, 64), "black"),
+        ]
+        pages[0].save(
+            pdf_bytes,
+            format="PDF",
+            save_all=True,
+            append_images=pages[1:],
+        )
+
+        document = PDFScannedParser(file_name="scan.pdf").parse_into_text(
+            pdf_bytes.getvalue()
+        )
+
+        image_ref = "images/scan_page_1.jpg"
+        self.assertIn(f"![scan_page_1.jpg]({image_ref})", document.content)
+        self.assertIn(image_ref, document.images)
+        self.assertEqual(document.metadata["image_source_type"], "scanned_pdf")
+        self.assertEqual(document.metadata["page_count"], 2)
+        self.assertEqual(len(document.images), 2)
+        self.assertIn("images/scan_page_2.jpg", document.images)
+        image_bytes = base64.b64decode(document.images[image_ref])
+        self.assertTrue(image_bytes.startswith(b"\xff\xd8"))
+
+    def test_scanned_pdf_parser_logs_malformed_pdf_without_format_error(self):
+        parser = PDFScannedParser(file_name="broken.pdf")
+
+        with self.assertLogs("docreader.parser.pdf_parser", level="ERROR") as logs:
+            with self.assertRaises(Exception):
+                parser.parse_into_text(b"not a pdf")
+
+        self.assertTrue(
+            any("PDFScannedParser failed to parse PDF:" in line for line in logs.output)
+        )
+
+    def test_normalize_image_quality_bounds_jpeg_quality(self):
+        self.assertEqual(_normalize_image_quality(-1), 1)
+        self.assertEqual(_normalize_image_quality(90), 90)
+        self.assertEqual(_normalize_image_quality(120), 95)
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_parser_routing.py

@@ -0,0 +1,105 @@
+import unittest
+
+from docreader.models.document import Document
+from docreader.parser.base_parser import BaseParser
+from docreader.parser.parser import Parser, detect_effective_file_type
+from docreader.parser.markitdown_parser import MarkitdownParser
+from docreader.parser.pdf_parser import PDFParser
+from docreader.parser.registry import BUILTIN_ENGINE, ParserEngineRegistry, registry
+from docreader.parser.xmind_parser import XMindParser
+
+
+OLE_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
+
+
+class _RecordingParser(BaseParser):
+    last_file_type = ""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        type(self).last_file_type = self.file_type
+
+    def parse_into_text(self, content: bytes) -> Document:
+        return Document(content="parsed")
+
+
+class _RecordingRegistry:
+    def __init__(self):
+        self.requested_file_type = ""
+
+    def get_parser_class(self, engine: str, file_type: str):
+        self.requested_file_type = file_type
+        return _RecordingParser
+
+
+class ParserRoutingTest(unittest.TestCase):
+    def test_builtin_registry_routes_xmind_to_xmind_parser(self):
+        parser_class = registry.get_parser_class(BUILTIN_ENGINE, "xmind")
+
+        self.assertIs(parser_class, XMindParser)
+
+    def test_legacy_doc_payload_renamed_to_docx_uses_doc_parser(self):
+        registry = _RecordingRegistry()
+        parser = Parser()
+        parser.registry = registry
+
+        result = parser.parse_file(
+            "legacy.docx",
+            "docx",
+            OLE_MAGIC + b"legacy-word-payload",
+        )
+
+        self.assertEqual("parsed", result.content)
+        self.assertEqual("doc", registry.requested_file_type)
+        self.assertEqual("doc", _RecordingParser.last_file_type)
+
+    def test_real_docx_payload_keeps_docx_parser_route(self):
+        self.assertEqual(
+            "docx",
+            detect_effective_file_type(".DOCX", b"PK\x03\x04ooxml-payload"),
+        )
+
+    def test_unrelated_ole_type_is_not_reclassified(self):
+        self.assertEqual("xls", detect_effective_file_type("xls", OLE_MAGIC))
+
+    def test_empty_engine_routes_pptx_to_markitdown(self):
+        self.assertIs(registry.get_parser_class("", "pptx"), MarkitdownParser)
+        self.assertIs(registry.get_parser_class("", "ppt"), MarkitdownParser)
+        self.assertIs(registry.get_parser_class("", ".PPTX"), MarkitdownParser)
+
+    def test_builtin_engine_still_parses_pptx(self):
+        self.assertIs(registry.get_parser_class(BUILTIN_ENGINE, "pptx"), MarkitdownParser)
+
+    def test_type_default_when_builtin_lacks_pptx(self):
+        isolated = ParserEngineRegistry()
+        isolated.register(BUILTIN_ENGINE, {"pdf": PDFParser})
+        isolated.register(
+            "markitdown",
+            {"pptx": MarkitdownParser, "ppt": MarkitdownParser, "csv": MarkitdownParser},
+        )
+
+        self.assertIs(isolated.get_parser_class("", "pptx"), MarkitdownParser)
+        self.assertIs(isolated.get_parser_class(BUILTIN_ENGINE, "pptx"), MarkitdownParser)
+        self.assertIs(isolated.get_parser_class("", "csv"), MarkitdownParser)
+        self.assertIs(isolated.get_parser_class(BUILTIN_ENGINE, "csv"), MarkitdownParser)
+        self.assertIs(isolated.get_parser_class(BUILTIN_ENGINE, "pdf"), PDFParser)
+        self.assertIs(isolated.get_parser_class("", "pdf"), PDFParser)
+
+        with self.assertRaises(ValueError) as ctx:
+            isolated.get_parser_class("", "xyz")
+        self.assertIn("Unsupported file type: xyz", str(ctx.exception))
+
+    def test_actionable_error_lists_supporting_engines(self):
+        isolated = ParserEngineRegistry()
+        isolated.register(BUILTIN_ENGINE, {"pdf": PDFParser})
+        isolated.register("opendataloader", {"xmind": XMindParser})
+
+        with self.assertRaises(ValueError) as ctx:
+            isolated.get_parser_class(BUILTIN_ENGINE, "xmind")
+        message = str(ctx.exception)
+        self.assertIn("Configure one of:", message)
+        self.assertIn("opendataloader", message)
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_pdf_embedded_images.py

@@ -0,0 +1,258 @@
+"""Embedded-figure extraction from native PDF pages.
+
+Focus: images whose visible content lives in a /SMask (soft transparency
+mask) while the base RGB plane is black. ``get_bitmap()`` alone decodes only
+the base plane, producing an all-black JPEG; the decode path must render the
+mask in and composite over white. Reproduces the all-black figures extracted
+from a production PDF exported by a plotting tool.
+"""
+
+import base64
+import gc
+import io
+import unittest
+from unittest.mock import patch
+
+import pypdfium2 as pdfium
+import pypdfium2.raw as pdfium_raw
+from PIL import Image
+
+from docreader.parser.pdf_parser import (
+    _decode_embedded_image_pil,
+    _extract_embedded_images,
+)
+
+PAGE_W, PAGE_H = 612, 792
+IMG_W, IMG_H = 120, 90
+
+
+def _pdf_from_objects(objects: list) -> bytes:
+    out = bytearray(b"%PDF-1.7\n")
+    offsets = []
+    for i, body in enumerate(objects, start=1):
+        offsets.append(len(out))
+        out += b"%d 0 obj\n" % i + body + b"\nendobj\n"
+    xref_pos = len(out)
+    out += b"xref\n0 %d\n" % (len(objects) + 1)
+    out += b"0000000000 65535 f \n"
+    for off in offsets:
+        out += b"%010d 00000 n \n" % off
+    out += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (
+        len(objects) + 1,
+        xref_pos,
+    )
+    return bytes(out)
+
+
+def _page_objects(image_obj: bytes, extra_objs: list | None = None) -> list:
+    contents = b"q 480 0 0 360 66 216 cm /Im0 Do Q\n"
+    objects = [
+        b"<< /Type /Catalog /Pages 2 0 R >>",
+        b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+        (
+            b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 %d %d] "
+            b"/Resources << /XObject << /Im0 5 0 R >> >> /Contents 4 0 R >>"
+            % (PAGE_W, PAGE_H)
+        ),
+        b"<< /Length %d >>\nstream\n" % len(contents) + contents + b"endstream",
+        image_obj,
+    ]
+    if extra_objs:
+        objects.extend(extra_objs)
+    return objects
+
+
+def _circle_mask() -> bytes:
+    cx, cy, radius = IMG_W // 2, IMG_H // 2, 30
+    mask = bytearray(IMG_W * IMG_H)
+    for y in range(IMG_H):
+        for x in range(IMG_W):
+            if (x - cx) ** 2 + (y - cy) ** 2 <= radius * radius:
+                mask[y * IMG_W + x] = 255
+    return bytes(mask)
+
+
+def _build_smask_pdf(base_rgb=(0, 0, 0)) -> bytes:
+    """Hand-built PDF: one RGB image shaped by an SMask circle.
+
+    The SMask is opaque (255) inside a filled circle and transparent (0)
+    elsewhere. A decoder that drops the mask shows the raw base plane only.
+    """
+    r, g, b = base_rgb
+    base = bytes((r, g, b)) * (IMG_W * IMG_H)
+    mask = _circle_mask()
+    image = (
+        b"<< /Type /XObject /Subtype /Image /Width %d /Height %d "
+        b"/ColorSpace /DeviceRGB /BitsPerComponent 8 /SMask 6 0 R "
+        b"/Length %d >>\nstream\n" % (IMG_W, IMG_H, len(base))
+        + base
+        + b"\nendstream"
+    )
+    mask_obj = (
+        b"<< /Type /XObject /Subtype /Image /Width %d /Height %d "
+        b"/ColorSpace /DeviceGray /BitsPerComponent 8 /Length %d >>\nstream\n"
+        % (IMG_W, IMG_H, len(mask))
+        + mask
+        + b"\nendstream"
+    )
+    return _pdf_from_objects(_page_objects(image, [mask_obj]))
+
+
+def _build_opaque_gray_pdf(level: int = 128) -> bytes:
+    data = bytes((level,)) * (IMG_W * IMG_H)
+    image = (
+        b"<< /Type /XObject /Subtype /Image /Width %d /Height %d "
+        b"/ColorSpace /DeviceGray /BitsPerComponent 8 /Length %d >>\nstream\n"
+        % (IMG_W, IMG_H, len(data))
+        + data
+        + b"\nendstream"
+    )
+    return _pdf_from_objects(_page_objects(image))
+
+
+def _first_image_object(pdf):
+    for page in pdf:
+        for obj in page.get_objects():
+            if obj.type == pdfium_raw.FPDF_PAGEOBJ_IMAGE:
+                return obj
+    raise AssertionError("test PDF has no image object")
+
+
+def _raw_decode(obj):
+    bitmap = obj.get_bitmap()
+    try:
+        pil = bitmap.to_pil()
+        if pil.mode in ("RGB", "L"):
+            return pil.copy()
+        return pil.convert("RGB")
+    finally:
+        bitmap.close()
+
+
+class DecodeEmbeddedImageTest(unittest.TestCase):
+    def test_smask_image_composites_over_white(self):
+        pdf = pdfium.PdfDocument(_build_smask_pdf())
+        try:
+            pil = _decode_embedded_image_pil(_first_image_object(pdf))
+        finally:
+            pdf.close()
+        self.assertEqual(pil.mode, "RGB")
+        # Transparent corners become white page background, not black.
+        self.assertGreater(pil.getpixel((2, 2))[0], 200)
+        self.assertGreater(pil.getpixel((IMG_W - 3, IMG_H - 3))[0], 200)
+        # Opaque circle center keeps the base-plane colour (black).
+        self.assertLess(pil.getpixel((IMG_W // 2, IMG_H // 2))[0], 40)
+
+    def test_colored_smask_keeps_base_color(self):
+        pdf = pdfium.PdfDocument(_build_smask_pdf(base_rgb=(200, 30, 40)))
+        try:
+            pil = _decode_embedded_image_pil(_first_image_object(pdf))
+        finally:
+            pdf.close()
+        self.assertEqual(pil.mode, "RGB")
+        self.assertGreater(pil.getpixel((2, 2))[0], 200)
+        center = pil.getpixel((IMG_W // 2, IMG_H // 2))
+        self.assertGreater(center[0], 150)
+        self.assertLess(center[1], 80)
+        self.assertLess(center[2], 80)
+
+    def test_opaque_image_matches_raw_pixels(self):
+        buf = io.BytesIO()
+        Image.new("RGB", (100, 80), (10, 200, 30)).save(buf, format="PDF")
+        pdf = pdfium.PdfDocument(buf.getvalue())
+        try:
+            obj = _first_image_object(pdf)
+            raw = _raw_decode(obj)
+            pil = _decode_embedded_image_pil(obj)
+        finally:
+            pdf.close()
+        self.assertEqual(pil.size, raw.size)
+        self.assertEqual(pil.mode, raw.mode)
+        self.assertEqual(pil.tobytes(), raw.tobytes())
+
+    def test_opaque_gray_fallback_keeps_l_mode(self):
+        # render=True typically returns BGRA even for DeviceGray; the raw
+        # fallback must keep L so grayscale hashes/JPEGs stay single-channel.
+        pdf = pdfium.PdfDocument(_build_opaque_gray_pdf(128))
+        try:
+            obj = _first_image_object(pdf)
+            raw = _raw_decode(obj)
+            self.assertEqual(raw.mode, "L")
+            original = obj.get_bitmap
+
+            def _boom(render=False, **kwargs):
+                if render:
+                    raise RuntimeError("render unavailable")
+                return original(render=render, **kwargs)
+
+            with patch.object(obj, "get_bitmap", side_effect=_boom):
+                with self.assertLogs("docreader.parser.pdf_parser", level="WARNING"):
+                    pil = _decode_embedded_image_pil(obj)
+        finally:
+            pdf.close()
+        self.assertEqual(pil.mode, "L")
+        self.assertEqual(pil.getpixel((IMG_W // 2, IMG_H // 2)), 128)
+        self.assertEqual(pil.tobytes(), raw.tobytes())
+
+    def test_decoded_image_survives_bitmap_gc(self):
+        buf = io.BytesIO()
+        Image.new("RGB", (100, 80), (10, 200, 30)).save(buf, format="PDF")
+        pdf = pdfium.PdfDocument(buf.getvalue())
+        try:
+            pil = _decode_embedded_image_pil(_first_image_object(pdf))
+        finally:
+            pdf.close()
+        gc.collect()
+        self.assertEqual(pil.getpixel((50, 40)), (10, 200, 30))
+
+    def test_render_failure_falls_back_to_raw(self):
+        buf = io.BytesIO()
+        Image.new("RGB", (100, 80), (10, 200, 30)).save(buf, format="PDF")
+        pdf = pdfium.PdfDocument(buf.getvalue())
+        try:
+            obj = _first_image_object(pdf)
+            raw = _raw_decode(obj)
+            original = obj.get_bitmap
+
+            def _boom(render=False, **kwargs):
+                if render:
+                    raise RuntimeError("render unavailable")
+                return original(render=render, **kwargs)
+
+            with patch.object(obj, "get_bitmap", side_effect=_boom):
+                with self.assertLogs(
+                    "docreader.parser.pdf_parser", level="WARNING"
+                ) as logs:
+                    pil = _decode_embedded_image_pil(obj)
+        finally:
+            pdf.close()
+        self.assertTrue(any("falling back to raw decode" in m for m in logs.output))
+        self.assertEqual(pil.mode, raw.mode)
+        self.assertEqual(pil.tobytes(), raw.tobytes())
+
+
+class ExtractEmbeddedImagesTest(unittest.TestCase):
+    def test_smask_figure_is_not_all_black(self):
+        pdf = pdfium.PdfDocument(_build_smask_pdf())
+        try:
+            result = _extract_embedded_images(
+                pdf, ["text"], pdfium_raw, "smask_doc", 85
+            )
+        finally:
+            pdf.close()
+        self.assertIn(0, result)
+        ref_path, b64_jpeg, _y_top = result[0][0]
+        self.assertTrue(ref_path.startswith("images/smask_doc_p1_img1"))
+        pil = Image.open(io.BytesIO(base64.b64decode(b64_jpeg))).convert("RGB")
+        total = bright = 0
+        for x in range(0, pil.width, 4):
+            for y in range(0, pil.height, 4):
+                total += 1
+                if max(pil.getpixel((x, y))) >= 200:
+                    bright += 1
+        # Dropping the mask would make this ~0: the whole figure is black.
+        self.assertGreater(bright / total, 0.5)
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_pdf_router.py

@@ -0,0 +1,504 @@
+import io
+import unittest
+
+from PIL import Image
+
+from docreader.parser.pdf_parser import (
+    PDFParser,
+    _classify_page,
+    _filter_reading_columns,
+    _group_lines,
+    _is_artifact_column,
+    _join_line_glyphs,
+    _merge_orphan_punctuation_lines,
+    _point_in_boxes,
+    _segments_to_markdown,
+    _select_embedded_images,
+    _should_prefer_plain,
+    _split_columns,
+    _strip_repeating_lines,
+)
+
+
+def _char(ch, x0, x1, y0, y1):
+    return {"x0": x0, "x1": x1, "y0": y0, "y1": y1, "ch": ch}
+
+
+def _line(text, h):
+    return {"text": text, "h": h}
+
+
+def _make_image_only_pdf(num_pages: int = 2) -> bytes:
+    buf = io.BytesIO()
+    pages = [Image.new("RGB", (64, 64), color) for color in ("white", "black")]
+    pages = (pages * ((num_pages // 2) + 1))[:num_pages]
+    pages[0].save(buf, format="PDF", save_all=True, append_images=pages[1:])
+    return buf.getvalue()
+
+
+class ClassifyPageTest(unittest.TestCase):
+    def test_full_page_image_is_scanned_even_with_text(self):
+        # Scanned newspaper: image covers the page, embedded OCR text exists.
+        self.assertEqual(_classify_page(2.0, 1620), "scanned")
+        # Scanned UN doc with garbled OCR text layer (ratio ~1.0).
+        self.assertEqual(_classify_page(1.0, 1500), "scanned")
+
+    def test_native_text_page_is_text(self):
+        self.assertEqual(_classify_page(0.01, 673), "text")
+        self.assertEqual(_classify_page(0.0, 1200), "text")
+
+    def test_sparse_text_with_image_is_scanned(self):
+        self.assertEqual(_classify_page(0.3, 2), "scanned")
+
+    def test_blank_page_is_text(self):
+        # No image, no text -> not rendered as an image.
+        self.assertEqual(_classify_page(0.0, 0), "text")
+
+
+class StripRepeatingLinesTest(unittest.TestCase):
+    def test_removes_repeated_header_footer(self):
+        header = "ACME CONFIDENTIAL"
+        texts = [f"{header}\nbody page {i}\npage {i} footer" for i in range(6)]
+        # Make the footer identical across pages so it is detected.
+        texts = [f"{header}\nbody page {i}\nshared footer" for i in range(6)]
+        classes = ["text"] * 6
+        cleaned = _strip_repeating_lines(texts, classes)
+        for page in cleaned:
+            self.assertNotIn(header, page)
+            self.assertNotIn("shared footer", page)
+        self.assertIn("body page 0", cleaned[0])
+
+    def test_keeps_lines_when_too_few_pages(self):
+        texts = ["HEADER\nbody"] * 2
+        classes = ["text"] * 2
+        self.assertEqual(_strip_repeating_lines(texts, classes), texts)
+
+
+class SelectEmbeddedImagesTest(unittest.TestCase):
+    def _fig(self, page, h="fig", w=200, ht=200, area=0.2):
+        return {"page": page, "width": w, "height": ht, "area_ratio": area, "hash": h}
+
+    def test_keeps_real_figure(self):
+        meta = [self._fig(0, "a")]
+        self.assertEqual(_select_embedded_images(meta, 1), [0])
+
+    def test_drops_tiny_and_small_images(self):
+        meta = [
+            self._fig(0, "tiny_area", area=0.001),  # too small a share
+            self._fig(0, "tiny_px", w=20, ht=20),  # too few pixels
+        ]
+        self.assertEqual(_select_embedded_images(meta, 1), [])
+
+    def test_drops_repeated_logo_watermark(self):
+        # Same hash on 5 of 6 text pages -> running logo/watermark.
+        meta = [self._fig(p, "logo") for p in range(5)]
+        meta.append(self._fig(5, "unique"))
+        kept = _select_embedded_images(meta, 6)
+        kept_hashes = {meta[i]["hash"] for i in kept}
+        self.assertNotIn("logo", kept_hashes)
+        self.assertIn("unique", kept_hashes)
+
+    def test_dedups_identical_image_on_same_page(self):
+        meta = [self._fig(0, "dup"), self._fig(0, "dup")]
+        self.assertEqual(len(_select_embedded_images(meta, 1)), 1)
+
+    def test_respects_max_images_cap(self):
+        meta = [self._fig(i, f"h{i}") for i in range(10)]
+        self.assertEqual(len(_select_embedded_images(meta, 10, max_images=3)), 3)
+
+
+class ReadingOrderTest(unittest.TestCase):
+    def test_single_column_stays_single(self):
+        # One column of glyphs at x~100, no full-height gutter.
+        chars = [_char("a", 100, 110, 700 - i * 12, 712 - i * 12) for i in range(5)]
+        cols = _split_columns(chars, scale=12.0, width=600.0)
+        self.assertEqual(len(cols), 1)
+
+    def test_two_columns_split_left_to_right(self):
+        # Left column x~50-150, right column x~400-500, wide empty gutter between.
+        left = [_char("L", 50, 150, 700 - i * 12, 712 - i * 12) for i in range(4)]
+        right = [_char("R", 400, 500, 700 - i * 12, 712 - i * 12) for i in range(4)]
+        cols = _split_columns(left + right, scale=12.0, width=600.0)
+        self.assertEqual(len(cols), 2)
+        # Reading order: left column before right column.
+        self.assertEqual(cols[0][0]["ch"], "L")
+        self.assertEqual(cols[1][0]["ch"], "R")
+
+    def test_group_lines_orders_by_y_then_x(self):
+        # Two visual lines; within a line glyphs given out of x-order.
+        chars = [
+            _char("B", 110, 120, 700, 712),  # adjacent to A (no word-sized gap)
+            _char("A", 100, 110, 700, 712),
+            _char("C", 100, 110, 680, 692),  # next line down
+        ]
+        lines = _group_lines(chars)
+        self.assertEqual([ln["text"] for ln in lines], ["AB", "C"])
+
+    def test_join_line_glyphs_inserts_word_spaces(self):
+        # Wide gap between "copy" and "of" mimics positioned OCR / text layers.
+        chars = [
+            _char("c", 0, 4, 0, 10),
+            _char("f", 10, 14, 0, 10),
+        ]
+        self.assertEqual(_join_line_glyphs(chars), "c f")
+
+    def test_join_line_glyphs_keeps_adjacent_letters(self):
+        chars = [_char("A", 100, 110, 700, 712), _char("B", 110, 120, 700, 712)]
+        self.assertEqual(_join_line_glyphs(chars), "AB")
+
+
+class HeadingDetectionTest(unittest.TestCase):
+    def test_promotes_large_line_to_heading(self):
+        lines = [_line("Big Title", 24.0)] + [_line(f"body {i}", 10.0) for i in range(6)]
+        md = _segments_to_markdown(lines)
+        self.assertTrue(md.startswith("# Big Title"))
+        self.assertIn("\nbody 0", md)
+
+    def test_does_not_promote_when_sizes_uniform(self):
+        lines = [_line(f"line {i}", 10.0) for i in range(6)]
+        md = _segments_to_markdown(lines)
+        self.assertNotIn("#", md)
+
+    def test_skips_sentence_like_long_lines(self):
+        # Large but ends with a period and is long -> body text, not a heading.
+        lines = [_line("x" * 90 + ".", 30.0)] + [_line("y", 10.0) for _ in range(6)]
+        md = _segments_to_markdown(lines)
+        self.assertFalse(md.startswith("#"))
+
+
+class HiddenTextFilterTest(unittest.TestCase):
+    def test_point_in_boxes(self):
+        boxes = [(0.0, 0.0, 10.0, 10.0)]
+        self.assertTrue(_point_in_boxes(5.0, 5.0, boxes))
+        self.assertFalse(_point_in_boxes(20.0, 5.0, boxes))
+
+
+class MarginColumnFilterTest(unittest.TestCase):
+    def test_drops_narrow_vertical_margin_column(self):
+        # Mimics arXiv sidebar: narrow x span, one glyph per line.
+        margin = [
+            _char(c, 20, 28, 500 - i * 14, 512 - i * 14)
+            for i, c in enumerate("0202luJ22")
+        ]
+        body = [
+            _char("L", 160, 170, 700, 712),
+            _char("a", 170, 180, 700, 712),
+            _char("n", 180, 190, 700, 712),
+        ]
+        cols = _filter_reading_columns(margin + body, scale=10.0, width=612.0)
+        self.assertEqual(len(cols), 1)
+        self.assertEqual(cols[0][0]["ch"], "L")
+
+    def test_keeps_real_two_column_layout(self):
+        left = [_char("L", 50, 150, 700 - i * 12, 712 - i * 12) for i in range(4)]
+        right = [_char("R", 400, 500, 700 - i * 12, 712 - i * 12) for i in range(4)]
+        cols = _filter_reading_columns(left + right, scale=12.0, width=600.0)
+        self.assertEqual(len(cols), 2)
+
+
+class PunctuationMergeTest(unittest.TestCase):
+    def test_merges_orphan_periods(self):
+        lines = [
+            _line("Figure 1 2", 10.0),
+            _line(". .", 10.0),
+            _line("Next", 10.0),
+        ]
+        merged = _merge_orphan_punctuation_lines(lines)
+        self.assertEqual([ln["text"] for ln in merged], ["Figure 1 2..", "Next"])
+
+
+class PdfTextSanitizeTest(unittest.TestCase):
+    def test_removes_fffe_placeholder(self):
+        from docreader.parser.pdf_parser import _postprocess_pdf_text
+
+        raw = "multi\ufffelayer and non\ufffetrivial"
+        out = _postprocess_pdf_text(raw)
+        self.assertEqual(out, "multilayer and nontrivial")
+
+    def test_strips_chart_axis_run(self):
+        from docreader.parser.pdf_parser import _postprocess_pdf_text
+
+        raw = (
+            "Deep convolutional neural networks have led to breakthroughs.\n"
+            "0 1 2 3 4 5 6 0\n"
+            "10\n"
+            "20\n"
+            "iter. (1e4)\n"
+            "training error (%)\n"
+            "56-layer\n"
+            "20-layer\n"
+            "Figure 1. Training error on CIFAR-10.\n"
+        )
+        out = _postprocess_pdf_text(raw)
+        self.assertIn("breakthroughs", out)
+        self.assertNotIn("56-layer", out)
+        self.assertIn("Figure 1.", out)
+
+    def test_strips_diagram_labels_above_caption(self):
+        from docreader.parser.pdf_parser import _postprocess_pdf_text
+
+        raw = (
+            "Paragraph before.\n"
+            "identity\n"
+            "weight layer\n"
+            "relu\n"
+            "Figure 2. Residual learning block.\n"
+            "Paragraph after.\n"
+        )
+        out = _postprocess_pdf_text(raw)
+        self.assertIn("Paragraph before.", out)
+        self.assertIn("Figure 2.", out)
+        self.assertIn("Paragraph after.", out)
+        self.assertNotIn("identity", out)
+        self.assertNotIn("weight layer", out)
+
+    def test_strips_arxiv_header_line(self):
+        from docreader.parser.pdf_parser import _postprocess_pdf_text
+
+        raw = "Body text.\n1\narXiv:1512.03385v1 [cs.CV] 10 Dec 2015\nMore body."
+        out = _postprocess_pdf_text(raw)
+        self.assertNotIn("arXiv:", out)
+        self.assertIn("Body text.", out)
+
+
+class PlainWellFormedTest(unittest.TestCase):
+    def test_academic_plain_skips_layout(self):
+        from docreader.parser.pdf_parser import _plain_is_well_formed
+
+        plain = (
+            "Recent work [DL15, MBXS17] shows progress on NLP tasks "
+            "with pre-trained models."
+        )
+        self.assertTrue(_plain_is_well_formed(plain))
+
+    def test_glued_scan_plain_needs_layout(self):
+        from docreader.parser.pdf_parser import _plain_is_well_formed
+
+        self.assertFalse(_plain_is_well_formed("Thisisadigitalcopyofabook"))
+
+
+class LayoutQualityFallbackTest(unittest.TestCase):
+    def test_prefers_plain_when_many_single_char_lines(self):
+        plain = "Language Models are Few-Shot Learners\nTom Brown"
+        layout = "0\n2\n0\n2\nl\nu\nJ\nLan ua e Models"
+        self.assertTrue(_should_prefer_plain(plain, layout))
+
+    def test_keeps_good_layout(self):
+        plain = "Hello world"
+        layout = "Hello world"
+        self.assertFalse(_should_prefer_plain(plain, layout))
+
+
+class ResNetPaperFigureTest(unittest.TestCase):
+    """Regression: ResNet PDF (arXiv:1512.03385) vector figures and captions."""
+
+    def test_resnet_figures_and_captions(self):
+        import os
+
+        from docreader.parser.pdf_parser import PDFParser
+
+        for path in (
+            os.path.join(
+                os.path.dirname(__file__),
+                "..",
+                "..",
+                "testdata",
+                "rag_test",
+                "pdf_en",
+                "resnet.pdf",
+            ),
+            "/tmp/resnet.pdf",
+        ):
+            if os.path.isfile(path):
+                break
+        else:
+            self.skipTest("resnet.pdf not available")
+
+        with open(path, "rb") as f:
+            doc = PDFParser(file_name="resnet.pdf", file_type="pdf").parse_into_text(
+                f.read()
+            )
+        self.assertGreater(doc.metadata.get("vector_figure_count", 0), 0)
+        self.assertIn("![", doc.content)
+        self.assertIn("Figure 2. Residual learning", doc.content)
+        self.assertNotIn("arXiv:", doc.content)
+        fig2 = doc.content.find("Figure 2. Residual learning")
+        before = doc.content[max(0, fig2 - 120) : fig2]
+        self.assertIn("![", before)
+        self.assertNotIn("identity", before)
+
+
+class Gpt3PaperLayoutTest(unittest.TestCase):
+    """Regression: arXiv GPT-3 paper title page must not be one-glyph-per-line."""
+
+    def test_gpt3_page0_title_and_authors(self):
+        import os
+
+        import pypdfium2 as pdfium
+        import pypdfium2.raw as pdfium_r
+
+        from docreader.parser.pdf_parser import PDFParser, _extract_layout_text
+
+        pdf_path = os.path.join(
+            os.path.dirname(__file__),
+            "..",
+            "..",
+            "testdata",
+            "rag_test",
+            "pdf_en",
+            "gpt3.pdf",
+        )
+        if not os.path.isfile(pdf_path):
+            self.skipTest("gpt3.pdf not in testdata")
+        with open(pdf_path, "rb") as f:
+            content = f.read()
+        with pdfium.PdfDocument(content) as pdf:
+            page = pdf[0]
+            try:
+                layout = _extract_layout_text(page, pdfium_r)
+            finally:
+                page.close()
+        # Margin sidebar must not appear as one-glyph-per-line prefix.
+        self.assertNotRegex(layout[:300], r"^0\n2\n0\n2")
+        self.assertIn("Few-Shot Learners", layout)
+
+        doc = PDFParser(file_name="gpt3.pdf", file_type="pdf").parse_into_text(content)
+        self.assertIn("Language Models are Few-Shot Learners", doc.content)
+        self.assertIn("Tom B. Brown", doc.content[:1200])
+        self.assertIn("[DL15, MBXS17, PNZtY18]", doc.content)
+        self.assertIn("task-specific architectures), and more recently", doc.content)
+        self.assertNotIn("k ifi hi d l", doc.content)
+
+
+class ScanEnglishDictLayoutTest(unittest.TestCase):
+    """Regression: Google Books-style PDFs lose spaces without gap inference."""
+
+    def test_scan_en_dict_page0_has_word_spaces(self):
+        import os
+
+        import pypdfium2 as pdfium
+        import pypdfium2.raw as pdfium_r
+
+        from docreader.parser.pdf_parser import _extract_layout_text
+
+        pdf_path = os.path.join(
+            os.path.dirname(__file__),
+            "..",
+            "..",
+            "testdata",
+            "rag_test",
+            "pdf_scan",
+            "scan_en_dict.pdf",
+        )
+        if not os.path.isfile(pdf_path):
+            self.skipTest("scan_en_dict.pdf not in testdata")
+        with open(pdf_path, "rb") as f:
+            pdf = pdfium.PdfDocument(f.read())
+        try:
+            text = _extract_layout_text(pdf[0], pdfium_r)
+        finally:
+            pdf.close()
+        self.assertIn("This is a digital copy of a book", text)
+        self.assertNotIn("Thisisadigitalcopyofabook", text)
+
+
+class PDFRouterIntegrationTest(unittest.TestCase):
+    def test_image_only_pdf_routes_to_scanned(self):
+        pdf_bytes = _make_image_only_pdf(2)
+        doc = PDFParser(file_name="imgonly.pdf", file_type="pdf").parse_into_text(
+            pdf_bytes
+        )
+
+        self.assertEqual(doc.metadata["image_source_type"], "scanned_pdf")
+        self.assertEqual(doc.metadata["page_count"], 2)
+        self.assertEqual(doc.metadata["scanned_page_count"], 2)
+        self.assertEqual(len(doc.images), 2)
+        self.assertIn("images/imgonly_page_1.jpg", doc.images)
+        self.assertIn("![imgonly_page_1.jpg](images/imgonly_page_1.jpg)", doc.content)
+        # JPEG magic bytes after decoding.
+        import base64
+
+        self.assertTrue(
+            base64.b64decode(doc.images["images/imgonly_page_1.jpg"]).startswith(
+                b"\xff\xd8"
+            )
+        )
+
+    def test_malformed_pdf_raises_after_fallback(self):
+        # Routing fails to open the PDF, falls back to full rendering which also
+        # fails on garbage input; the error surfaces to the caller.
+        with self.assertRaises(Exception):
+            PDFParser(file_name="broken.pdf", file_type="pdf").parse_into_text(
+                b"not a pdf"
+            )
+
+
+class ForceScannedTest(unittest.TestCase):
+    """Tests for the force-scanned PDF parsing mode."""
+
+    def test_default_text_pdf_is_not_force_scanned(self):
+        """Without any override, a text PDF should NOT use scanned mode."""
+        parser = PDFParser(file_name="normal.pdf", file_type="pdf")
+        self.assertFalse(parser._force_scanned)
+
+    def test_force_scanned_via_kwarg_true(self):
+        """Per-upload override pdf_force_scanned=true enables scanned mode."""
+        parser = PDFParser(
+            file_name="test.pdf", file_type="pdf", pdf_force_scanned="true"
+        )
+        self.assertTrue(parser._force_scanned)
+
+    def test_force_scanned_via_kwarg_false(self):
+        """Per-upload override pdf_force_scanned=false disables scanned mode."""
+        parser = PDFParser(
+            file_name="test.pdf", file_type="pdf", pdf_force_scanned="false"
+        )
+        self.assertFalse(parser._force_scanned)
+
+    def test_force_scanned_via_kwarg_overrides_env(self, monkeypatch=None):
+        """Per-upload override takes priority over global env var."""
+        import docreader.parser.pdf_parser as pdf_mod
+
+        old = pdf_mod.FORCE_SCANNED_PDF
+        try:
+            pdf_mod.FORCE_SCANNED_PDF = True
+            # Explicit false override should win over env=true
+            parser = PDFParser(
+                file_name="test.pdf", file_type="pdf", pdf_force_scanned="false"
+            )
+            self.assertFalse(parser._force_scanned)
+        finally:
+            pdf_mod.FORCE_SCANNED_PDF = old
+
+    def test_force_scanned_via_env(self):
+        """Global env variable enables force scanned when no override."""
+        import docreader.parser.pdf_parser as pdf_mod
+
+        old = pdf_mod.FORCE_SCANNED_PDF
+        try:
+            pdf_mod.FORCE_SCANNED_PDF = True
+            parser = PDFParser(file_name="test.pdf", file_type="pdf")
+            self.assertTrue(parser._force_scanned)
+        finally:
+            pdf_mod.FORCE_SCANNED_PDF = old
+
+    def test_force_scanned_output_is_scanned_pdf(self):
+        """Force-scanned mode produces scanned_pdf metadata and image refs."""
+        pdf_bytes = _make_image_only_pdf(2)
+        parser = PDFParser(
+            file_name="forced.pdf", file_type="pdf", pdf_force_scanned="true"
+        )
+        doc = parser.parse_into_text(pdf_bytes)
+        self.assertEqual(doc.metadata.get("image_source_type"), "scanned_pdf")
+        self.assertEqual(doc.metadata.get("page_count"), 2)
+        self.assertEqual(doc.metadata.get("scanned_page_count"), 2)
+        self.assertEqual(doc.metadata.get("text_page_count"), 0)
+        self.assertEqual(doc.metadata.get("embedded_image_count"), 0)
+        self.assertEqual(doc.metadata.get("vector_figure_count"), 0)
+        self.assertIn("![forced_page_1.jpg]", doc.content)
+        self.assertIn("![forced_page_2.jpg]", doc.content)
+        self.assertEqual(len(doc.images), 2)
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_ppt_convert.py

@@ -0,0 +1,94 @@
+import shutil
+import unittest
+from pathlib import Path
+
+from docreader.parser.ppt_convert import (
+    convert_ppt_to_pptx_bytes,
+    is_ole_compound,
+    is_zip_openxml,
+    needs_ppt_to_pptx_conversion,
+    normalize_ppt_bytes,
+)
+
+TESTDATA = Path(__file__).resolve().parents[2] / "testdata" / "rag_test"
+LEGACY_PPT = TESTDATA / "ppt_old" / "en_38256.ppt"
+WMF_IMAGE_PPT = LEGACY_PPT
+IMAGE_HEAVY_PPT = TESTDATA / "ppt_old" / "en_41384.ppt"
+PPTX_SAMPLE = TESTDATA / "pptx" / "en_marker.pptx"
+
+
+class TestPptConvert(unittest.TestCase):
+    def test_legacy_ppt_magic(self):
+        if not LEGACY_PPT.is_file():
+            self.skipTest("legacy PPT fixture not available")
+        content = LEGACY_PPT.read_bytes()
+        self.assertTrue(is_ole_compound(content))
+        self.assertFalse(is_zip_openxml(content))
+        self.assertTrue(needs_ppt_to_pptx_conversion(content, "ppt"))
+
+    def test_pptx_does_not_need_conversion(self):
+        if not PPTX_SAMPLE.is_file():
+            self.skipTest("PPTX fixture not available")
+        content = PPTX_SAMPLE.read_bytes()
+        self.assertTrue(is_zip_openxml(content))
+        self.assertFalse(needs_ppt_to_pptx_conversion(content, "pptx"))
+
+    def test_normalize_pptx_passthrough(self):
+        if not PPTX_SAMPLE.is_file():
+            self.skipTest("PPTX fixture not available")
+        content = PPTX_SAMPLE.read_bytes()
+        out, ext = normalize_ppt_bytes(content, "pptx")
+        self.assertEqual(out, content)
+        self.assertEqual(ext, ".pptx")
+
+    def test_legacy_ppt_requires_soffice(self):
+        if not LEGACY_PPT.is_file():
+            self.skipTest("legacy PPT fixture not available")
+        if not shutil.which("soffice"):
+            with self.assertRaises(ValueError) as ctx:
+                normalize_ppt_bytes(LEGACY_PPT.read_bytes(), "ppt")
+            self.assertIn("LibreOffice", str(ctx.exception))
+            self.skipTest("LibreOffice not available")
+        converted = convert_ppt_to_pptx_bytes(LEGACY_PPT.read_bytes(), suffix=".ppt")
+        self.assertIsNotNone(converted)
+        self.assertTrue(is_zip_openxml(converted))
+        out, ext = normalize_ppt_bytes(LEGACY_PPT.read_bytes(), "ppt")
+        self.assertEqual(ext, ".pptx")
+        self.assertTrue(is_zip_openxml(out))
+
+    def test_wmf_legacy_ppt_extracts_rasterized_image(self):
+        if not shutil.which("soffice"):
+            self.skipTest("LibreOffice not available")
+        if not shutil.which("convert"):
+            self.skipTest("ImageMagick convert not available")
+        if not WMF_IMAGE_PPT.is_file():
+            self.skipTest("testdata missing")
+
+        from docreader.parser.markitdown_parser import MarkitdownParser
+
+        doc = MarkitdownParser(file_type="ppt").parse_into_text(
+            WMF_IMAGE_PPT.read_bytes()
+        )
+        self.assertEqual(len(doc.images), 1)
+        self.assertNotIn("bd10496_.jpg", doc.content)
+        self.assertIn("images/", doc.content)
+
+    def test_image_heavy_legacy_ppt_extracts_images(self):
+        if not shutil.which("soffice"):
+            self.skipTest("LibreOffice not available")
+        if not IMAGE_HEAVY_PPT.is_file():
+            self.skipTest("testdata missing")
+
+        from docreader.parser.markitdown_parser import MarkitdownParser
+
+        doc = MarkitdownParser(file_type="ppt").parse_into_text(
+            IMAGE_HEAVY_PPT.read_bytes()
+        )
+        self.assertGreaterEqual(len(doc.images), 2)
+        self.assertNotIn("![](.jpg)", doc.content)
+        for ref in doc.images:
+            self.assertTrue(ref.startswith("images/"))
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_ssrf.py

@@ -0,0 +1,61 @@
+import ipaddress
+import os
+import unittest
+from unittest.mock import patch
+
+from docreader.utils.ssrf import is_ssrf_safe_url, reset_ssrf_whitelist_cache_for_test
+
+
+class TestSSRFValidation(unittest.TestCase):
+    def setUp(self) -> None:
+        self._env_patch = patch.dict(
+            os.environ,
+            {"SSRF_WHITELIST": "", "SSRF_WHITELIST_EXTRA": ""},
+            clear=False,
+        )
+        self._env_patch.start()
+        reset_ssrf_whitelist_cache_for_test()
+
+    def tearDown(self) -> None:
+        self._env_patch.stop()
+        reset_ssrf_whitelist_cache_for_test()
+
+    def test_blocks_loopback_ip(self):
+        safe, reason = is_ssrf_safe_url("http://127.0.0.1:8080/page")
+        self.assertFalse(safe)
+        self.assertTrue(reason)
+
+    def test_blocks_restricted_hostname(self):
+        safe, reason = is_ssrf_safe_url("http://host.docker.internal/secret")
+        self.assertFalse(safe)
+        self.assertIn("restricted", reason)
+
+    def test_blocks_metadata_host(self):
+        safe, reason = is_ssrf_safe_url(
+            "http://169.254.169.254/latest/meta-data/iam/security-credentials/"
+        )
+        self.assertFalse(safe)
+        self.assertTrue(reason)
+
+    def test_blocks_invalid_port(self):
+        safe, reason = is_ssrf_safe_url("https://example.com:99999/path")
+        self.assertFalse(safe)
+        self.assertIn("invalid port", reason)
+
+    def test_blocks_ipv4_mapped_private_ipv6_resolution(self):
+        with patch(
+            "docreader.utils.ssrf._resolve_host_ips",
+            return_value=((ipaddress.ip_address("::ffff:127.0.0.1"),), None),
+        ):
+            safe, reason = is_ssrf_safe_url("https://example.invalid/path")
+        self.assertFalse(safe)
+        self.assertIn("restricted", reason)
+
+    def test_allows_public_https(self):
+        safe, reason = is_ssrf_safe_url("https://example.com/article")
+        self.assertTrue(safe, reason)
+        self.assertEqual(reason, "")
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_web_parser.py

@@ -0,0 +1,153 @@
+import os
+import unittest
+from unittest.mock import AsyncMock, patch
+
+from docreader.parser.web_parser import (
+    StdWebParser,
+    WebParseError,
+    WebParser,
+    _ScrapeResult,
+    build_visible_text_fallback,
+    extract_markdown_from_html,
+    install_ssrf_route_guard,
+)
+from docreader.utils.ssrf import is_ssrf_safe_url, reset_ssrf_whitelist_cache_for_test
+
+
+class TestWebParserHelpers(unittest.TestCase):
+    def setUp(self) -> None:
+        self._env_patch = patch.dict(
+            os.environ,
+            {"SSRF_WHITELIST": "", "SSRF_WHITELIST_EXTRA": ""},
+            clear=False,
+        )
+        self._env_patch.start()
+        reset_ssrf_whitelist_cache_for_test()
+
+    def tearDown(self) -> None:
+        self._env_patch.stop()
+        reset_ssrf_whitelist_cache_for_test()
+
+    def test_extract_markdown_empty_html(self):
+        self.assertIsNone(extract_markdown_from_html(""))
+        self.assertIsNone(extract_markdown_from_html("   "))
+
+    def test_extract_markdown_article_html(self):
+        html = """
+        <html><head><title>Demo</title></head><body>
+        <article><h1>Hello</h1><p>World paragraph with enough text for extraction.</p></article>
+        </body></html>
+        """
+        md = extract_markdown_from_html(html)
+        self.assertIsNotNone(md)
+        self.assertIn("Hello", md)
+
+    def test_build_fallback_too_short(self):
+        self.assertIsNone(build_visible_text_fallback("short"))
+        self.assertIsNone(build_visible_text_fallback(""))
+
+    def test_build_fallback_with_title(self):
+        text = "A" * 60
+        md = build_visible_text_fallback(text, page_title="WeKnora")
+        self.assertIsNotNone(md)
+        self.assertTrue(md.startswith("# WeKnora"))
+        self.assertIn(text, md)
+
+    def test_build_fallback_without_title(self):
+        text = "B" * 60
+        md = build_visible_text_fallback(text, page_title="")
+        self.assertEqual(md, text)
+
+    def test_install_ssrf_route_guard_is_importable(self):
+        self.assertTrue(callable(install_ssrf_route_guard))
+
+    def test_redirect_target_blocked_before_navigation(self):
+        safe, reason = is_ssrf_safe_url("http://127.0.0.1:39127/audit.txt")
+        self.assertFalse(safe)
+        self.assertTrue(reason)
+
+
+class TestStdWebParserFailures(unittest.TestCase):
+    """Scrape/parse failures must raise, not become indexable document body."""
+
+    def _parser(self) -> StdWebParser:
+        return StdWebParser(title="page")
+
+    def test_empty_scrape_raises_instead_of_error_body(self):
+        empty = _ScrapeResult(
+            html="",
+            visible_text="",
+            page_title="",
+            error="navigation failed: Timeout",
+        )
+        parser = self._parser()
+        with patch.object(parser, "scrape", new=AsyncMock(return_value=empty)):
+            with self.assertRaises(WebParseError) as ctx:
+                parser.parse_into_text(b"https://example.com/blocked")
+        message = str(ctx.exception)
+        self.assertIn("https://example.com/blocked", message)
+        self.assertIn("navigation failed", message)
+        self.assertNotIn("Error parsing web page:", message)
+
+    def test_empty_scrape_without_error_field_still_raises(self):
+        empty = _ScrapeResult(html="", visible_text="", page_title="")
+        parser = self._parser()
+        with patch.object(parser, "scrape", new=AsyncMock(return_value=empty)):
+            with self.assertRaises(WebParseError) as ctx:
+                parser.parse_into_text(b"https://example.com/blank")
+        self.assertIn("no HTML or visible text", str(ctx.exception))
+
+    def test_unextractable_page_raises_instead_of_error_body(self):
+        scrape = _ScrapeResult(
+            html="<html><body><div></div></body></html>",
+            visible_text="short",
+            page_title="",
+        )
+        parser = self._parser()
+        with patch.object(parser, "scrape", new=AsyncMock(return_value=scrape)):
+            with patch(
+                "docreader.parser.web_parser.extract_markdown_from_html",
+                return_value=None,
+            ):
+                with patch(
+                    "docreader.parser.web_parser.build_visible_text_fallback",
+                    return_value=None,
+                ):
+                    with self.assertRaises(WebParseError) as ctx:
+                        parser.parse_into_text(b"https://example.com/empty")
+        self.assertIn("Failed to parse web page", str(ctx.exception))
+        self.assertIn("https://example.com/empty", str(ctx.exception))
+
+    def test_successful_scrape_returns_markdown_document(self):
+        html = """
+        <html><head><title>Demo</title></head><body>
+        <article><h1>Hello</h1><p>World paragraph with enough text for extraction.</p></article>
+        </body></html>
+        """
+        scrape = _ScrapeResult(
+            html=html,
+            visible_text="Hello World paragraph with enough text for extraction.",
+            page_title="Demo",
+        )
+        parser = self._parser()
+        with patch.object(parser, "scrape", new=AsyncMock(return_value=scrape)):
+            doc = parser.parse_into_text(b"https://example.com/ok")
+        self.assertTrue(doc.is_valid())
+        self.assertNotIn("Error parsing web page:", doc.content)
+        self.assertIn("Hello", doc.content)
+
+    def test_pipeline_web_parser_does_not_index_scrape_error(self):
+        empty = _ScrapeResult(
+            html="",
+            visible_text="",
+            page_title="",
+            error="URL blocked by SSRF guard: loopback",
+        )
+        pipeline = WebParser(title="page")
+        with patch.object(StdWebParser, "scrape", new=AsyncMock(return_value=empty)):
+            with self.assertRaises(WebParseError):
+                pipeline.parse_into_text(b"https://example.com/ssrf")
+
+
+if __name__ == "__main__":
+    unittest.main()

docreader/tests/test_xmind_parser.py

@@ -0,0 +1,286 @@
+import io
+import json
+import unittest
+import zipfile
+from unittest.mock import patch
+
+from docreader.parser.xmind_parser import XMindParser
+
+
+def _xmind_zip(entries: dict[str, bytes | str]) -> bytes:
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive:
+        for name, value in entries.items():
+            payload = value.encode("utf-8") if isinstance(value, str) else value
+            archive.writestr(name, payload)
+    return buffer.getvalue()
+
+
+def _modern_xmind_bytes(sheets: list[dict]) -> bytes:
+    return _xmind_zip({"content.json": json.dumps(sheets, ensure_ascii=False)})
+
+
+def _classic_xmind_bytes(xml: str) -> bytes:
+    return _xmind_zip({"content.xml": xml})
+
+
+def _mark_entry_encrypted(payload: bytes) -> bytes:
+    marked = bytearray(payload)
+    local_header = marked.index(b"PK\x03\x04")
+    central_header = marked.index(b"PK\x01\x02")
+    for offset in (local_header + 6, central_header + 8):
+        flags = int.from_bytes(marked[offset : offset + 2], "little") | 0x1
+        marked[offset : offset + 2] = flags.to_bytes(2, "little")
+    return bytes(marked)
+
+
+class XMindParserModernTests(unittest.TestCase):
+    def test_finds_content_without_enumerating_unrelated_entries(self):
+        payload = _xmind_zip(
+            {
+                "attachments/preview.png": b"preview",
+                "content.json": json.dumps(
+                    [{"title": "Outline", "rootTopic": {"title": "Root"}}]
+                ),
+            }
+        )
+
+        with patch.object(
+            zipfile.ZipFile,
+            "namelist",
+            side_effect=AssertionError("archive entries must not be enumerated"),
+        ):
+            document = XMindParser().parse_into_text(payload)
+
+        self.assertEqual("# Outline\n\n- Root", document.content)
+
+    def test_parses_topic_hierarchy_and_plain_notes(self):
+        payload = _modern_xmind_bytes(
+            [
+                {
+                    "title": "Launch Plan",
+                    "rootTopic": {
+                        "title": "Release",
+                        "notes": {"plain": {"content": "Coordinate teams"}},
+                        "children": {
+                            "attached": [
+                                {
+                                    "title": "Backend",
+                                    "children": {
+                                        "attached": [{"title": "API freeze"}]
+                                    },
+                                }
+                            ]
+                        },
+                    },
+                }
+            ]
+        )
+
+        document = XMindParser(file_name="launch.xmind").parse_into_text(payload)
+
+        self.assertEqual(
+            "# Launch Plan\n\n"
+            "- Release\n"
+            "  > Coordinate teams\n"
+            "  - Backend\n"
+            "    - API freeze",
+            document.content,
+        )
+        self.assertEqual(document.metadata["source_format"], "xmind")
+        self.assertEqual(document.metadata["xmind_content_format"], "json")
+        self.assertEqual(document.metadata["sheet_count"], 1)
+        self.assertEqual(document.metadata["topic_count"], 3)
+        self.assertEqual(document.metadata["note_count"], 1)
+        self.assertEqual(document.metadata["file_size"], len(payload))
+
+    def test_renders_multiple_sheets_with_fallback_title(self):
+        payload = _modern_xmind_bytes(
+            [
+                {"title": "One", "rootTopic": {"title": "Alpha"}},
+                {"title": "  ", "rootTopic": {"title": "Beta"}},
+            ]
+        )
+
+        document = XMindParser().parse_into_text(payload)
+
+        self.assertEqual(
+            "# One\n\n- Alpha\n\n---\n\n# Sheet 2\n\n- Beta",
+            document.content,
+        )
+        self.assertEqual(document.metadata["sheet_count"], 2)
+        self.assertEqual(document.metadata["topic_count"], 2)
+
+    def test_promotes_children_of_blank_topic(self):
+        payload = _modern_xmind_bytes(
+            [
+                {
+                    "title": "Outline",
+                    "rootTopic": {
+                        "title": "  ",
+                        "children": {
+                            "attached": [
+                                {
+                                    "title": "Visible",
+                                    "children": {
+                                        "attached": [{"title": "Nested"}]
+                                    },
+                                }
+                            ]
+                        },
+                    },
+                }
+            ]
+        )
+
+        document = XMindParser().parse_into_text(payload)
+
+        self.assertEqual("# Outline\n\n- Visible\n  - Nested", document.content)
+        self.assertEqual(document.metadata["topic_count"], 2)
+
+    def test_renders_multiline_note_as_blockquotes(self):
+        payload = _modern_xmind_bytes(
+            [
+                {
+                    "title": "Notes",
+                    "rootTopic": {
+                        "title": "Root",
+                        "notes": {
+                            "plain": {"content": " First line \n\n Second line "}
+                        },
+                    },
+                }
+            ]
+        )
+
+        document = XMindParser().parse_into_text(payload)
+
+        self.assertEqual(
+            "# Notes\n\n- Root\n  > First line\n  >\n  > Second line",
+            document.content,
+        )
+
+
+class XMindParserClassicTests(unittest.TestCase):
+    def test_parses_namespaced_xml_hierarchy_and_notes(self):
+        payload = _classic_xmind_bytes(
+            """<?xml version="1.0" encoding="UTF-8"?>
+<xmap-content xmlns="urn:xmind:xmap:xmlns:content:2.0">
+  <sheet>
+    <title>Architecture</title>
+    <topic>
+      <title>Platform</title>
+      <notes><plain>Owns ingress</plain></notes>
+      <children>
+        <topics type="attached">
+          <topic><title>Gateway</title></topic>
+        </topics>
+      </children>
+    </topic>
+  </sheet>
+</xmap-content>
+"""
+        )
+
+        document = XMindParser().parse_into_text(payload)
+
+        self.assertEqual(
+            "# Architecture\n\n- Platform\n  > Owns ingress\n  - Gateway",
+            document.content,
+        )
+        self.assertEqual(document.metadata["xmind_content_format"], "xml")
+        self.assertEqual(document.metadata["sheet_count"], 1)
+        self.assertEqual(document.metadata["topic_count"], 2)
+        self.assertEqual(document.metadata["note_count"], 1)
+
+    def test_prefers_content_json_when_both_entries_exist(self):
+        xml = """<xmap-content><sheet><title>XML</title>
+<topic><title>XML topic</title></topic></sheet></xmap-content>"""
+        json_content = json.dumps(
+            [{"title": "JSON", "rootTopic": {"title": "JSON topic"}}]
+        )
+        payload = _xmind_zip(
+            {"content.xml": xml, "content.json": json_content}
+        )
+
+        document = XMindParser().parse_into_text(payload)
+
+        self.assertEqual("# JSON\n\n- JSON topic", document.content)
+        self.assertEqual(document.metadata["xmind_content_format"], "json")
+
+
+class XMindParserValidationTests(unittest.TestCase):
+    def test_rejects_invalid_zip(self):
+        with self.assertRaises(ValueError) as context:
+            XMindParser().parse_into_text(b"not a ZIP archive")
+
+        self.assertEqual(str(context.exception), "invalid XMind archive")
+
+    def test_rejects_archive_without_supported_content(self):
+        payload = _xmind_zip({"manifest.json": "{}"})
+
+        with self.assertRaises(ValueError) as context:
+            XMindParser().parse_into_text(payload)
+
+        self.assertEqual(
+            str(context.exception),
+            "XMind archive is missing content.json or content.xml",
+        )
+
+    def test_rejects_malformed_json(self):
+        payload = _xmind_zip({"content.json": "{"})
+
+        with self.assertRaises(ValueError) as context:
+            XMindParser().parse_into_text(payload)
+
+        self.assertEqual(str(context.exception), "invalid XMind content.json")
+
+    def test_rejects_malformed_xml(self):
+        payload = _classic_xmind_bytes("<xmap-content>")
+
+        with self.assertRaises(ValueError) as context:
+            XMindParser().parse_into_text(payload)
+
+        self.assertEqual(str(context.exception), "invalid XMind content.xml")
+
+    def test_rejects_archive_without_renderable_topics(self):
+        payload = _modern_xmind_bytes(
+            [{"title": "Empty", "rootTopic": {"title": " "}}]
+        )
+
+        with self.assertRaises(ValueError) as context:
+            XMindParser().parse_into_text(payload)
+
+        self.assertEqual(
+            str(context.exception),
+            "XMind archive contains no renderable topics",
+        )
+
+    def test_rejects_content_entry_over_limit(self):
+        payload = _xmind_zip({"content.json": "12345"})
+
+        with patch("docreader.parser.xmind_parser.MAX_CONTENT_BYTES", 4):
+            with self.assertRaises(ValueError) as context:
+                XMindParser().parse_into_text(payload)
+
+        self.assertEqual(
+            str(context.exception),
+            "XMind content entry exceeds the 32 MiB limit",
+        )
+
+    def test_rejects_encrypted_content_entry(self):
+        payload = _mark_entry_encrypted(
+            _xmind_zip({"content.json": "[]"})
+        )
+
+        with self.assertRaises(ValueError) as context:
+            XMindParser().parse_into_text(payload)
+
+        self.assertEqual(
+            str(context.exception),
+            "encrypted XMind content is not supported",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()