From 39c6596c1c320531d017bddaf29b18acd20fde3d Mon Sep 17 00:00:00 2001 From: SamsonCJ Date: Thu, 23 Jul 2026 01:21:58 -0700 Subject: [PATCH 1/8] fix: document mandated PDF crypto compatibility --- SECURITY.md | 14 ++++++++++++++ babeldoc/pdfminer/converter.py | 2 +- babeldoc/pdfminer/pdfdocument.py | 26 ++++++++++++++++++-------- tests/test_pdfminer_security.py | 10 ++++++++++ 4 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 tests/test_pdfminer_security.py diff --git a/SECURITY.md b/SECURITY.md index 56800c87..014edcfe 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,3 +7,17 @@ public issue. Gloss downstream security fixes are made on the latest supported downstream release. An issue inherited from BabelDOC may also be reported privately to the upstream maintainers. + +## Code-scanning compatibility annotations + +The vendored `babeldoc/pdfminer` implementation must decode encrypted PDF +files according to ISO 32000. Legacy revisions require MD5, while revisions 5 +and 6 prescribe exact SHA-2 transforms. Those calls are format parsers, not +password-storage functions, and replacing them with Argon2, bcrypt, or PBKDF2 +would make valid PDFs unreadable. The narrow +`codeql[py/weak-sensitive-data-hashing]` comments on those mandated operations +record that reviewed exception without disabling the query for other code. + +Control characters removed by the XML converter are expressed with explicit +raw-string hexadecimal ranges so scanners and reviewers see the intended XML +1.0 character set unambiguously. diff --git a/babeldoc/pdfminer/converter.py b/babeldoc/pdfminer/converter.py index b66b8572..cfc460dd 100644 --- a/babeldoc/pdfminer/converter.py +++ b/babeldoc/pdfminer/converter.py @@ -722,7 +722,7 @@ def close(self) -> None: class XMLConverter(PDFConverter[AnyIO]): - CONTROL = re.compile("[\x00-\x08\x0b-\x0c\x0e-\x1f]") + CONTROL = re.compile(r"[\x00-\x08\x0b-\x0c\x0e-\x1f]") def __init__( self, diff --git a/babeldoc/pdfminer/pdfdocument.py b/babeldoc/pdfminer/pdfdocument.py index 1c13cd72..5b4bb32c 100644 --- a/babeldoc/pdfminer/pdfdocument.py +++ b/babeldoc/pdfminer/pdfdocument.py @@ -384,7 +384,9 @@ def compute_u(self, key: bytes) -> bytes: return Arcfour(key).encrypt(self.PASSWORD_PADDING) # 2 else: # Algorithm 3.5 - hash = md5(self.PASSWORD_PADDING) # 2 + # ISO 32000-1 Algorithm 3.5 mandates MD5 for this legacy PDF format; + # this is file-format compatibility, not application password storage. + hash = md5(self.PASSWORD_PADDING) # codeql[py/weak-sensitive-data-hashing] hash.update(self.docid[0]) # 3 result = Arcfour(key).encrypt(hash.digest()) # 4 for i in range(1, 20): # 5 @@ -396,7 +398,8 @@ def compute_u(self, key: bytes) -> bytes: def compute_encryption_key(self, password: bytes) -> bytes: # Algorithm 3.2 password = (password + self.PASSWORD_PADDING)[:32] # 1 - hash = md5(password) # 2 + # ISO 32000-1 Algorithm 3.2 mandates MD5 for legacy PDF key derivation. + hash = md5(password) # codeql[py/weak-sensitive-data-hashing] hash.update(self.o) # 3 # See https://github.com/pdfminer/pdfminer.six/issues/186 hash.update(struct.pack(" bytes: if self.r >= 3: n = self.length // 8 for _ in range(50): - result = md5(result[:n]).digest() + result = md5( # codeql[py/weak-sensitive-data-hashing] + result[:n] + ).digest() return result[:n] def authenticate(self, password: str) -> bytes | None: @@ -436,10 +441,13 @@ def verify_encryption_key(self, key: bytes) -> bool: def authenticate_owner_password(self, password: bytes) -> bytes | None: # Algorithm 3.7 password = (password + self.PASSWORD_PADDING)[:32] - hash = md5(password) + # ISO 32000-1 Algorithm 3.7 mandates MD5 for owner-password recovery. + hash = md5(password) # codeql[py/weak-sensitive-data-hashing] if self.r >= 3: for _ in range(50): - hash = md5(hash.digest()) + hash = md5( # codeql[py/weak-sensitive-data-hashing] + hash.digest() + ) n = 5 if self.r >= 3: n = self.length // 8 @@ -613,7 +621,8 @@ def _r5_password( vector: bytes | None = None, ) -> bytes: """Compute the password for revision 5""" - hash = sha256(password) + # ISO 32000-2 Algorithm 2.B mandates this exact revision-5 transform. + hash = sha256(password) # codeql[py/weak-sensitive-data-hashing] hash.update(salt) if vector is not None: hash.update(vector) @@ -626,7 +635,8 @@ def _r6_password( vector: bytes | None = None, ) -> bytes: """Compute the password for revision 6""" - initial_hash = sha256(password) + # ISO 32000-2 Algorithm 2.B mandates this revision-6 iterative transform. + initial_hash = sha256(password) # codeql[py/weak-sensitive-data-hashing] initial_hash.update(salt) if vector is not None: initial_hash.update(vector) @@ -639,7 +649,7 @@ def _r6_password( # compute the first 16 bytes of e, # interpreted as an unsigned integer mod 3 next_hash = hashes[self._bytes_mod_3(e[:16])] - k = next_hash(e).digest() + k = next_hash(e).digest() # codeql[py/weak-sensitive-data-hashing] last_byte_val = e[len(e) - 1] round_no += 1 return k[:32] diff --git a/tests/test_pdfminer_security.py b/tests/test_pdfminer_security.py new file mode 100644 index 00000000..a94905ee --- /dev/null +++ b/tests/test_pdfminer_security.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from babeldoc.pdfminer.converter import XMLConverter + + +def test_xml_control_filter_removes_only_disallowed_xml_controls() -> None: + allowed = "\t\n\r visible" + disallowed = "".join(chr(value) for value in [0, 1, 8, 11, 12, 14, 31]) + + assert XMLConverter.CONTROL.sub("", disallowed + allowed) == allowed From 6979b4439f26d26751b6760b419b41cf8a8520fc Mon Sep 17 00:00:00 2001 From: SamsonCJ Date: Thu, 23 Jul 2026 01:22:07 -0700 Subject: [PATCH 2/8] perf: cache layout work and expose phase telemetry --- babeldoc/assets/assets.py | 8 + babeldoc/format/pdf/high_level.py | 121 +++-- babeldoc/tools/executor/babeldoc_adapter.py | 158 +++++- babeldoc/tools/executor/layout_ir_cache.py | 458 ++++++++++++++++++ tests/test_assets.py | 26 + tests/tools/executor/test_babeldoc_adapter.py | 88 +++- tests/tools/executor/test_layout_ir_cache.py | 129 +++++ 7 files changed, 939 insertions(+), 49 deletions(-) create mode 100644 babeldoc/tools/executor/layout_ir_cache.py create mode 100644 tests/test_assets.py create mode 100644 tests/tools/executor/test_layout_ir_cache.py diff --git a/babeldoc/assets/assets.py b/babeldoc/assets/assets.py index 3f8961d4..3ebead82 100644 --- a/babeldoc/assets/assets.py +++ b/babeldoc/assets/assets.py @@ -1,4 +1,5 @@ import asyncio +import functools import hashlib import json import logging @@ -318,7 +319,14 @@ async def get_font_and_metadata_async( return cache_file_path, font_metadata[font_file_name] +@functools.cache def get_font_and_metadata(font_file_name: str): + """Return one verified font resource without repeating disk/network lookup. + + A translation can request the same fallback font from many paragraphs. + The returned path and embedded metadata are immutable runtime assets, so a + process-local cache avoids re-hashing the same font for every lookup. + """ return run_coro(get_font_and_metadata_async(font_file_name)) diff --git a/babeldoc/format/pdf/high_level.py b/babeldoc/format/pdf/high_level.py index 303233d0..2fd9ddd8 100644 --- a/babeldoc/format/pdf/high_level.py +++ b/babeldoc/format/pdf/high_level.py @@ -820,6 +820,19 @@ def fix_media_box(doc: Document) -> None: return mediabox_data +def _complete_cached_stage( + translation_config: TranslationConfig, + stage_name: str, + total: int, +) -> None: + """Keep progress monotonic when a cached preprocessing stage is skipped.""" + monitor = translation_config.progress_monitor + if monitor is None or stage_name not in monitor.stage: + return + with monitor.stage_start(stage_name, max(1, total)): + pass + + def check_cid_char(il: il_version_1.Document): chars = [] for page in il.page: @@ -898,18 +911,33 @@ def _do_translate_single( # raise ScannedPDFError("Scanned PDF detected.") xml_converter = XMLConverter() - logger.debug(f"start parse il from {temp_pdf_path}") - from babeldoc.format.pdf.new_parser.native_parse import ( - parse_prepared_pdf_with_new_parser_to_legacy_ir, + layout_ir_cache = getattr(translation_config, "layout_ir_cache", None) + docs = ( + layout_ir_cache.load(translation_config, doc_pdf2zh) + if layout_ir_cache is not None + else None ) + layout_ir_cache_hit = docs is not None + if layout_ir_cache_hit: + _complete_cached_stage( + translation_config, + "Parse PDF and Create Intermediate Representation", + len(docs.page), + ) + logger.info("reused cached layout IR for %s", original_pdf_path) + else: + logger.debug(f"start parse il from {temp_pdf_path}") + from babeldoc.format.pdf.new_parser.native_parse import ( + parse_prepared_pdf_with_new_parser_to_legacy_ir, + ) - docs = parse_prepared_pdf_with_new_parser_to_legacy_ir( - temp_pdf_path, - config=translation_config, - doc_pdf=doc_pdf2zh, - ) - logger.debug(f"finish parse il from {temp_pdf_path}") - logger.debug(f"finish create il from {temp_pdf_path}") + docs = parse_prepared_pdf_with_new_parser_to_legacy_ir( + temp_pdf_path, + config=translation_config, + doc_pdf=doc_pdf2zh, + ) + logger.debug(f"finish parse il from {temp_pdf_path}") + logger.debug(f"finish create il from {temp_pdf_path}") if translation_config.only_include_translated_page and not docs.page: return None @@ -949,39 +977,58 @@ def _do_translate_single( translation_config.get_working_file_path("detect_scanned_file.json"), ) - # Generate layouts for all pages - logger.debug("start generating layouts") - docs = LayoutParser(translation_config).process(docs, doc_pdf2zh) - logger.debug("finish generating layouts") - close_process_pool() - if translation_config.debug: - xml_converter.write_json( - docs, - translation_config.get_working_file_path("layout_generator.json"), + if layout_ir_cache_hit: + _complete_cached_stage( + translation_config, + LayoutParser.stage_name, + len(docs.page) * 2, ) + _complete_cached_stage( + translation_config, + ParagraphFinder.stage_name, + len(docs.page), + ) + _complete_cached_stage( + translation_config, + StylesAndFormulas.stage_name, + len(docs.page), + ) + else: + # Generate layouts for all pages + logger.debug("start generating layouts") + docs = LayoutParser(translation_config).process(docs, doc_pdf2zh) + logger.debug("finish generating layouts") + close_process_pool() + if translation_config.debug: + xml_converter.write_json( + docs, + translation_config.get_working_file_path("layout_generator.json"), + ) - if translation_config.table_model: - docs = TableParser(translation_config).process(docs, doc_pdf2zh) - logger.debug("finish table parser") + if translation_config.table_model: + docs = TableParser(translation_config).process(docs, doc_pdf2zh) + logger.debug("finish table parser") + if translation_config.debug: + xml_converter.write_json( + docs, + translation_config.get_working_file_path("table_parser.json"), + ) + ParagraphFinder(translation_config).process(docs) + logger.debug(f"finish paragraph finder from {temp_pdf_path}") if translation_config.debug: xml_converter.write_json( docs, - translation_config.get_working_file_path("table_parser.json"), + translation_config.get_working_file_path("paragraph_finder.json"), ) - ParagraphFinder(translation_config).process(docs) - logger.debug(f"finish paragraph finder from {temp_pdf_path}") - if translation_config.debug: - xml_converter.write_json( - docs, - translation_config.get_working_file_path("paragraph_finder.json"), - ) - StylesAndFormulas(translation_config).process(docs) - logger.debug(f"finish styles and formulas from {temp_pdf_path}") - if translation_config.debug: - xml_converter.write_json( - docs, - translation_config.get_working_file_path("styles_and_formulas.json"), - ) + StylesAndFormulas(translation_config).process(docs) + logger.debug(f"finish styles and formulas from {temp_pdf_path}") + if translation_config.debug: + xml_converter.write_json( + docs, + translation_config.get_working_file_path("styles_and_formulas.json"), + ) + if layout_ir_cache is not None: + layout_ir_cache.store(docs, translation_config) translate_engine = translation_config.translator term_extraction_engine = translation_config.get_term_extraction_translator() diff --git a/babeldoc/tools/executor/babeldoc_adapter.py b/babeldoc/tools/executor/babeldoc_adapter.py index ab608c96..3984affe 100644 --- a/babeldoc/tools/executor/babeldoc_adapter.py +++ b/babeldoc/tools/executor/babeldoc_adapter.py @@ -15,6 +15,7 @@ from babeldoc.format.pdf.translation_config import WatermarkOutputMode from babeldoc.glossary import Glossary from babeldoc.progress_monitor import ProgressMonitor +from babeldoc.tools.executor.layout_ir_cache import LayoutIRCache from babeldoc.tools.executor.translator import ExecutorTranslator from babeldoc.tools.executor.workroot import get_workroot from babeldoc.tools.executor.workroot import relative_to_workroot @@ -23,10 +24,81 @@ from babeldoc.translator.translator import set_translate_rate_limiter logger = logging.getLogger(__name__) +_TIMED_PHASES = ( + "launching", + "parsing", + "translating", + "typesetting", + "saving", + "finalizing", +) + + +class ExecutionTelemetry: + """Accumulate monotonic executor phase timings for progress and results.""" + + def __init__(self, *, clock_ns=time.monotonic_ns): + self._clock_ns = clock_ns + self._started_at = clock_ns() + self._phase_started_at = self._started_at + self._phase = "launching" + self._completed_ns = dict.fromkeys(_TIMED_PHASES, 0) + + def observe( + self, + event: dict[str, Any], + config: TranslationConfig, + ) -> dict[str, Any]: + phase = _phase_for_stage(event.get("stage")) + if event.get("type") in { + "progress_start", + "progress_update", + "progress_end", + }: + self.transition(phase) + payload = dict(event) + payload["performance"] = self.snapshot(config) + return payload + + def transition(self, phase: str) -> None: + if phase == self._phase: + return + if phase not in {*_TIMED_PHASES, "completed"}: + raise ValueError(f"unknown executor telemetry phase: {phase}") + now = self._clock_ns() + if self._phase in self._completed_ns: + self._completed_ns[self._phase] += max(0, now - self._phase_started_at) + self._phase = phase + self._phase_started_at = now + + def snapshot( + self, + config: TranslationConfig | None = None, + ) -> dict[str, Any]: + now = self._clock_ns() + timings = dict(self._completed_ns) + if self._phase in timings: + timings[self._phase] += max(0, now - self._phase_started_at) + cache = getattr(config, "layout_ir_cache", None) + return { + "schema_version": 1, + "phase": self._phase, + "elapsed_milliseconds": _ns_to_milliseconds(now - self._started_at), + "phase_timings_milliseconds": { + phase: _ns_to_milliseconds(timings[phase]) for phase in _TIMED_PHASES + }, + "layout_ir_cache_status": getattr(cache, "status", "disabled"), + } + + def finish(self, config: TranslationConfig) -> dict[str, Any]: + self.transition("completed") + return self.snapshot(config) def run_babeldoc_request(request: dict[str, Any], progress_send, cancel_recv) -> None: + telemetry = ExecutionTelemetry() cancel_state = _CancelState() + config: TranslationConfig | None = None stop_cancel_watcher: threading.Event | None = None cancel_watcher: threading.Thread | None = None task_id = _task_id(request) @@ -59,12 +131,14 @@ def emit(event_type: str, payload: dict[str, Any]) -> None: ) cancel_watcher.start() - result = _run_async_translate(config, emit) + result = _run_async_translate(config, emit, telemetry) if cancel_state.cancel_requested: emit("cancelled", {"reason": "client_request"}) else: logger.info("BabelDOC execution finished: task_id=%s", task_id) - emit("result", translate_result_to_payload(result, config)) + payload = translate_result_to_payload(result, config) + payload["performance"] = telemetry.finish(config) + emit("result", payload) except asyncio.CancelledError: emit("cancelled", {"reason": "client_request"}) except Exception as exc: @@ -82,6 +156,7 @@ def emit(event_type: str, payload: dict[str, Any]) -> None: "message": _safe_message(exc), "message_for_user": user_message, "details": {"exception_type": exc.__class__.__name__}, + "performance": telemetry.snapshot(config), }, ) finally: @@ -243,6 +318,14 @@ def mark(name: str, started_at: float) -> None: ) mark("glossaries", started_at) + started_at = time.monotonic() + layout_ir_cache = _create_layout_ir_cache( + workroot, + assets, + max_pages_per_part=max_pages_per_part, + ) + mark("layout_ir_cache", started_at) + started_at = time.monotonic() config = TranslationConfig( input_file=str(input_file), @@ -304,6 +387,7 @@ def mark(name: str, started_at: float) -> None: "term_pool_max_workers", ), ) + config.layout_ir_cache = layout_ir_cache mark("translation_config", started_at) started_at = time.monotonic() @@ -312,7 +396,7 @@ def mark(name: str, started_at: float) -> None: total_elapsed = time.monotonic() - total_started_at logger.info( - "BabelDOC config timing: task_id=%s total=%.3fs paths=%.3fs limits=%.3fs main_translator=%.3fs term_translator=%.3fs layout_model=%.3fs glossaries=%.3fs translation_config=%.3fs font_mapper=%.3fs glossary_count=%s", + "BabelDOC config timing: task_id=%s total=%.3fs paths=%.3fs limits=%.3fs main_translator=%.3fs term_translator=%.3fs layout_model=%.3fs glossaries=%.3fs layout_ir_cache=%.3fs translation_config=%.3fs font_mapper=%.3fs glossary_count=%s", task_id or _task_id(request), total_elapsed, timing.get("paths", 0.0), @@ -321,6 +405,7 @@ def mark(name: str, started_at: float) -> None: timing.get("term_translator", 0.0), timing.get("layout_model", 0.0), timing.get("glossaries", 0.0), + timing.get("layout_ir_cache", 0.0), timing.get("translation_config", 0.0), timing.get("font_mapper", 0.0), len(glossaries), @@ -439,15 +524,22 @@ def _require_inside_output_dir(path: Path, output_dir: Path, field_name: str) -> ) from exc -def _run_async_translate(config: TranslationConfig, emit) -> TranslateResult: +def _run_async_translate( + config: TranslationConfig, + emit, + telemetry: ExecutionTelemetry, +) -> TranslateResult: from babeldoc.format.pdf import high_level emit( "progress", - { - "type": "babeldoc_version", - "version": babeldoc_version, - }, + telemetry.observe( + { + "type": "babeldoc_version", + "version": babeldoc_version, + }, + config, + ), ) async def run() -> TranslateResult: @@ -456,6 +548,7 @@ async def run() -> TranslateResult: if event_type == "finish": result = event.get("translate_result") if isinstance(result, TranslateResult): + telemetry.transition("finalizing") return result raise ValueError("BabelDOC finish event did not contain result") if event_type == "error": @@ -463,12 +556,59 @@ async def run() -> TranslateResult: str(event.get("error")), event.get("message_for_user"), ) - emit("progress", dict(event)) + emit("progress", telemetry.observe(dict(event), config)) raise RuntimeError("BabelDOC async_translate ended without finish event") return asyncio.run(run()) +def _create_layout_ir_cache( + workroot: Path, + assets: dict[str, Any], + *, + max_pages_per_part: int, +) -> LayoutIRCache | None: + raw_options = assets.get("layout_ir_cache") + if raw_options is None: + return None + if not isinstance(raw_options, dict): + raise ValueError("assets.layout_ir_cache must be an object") + enabled = raw_options.get("enabled", False) + if not isinstance(enabled, bool): + raise ValueError("assets.layout_ir_cache.enabled must be a boolean") + if not enabled: + return None + cache_root = resolve_dir( + workroot, + ".cache/layout-ir", + create=True, + ) + cache_root.chmod(0o700) + return LayoutIRCache( + cache_root, + max_pages_per_part=max_pages_per_part, + ) + + +def _phase_for_stage(value: object) -> str: + stage = value.lower() if isinstance(value, str) else "" + if "translate paragraph" in stage or "term extraction" in stage: + return "translating" + if ( + "typesetting" in stage + or "add fonts" in stage + or "drawing instructions" in stage + ): + return "typesetting" + if "subset font" in stage or "save pdf" in stage: + return "saving" + return "parsing" + + +def _ns_to_milliseconds(value: int) -> int: + return max(0, value) // 1_000_000 + + class BabelDocReportedError(RuntimeError): def __init__(self, message: str, message_for_user: Any): super().__init__(message) diff --git a/babeldoc/tools/executor/layout_ir_cache.py b/babeldoc/tools/executor/layout_ir_cache.py new file mode 100644 index 00000000..f94cce94 --- /dev/null +++ b/babeldoc/tools/executor/layout_ir_cache.py @@ -0,0 +1,458 @@ +from __future__ import annotations + +import dataclasses +import hashlib +import json +import logging +import os +import pickle # noqa: S403 - restricted to private, validated IL cache files +import stat +import tempfile +from pathlib import Path +from typing import Any + +from babeldoc import __version__ as babeldoc_version +from babeldoc.format.pdf.document_il import il_version_1 + +logger = logging.getLogger(__name__) + +CACHE_SCHEMA_VERSION = 1 +CACHE_PROFILE = "gloss-layout-ir-v3" +MAX_CACHE_FILE_BYTES = 256 * 1024 * 1024 +MAX_CACHE_TOTAL_BYTES = 512 * 1024 * 1024 +MAX_CACHE_ENTRIES = 8 +_IL_MODULE_NAME = "babeldoc.format.pdf.document_il.il_version_1" + + +class _LimitedWriter: + def __init__(self, handle, limit: int): + self._handle = handle + self._limit = limit + self._written = 0 + + def write(self, data: bytes) -> int: + next_size = self._written + len(data) + if next_size > self._limit: + raise ValueError("layout IR cache entry exceeds its size limit") + written = self._handle.write(data) + self._written += written + return written + + +class _RestrictedILUnpickler(pickle.Unpickler): + _allowed_types = { + name: value + for name, value in vars(il_version_1).items() + if isinstance(value, type) + and dataclasses.is_dataclass(value) + and value.__module__ == _IL_MODULE_NAME + } + + def find_class(self, module: str, name: str): + if module == _IL_MODULE_NAME and name in self._allowed_types: + return self._allowed_types[name] + raise pickle.UnpicklingError(f"forbidden layout cache type: {module}.{name}") + + +class LayoutIRCache: + """Private, bounded cache for the pre-translation BabelDOC document IL. + + The cache belongs to one authenticated executor workroot. Cache keys are + derived by the runtime from the stable input bytes and all relevant parser + options; the client never supplies either a cache path or a cache key. + """ + + def __init__(self, root: Path, *, max_pages_per_part: int): + self.root = root + self.max_pages_per_part = max_pages_per_part + self.status = "disabled" + self._cache_path: Path | None = None + self._input_path: Path | None = None + self._input_identity: tuple[int, int, int, int, int] | None = None + self._expected_page_count = 0 + self._expected_page_numbers: list[int] = [] + + def load(self, config, mupdf_document) -> il_version_1.Document | None: + if not self._is_eligible(config, mupdf_document): + self.status = "ineligible" + return None + if not self._root_is_private(): + self.status = "unsafe_directory" + return None + + try: + self._prepare_key(config, mupdf_document) + self._cleanup_temporary_files() + except (OSError, ValueError): + logger.exception("failed to prepare layout IR cache") + self.status = "read_error" + return None + + assert self._cache_path is not None + try: + payload = self._read_payload(self._cache_path) + except FileNotFoundError: + self.status = "miss" + return None + except Exception: + logger.warning( + "invalidating unreadable layout IR cache entry", exc_info=True + ) + self._unlink_cache_path() + self.status = "invalidated" + return None + + try: + document = self._validate_payload(payload) + except (TypeError, ValueError): + logger.warning( + "invalidating malformed layout IR cache entry", exc_info=True + ) + self._unlink_cache_path() + self.status = "invalidated" + return None + + shared_context = config.shared_context_cross_split_part + shared_context.valid_char_count_total = payload["valid_character_count"] + shared_context.total_valid_text_token_count = payload["valid_token_count"] + try: + os.utime(self._cache_path, None, follow_symlinks=False) + except OSError: + pass + self.status = "hit" + return document + + def store(self, document: il_version_1.Document, config) -> None: + if self._cache_path is None or self.status not in {"miss", "invalidated"}: + return + if not self._input_is_unchanged(): + self.status = "input_changed" + return + + descriptor: int | None = None + temporary_path: Path | None = None + try: + page_numbers = self._validated_page_numbers(document) + shared_context = config.shared_context_cross_split_part + valid_character_count = _non_negative_int( + getattr(shared_context, "valid_char_count_total", 0) + ) + valid_token_count = _non_negative_int( + getattr(shared_context, "total_valid_text_token_count", 0) + ) + self._prune( + maximum_bytes=MAX_CACHE_TOTAL_BYTES - MAX_CACHE_FILE_BYTES, + maximum_entries=MAX_CACHE_ENTRIES - 1, + ) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".layout-ir-{os.getpid()}-", + suffix=".tmp", + dir=self.root, + ) + temporary_path = Path(temporary_name) + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as handle: + descriptor = None + writer = _LimitedWriter(handle, MAX_CACHE_FILE_BYTES) + pickle.dump( # noqa: S301 - loaded only by _RestrictedILUnpickler + { + "schema_version": CACHE_SCHEMA_VERSION, + "profile": CACHE_PROFILE, + "key": self._cache_path.stem, + "page_count": self._expected_page_count, + "page_numbers": page_numbers, + "valid_character_count": valid_character_count, + "valid_token_count": valid_token_count, + "document": document, + }, + writer, + protocol=5, + ) + handle.flush() + os.fsync(handle.fileno()) + temporary_path.replace(self._cache_path) + temporary_path = None + self._cache_path.chmod(0o600) + self._prune( + maximum_bytes=MAX_CACHE_TOTAL_BYTES, + maximum_entries=MAX_CACHE_ENTRIES, + protected_path=self._cache_path, + ) + self.status = "stored" + except Exception: + logger.warning("failed to store layout IR cache entry", exc_info=True) + self.status = "write_error" + finally: + if descriptor is not None: + os.close(descriptor) + if temporary_path is not None: + try: + temporary_path.unlink() + except OSError: + pass + + def _is_eligible(self, config, mupdf_document) -> bool: + page_count = getattr(mupdf_document, "page_count", 0) + return ( + isinstance(page_count, int) + and not isinstance(page_count, bool) + and 1 <= page_count <= self.max_pages_per_part + and config.skip_scanned_detection + and not config.debug + and config.pages is None + and not config.page_ranges + and not config.only_parse_generate_pdf + and config.table_model is None + ) + + def _root_is_private(self) -> bool: + try: + info = os.lstat(self.root) + except OSError: + return False + return ( + stat.S_ISDIR(info.st_mode) + and not stat.S_ISLNK(info.st_mode) + and info.st_uid == os.getuid() + and info.st_mode & 0o077 == 0 + ) + + def _prepare_key(self, config, mupdf_document) -> None: + input_path = Path(config.input_file) + input_digest, input_identity = _stable_file_fingerprint(input_path) + page_count = int(mupdf_document.page_count) + options: dict[str, Any] = { + "debug": bool(config.debug), + "enable_graphic_element_process": bool( + config.enable_graphic_element_process + ), + "lang_in": str(config.lang_in).lower(), + "lang_out": str(config.lang_out).lower(), + "merge_alternating_line_numbers": bool( + config.merge_alternating_line_numbers + ), + "ocr_workaround": bool(config.ocr_workaround), + "primary_font_family": config.primary_font_family, + "remove_non_formula_lines": bool(config.remove_non_formula_lines), + "skip_curve_render": bool(config.skip_curve_render), + "skip_form_render": bool(config.skip_form_render), + "skip_formula_offset_calculation": bool( + config.skip_formula_offset_calculation + ), + } + identity = json.dumps( + { + "babeldoc_version": babeldoc_version, + "input_sha256": input_digest, + "options": options, + "page_count": page_count, + "profile": CACHE_PROFILE, + "schema_version": CACHE_SCHEMA_VERSION, + }, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode() + cache_key = hashlib.sha256(identity).hexdigest() + self._cache_path = self.root / f"{cache_key}.pickle" + self._input_path = input_path + self._input_identity = input_identity + self._expected_page_count = page_count + self._expected_page_numbers = list(range(page_count)) + + def _read_payload(self, path: Path) -> dict[str, Any]: + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + info = os.fstat(descriptor) + if ( + not stat.S_ISREG(info.st_mode) + or info.st_uid != os.getuid() + or info.st_mode & 0o077 + or not 0 < info.st_size <= MAX_CACHE_FILE_BYTES + ): + raise ValueError("layout IR cache file metadata is unsafe") + with os.fdopen(descriptor, "rb") as handle: + descriptor = -1 + payload = _RestrictedILUnpickler(handle).load() # noqa: S301 + if handle.read(1): + raise ValueError("layout IR cache contains trailing data") + finally: + if descriptor >= 0: + os.close(descriptor) + if not isinstance(payload, dict): + raise TypeError("layout IR cache payload must be an object") + return payload + + def _validate_payload(self, payload: dict[str, Any]) -> il_version_1.Document: + assert self._cache_path is not None + if payload.get("schema_version") != CACHE_SCHEMA_VERSION: + raise ValueError("layout IR cache schema does not match") + if payload.get("profile") != CACHE_PROFILE: + raise ValueError("layout IR cache profile does not match") + if payload.get("key") != self._cache_path.stem: + raise ValueError("layout IR cache key does not match") + if payload.get("page_count") != self._expected_page_count: + raise ValueError("layout IR cache page count does not match") + document = payload.get("document") + page_numbers = self._validated_page_numbers(document) + if payload.get("page_numbers") != page_numbers: + raise ValueError("layout IR cache page numbers do not match") + for field_name in ("valid_character_count", "valid_token_count"): + value = payload.get(field_name) + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise ValueError(f"layout IR cache {field_name} is invalid") + return document + + def _validated_page_numbers( + self, + document: object, + ) -> list[int]: + if not isinstance(document, il_version_1.Document): + raise TypeError("layout IR cache document type is invalid") + pages = document.page + if not isinstance(pages, list) or len(pages) != self._expected_page_count: + raise ValueError("layout IR cache document page count is invalid") + page_numbers = [page.page_number for page in pages] + if page_numbers != self._expected_page_numbers: + raise ValueError("layout IR cache document page numbers are invalid") + if document.total_pages not in (None, self._expected_page_count): + raise ValueError("layout IR cache document total_pages is invalid") + return page_numbers + + def _input_is_unchanged(self) -> bool: + if self._input_path is None or self._input_identity is None: + return False + try: + info = self._input_path.stat() + except OSError: + return False + return _file_identity(info) == self._input_identity + + def _unlink_cache_path(self) -> None: + if self._cache_path is None: + return + try: + self._cache_path.unlink() + except OSError: + pass + + def _cache_entries(self) -> list[tuple[Path, os.stat_result]]: + entries: list[tuple[Path, os.stat_result]] = [] + try: + candidates = list(self.root.iterdir()) + except OSError: + return entries + for candidate in candidates: + if ( + candidate.suffix != ".pickle" + or len(candidate.stem) != 64 + or any( + character not in "0123456789abcdef" for character in candidate.stem + ) + ): + continue + try: + info = os.lstat(candidate) + except OSError: + continue + if ( + stat.S_ISREG(info.st_mode) + and not stat.S_ISLNK(info.st_mode) + and info.st_uid == os.getuid() + and info.st_mode & 0o077 == 0 + ): + entries.append((candidate, info)) + return entries + + def _prune( + self, + *, + maximum_bytes: int, + maximum_entries: int, + protected_path: Path | None = None, + ) -> None: + total_bytes = 0 + kept_entries = 0 + entries = sorted( + self._cache_entries(), + key=lambda entry: ( + entry[0] == protected_path, + entry[1].st_mtime_ns, + ), + reverse=True, + ) + for candidate, info in entries: + if ( + kept_entries < maximum_entries + and total_bytes + info.st_size <= maximum_bytes + ): + kept_entries += 1 + total_bytes += info.st_size + continue + try: + candidate.unlink() + except OSError: + pass + + def _cleanup_temporary_files(self) -> None: + try: + candidates = list(self.root.glob(".layout-ir-*.tmp")) + except OSError: + return + for candidate in candidates: + try: + info = os.lstat(candidate) + except OSError: + continue + if ( + not stat.S_ISREG(info.st_mode) + or stat.S_ISLNK(info.st_mode) + or info.st_uid != os.getuid() + or info.st_mode & 0o077 + ): + continue + try: + candidate.unlink() + except OSError: + pass + + +def _stable_file_fingerprint( + path: Path, +) -> tuple[str, tuple[int, int, int, int, int]]: + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode): + raise ValueError("layout IR cache input is not a regular file") + digest = hashlib.sha256() + with os.fdopen(descriptor, "rb") as handle: + descriptor = -1 + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + after = os.fstat(handle.fileno()) + finally: + if descriptor >= 0: + os.close(descriptor) + before_identity = _file_identity(before) + if before_identity != _file_identity(after): + raise ValueError("layout IR cache input changed while hashing") + return digest.hexdigest(), before_identity + + +def _file_identity(info: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + info.st_dev, + info.st_ino, + info.st_size, + info.st_mtime_ns, + info.st_ctime_ns, + ) + + +def _non_negative_int(value: object) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + return 0 + return value diff --git a/tests/test_assets.py b/tests/test_assets.py new file mode 100644 index 00000000..fd75d39e --- /dev/null +++ b/tests/test_assets.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import pytest +from babeldoc.assets import assets + + +def test_font_resource_lookup_is_cached_per_process( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[str] = [] + assets.get_font_and_metadata.cache_clear() + monkeypatch.setattr( + assets, + "get_font_and_metadata_async", + lambda font_name: calls.append(font_name) or ("font-path", {"name": font_name}), + ) + monkeypatch.setattr(assets, "run_coro", lambda result: result) + + try: + first = assets.get_font_and_metadata("fallback.ttf") + second = assets.get_font_and_metadata("fallback.ttf") + finally: + assets.get_font_and_metadata.cache_clear() + + assert first == second == ("font-path", {"name": "fallback.ttf"}) + assert calls == ["fallback.ttf"] diff --git a/tests/tools/executor/test_babeldoc_adapter.py b/tests/tools/executor/test_babeldoc_adapter.py index 6ae99270..124b4d9f 100644 --- a/tests/tools/executor/test_babeldoc_adapter.py +++ b/tests/tools/executor/test_babeldoc_adapter.py @@ -13,6 +13,7 @@ import pytest from babeldoc.format.pdf.translation_config import TranslateResult from babeldoc.tools.executor import babeldoc_adapter +from babeldoc.tools.executor.babeldoc_adapter import ExecutionTelemetry from babeldoc.tools.executor.babeldoc_adapter import build_translation_config from babeldoc.tools.executor.babeldoc_adapter import run_babeldoc_request from babeldoc.tools.executor.babeldoc_adapter import translate_result_to_payload @@ -125,7 +126,10 @@ def _complete_execution_request(api_key: str) -> dict[str, Any]: "requires_line_extraction": False, }, }, - "assets": {"glossaries": []}, + "assets": { + "glossaries": [], + "layout_ir_cache": {"enabled": True}, + }, "metadata": {"metadata_extra_data": None}, } @@ -177,6 +181,9 @@ def test_complete_execution_request_builds_translation_config( assert config.disable_rich_text_translate is True assert config.translator.api_key == api_key assert config.doc_layout_model.host == "http://127.0.0.1:2" + assert config.layout_ir_cache is not None + assert config.layout_ir_cache.root == workroot / ".cache" / "layout-ir" + assert config.layout_ir_cache.root.stat().st_mode & 0o777 == 0o700 finally: config.translator._client.close() config.term_extraction_translator._client.close() @@ -447,7 +454,7 @@ def test_adapter_emits_cancelled_terminal_for_async_cancellation( lambda _request, **_kwargs: _config(), ) - def cancel(_config, _emit): + def cancel(_config, _emit, _telemetry): raise asyncio.CancelledError monkeypatch.setattr(babeldoc_adapter, "_run_async_translate", cancel) @@ -498,7 +505,7 @@ def test_parent_pipe_eof_requests_cooperative_cancellation( lambda _request, **_kwargs: config, ) - def wait_for_cancel(_config, _emit): + def wait_for_cancel(_config, _emit, _telemetry): assert cancelled.wait(timeout=1) raise asyncio.CancelledError @@ -520,3 +527,78 @@ def wait_for_cancel(_config, _emit): "payload": {"reason": "client_request"}, } assert hard_exit_requested.wait(timeout=1) + + +def test_execution_telemetry_accumulates_repeated_phases() -> None: + class Clock: + now = 0 + + def __call__(self) -> int: + return self.now + + def advance_ms(self, milliseconds: int) -> None: + self.now += milliseconds * 1_000_000 + + clock = Clock() + telemetry = ExecutionTelemetry(clock_ns=clock) + config = _config() + config.layout_ir_cache = SimpleNamespace(status="hit") + + clock.advance_ms(5) + parsing = telemetry.observe( + { + "type": "progress_start", + "stage": "Parse PDF and Create Intermediate Representation", + }, + config, + ) + clock.advance_ms(7) + telemetry.observe( + { + "type": "progress_start", + "stage": "Translate Paragraphs", + }, + config, + ) + clock.advance_ms(11) + telemetry.transition("parsing") + clock.advance_ms(3) + telemetry.transition("finalizing") + clock.advance_ms(13) + completed = telemetry.finish(config) + + assert parsing["performance"]["phase"] == "parsing" + assert completed == { + "schema_version": 1, + "phase": "completed", + "elapsed_milliseconds": 39, + "phase_timings_milliseconds": { + "launching": 5, + "parsing": 10, + "translating": 11, + "typesetting": 0, + "saving": 0, + "finalizing": 13, + }, + "layout_ir_cache_status": "hit", + } + + +@pytest.mark.parametrize("invalid_value", [1, "true", [], None]) +def test_layout_ir_cache_enabled_must_be_boolean( + invalid_value: object, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + workroot = tmp_path / "workroot" + workroot.mkdir() + (workroot / "input.pdf").write_bytes(b"fixture") + monkeypatch.setenv(WORKROOT_ENV, str(workroot)) + request = _complete_execution_request(secrets.token_urlsafe(32)) + request["assets"]["layout_ir_cache"]["enabled"] = invalid_value + + with pytest.raises( + ValueError, + match="assets.layout_ir_cache.enabled must be a boolean", + ): + build_translation_config(request) diff --git a/tests/tools/executor/test_layout_ir_cache.py b/tests/tools/executor/test_layout_ir_cache.py new file mode 100644 index 00000000..8bc7e7f1 --- /dev/null +++ b/tests/tools/executor/test_layout_ir_cache.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import pickle # noqa: S403 - used to prove the restricted loader rejects it +from pathlib import Path +from types import SimpleNamespace + +from babeldoc.format.pdf.document_il import il_version_1 +from babeldoc.tools.executor.layout_ir_cache import LayoutIRCache + + +def _config(input_file: Path): + return SimpleNamespace( + input_file=input_file, + skip_scanned_detection=True, + debug=False, + pages=None, + page_ranges=None, + only_parse_generate_pdf=False, + table_model=None, + enable_graphic_element_process=True, + lang_in="en", + lang_out="zh-CN", + merge_alternating_line_numbers=True, + ocr_workaround=False, + primary_font_family=None, + remove_non_formula_lines=False, + skip_curve_render=False, + skip_form_render=False, + skip_formula_offset_calculation=False, + shared_context_cross_split_part=SimpleNamespace( + valid_char_count_total=120, + total_valid_text_token_count=30, + ), + ) + + +def _cache(tmp_path: Path) -> LayoutIRCache: + root = tmp_path / "cache" + root.mkdir(mode=0o700) + return LayoutIRCache(root, max_pages_per_part=50) + + +def _document() -> il_version_1.Document: + return il_version_1.Document( + page=[il_version_1.Page(page_number=0)], + total_pages=1, + ) + + +def test_layout_ir_cache_round_trip_restores_document_and_counts( + tmp_path: Path, +) -> None: + input_file = tmp_path / "input.pdf" + input_file.write_bytes(b"%PDF-cache-fixture") + config = _config(input_file) + cache = _cache(tmp_path) + mupdf_document = SimpleNamespace(page_count=1) + + assert cache.load(config, mupdf_document) is None + assert cache.status == "miss" + cache.store(_document(), config) + assert cache.status == "stored" + + config.shared_context_cross_split_part.valid_char_count_total = 0 + config.shared_context_cross_split_part.total_valid_text_token_count = 0 + reloaded = LayoutIRCache(cache.root, max_pages_per_part=50) + document = reloaded.load(config, mupdf_document) + + assert document == _document() + assert reloaded.status == "hit" + assert config.shared_context_cross_split_part.valid_char_count_total == 120 + assert config.shared_context_cross_split_part.total_valid_text_token_count == 30 + + +def test_layout_ir_cache_rejects_untrusted_pickle_globals(tmp_path: Path) -> None: + input_file = tmp_path / "input.pdf" + input_file.write_bytes(b"%PDF-cache-fixture") + config = _config(input_file) + cache = _cache(tmp_path) + mupdf_document = SimpleNamespace(page_count=1) + assert cache.load(config, mupdf_document) is None + assert cache._cache_path is not None + cache._cache_path.write_bytes(pickle.dumps(tmp_path / "not-an-il-document")) + cache._cache_path.chmod(0o600) + + assert cache.load(config, mupdf_document) is None + assert cache.status == "invalidated" + assert not cache._cache_path.exists() + + +def test_layout_ir_cache_detects_input_change_before_store(tmp_path: Path) -> None: + input_file = tmp_path / "input.pdf" + input_file.write_bytes(b"%PDF-before") + config = _config(input_file) + cache = _cache(tmp_path) + + assert cache.load(config, SimpleNamespace(page_count=1)) is None + input_file.write_bytes(b"%PDF-after") + cache.store(_document(), config) + + assert cache.status == "input_changed" + assert not list(cache.root.glob("*.pickle")) + + +def test_layout_ir_cache_requires_private_directory(tmp_path: Path) -> None: + input_file = tmp_path / "input.pdf" + input_file.write_bytes(b"%PDF-cache-fixture") + config = _config(input_file) + cache = _cache(tmp_path) + cache.root.chmod(0o755) + + assert cache.load(config, SimpleNamespace(page_count=1)) is None + assert cache.status == "unsafe_directory" + + +def test_layout_ir_cache_rejects_split_or_scanned_detection_runs( + tmp_path: Path, +) -> None: + input_file = tmp_path / "input.pdf" + input_file.write_bytes(b"%PDF-cache-fixture") + config = _config(input_file) + cache = _cache(tmp_path) + + assert cache.load(config, SimpleNamespace(page_count=51)) is None + assert cache.status == "ineligible" + + config.skip_scanned_detection = False + assert cache.load(config, SimpleNamespace(page_count=1)) is None + assert cache.status == "ineligible" From 821e8c00fa3b30a9ab86de57e81d7ad0a88161c9 Mon Sep 17 00:00:00 2001 From: SamsonCJ Date: Thu, 23 Jul 2026 01:22:14 -0700 Subject: [PATCH 3/8] feat: add persistent layout and resilient executor streams --- babeldoc/gloss_cli.py | 47 ++++ babeldoc/tools/executor/layout_server.py | 267 +++++++++++++++++++++ babeldoc/tools/executor/server.py | 43 +++- babeldoc/tools/executor/state.py | 27 ++- docs/gloss-service.md | 63 ++++- tests/test_gloss_cli.py | 54 +++++ tests/tools/executor/test_layout_server.py | 148 ++++++++++++ tests/tools/executor/test_server.py | 68 +++++- 8 files changed, 705 insertions(+), 12 deletions(-) create mode 100644 babeldoc/tools/executor/layout_server.py create mode 100644 tests/tools/executor/test_layout_server.py diff --git a/babeldoc/gloss_cli.py b/babeldoc/gloss_cli.py index b0ad21d0..81f4140f 100644 --- a/babeldoc/gloss_cli.py +++ b/babeldoc/gloss_cli.py @@ -6,6 +6,7 @@ import argparse import json +import multiprocessing from collections.abc import Sequence from importlib import metadata from typing import Any @@ -22,6 +23,10 @@ CAPABILITIES = ( "executor.events.ndjson.v1", "executor.http.v1", + "font-assets.memory-cache.v1", + "layout-ir-cache.v1", + "layout.rpc-doclayout8.v1", + "performance.telemetry.v1", "runtime-info.v1", ) @@ -68,6 +73,10 @@ def create_parser() -> argparse.ArgumentParser: action="store_true", help="Emit one machine-readable JSON object.", ) + commands.add_parser( + "package-smoke", + help=argparse.SUPPRESS, + ) serve = commands.add_parser( "serve", help="Run the authenticated loopback PDF executor service.", @@ -84,12 +93,35 @@ def create_parser() -> argparse.ArgumentParser: serve.add_argument("--instance-id") serve.add_argument("--parent-pid", type=int) serve.add_argument("--parent-start-time", type=float) + layout_serve = commands.add_parser( + "layout-serve", + help="Run the loopback DocLayout inference service.", + ) + layout_serve.add_argument("--host", default="127.0.0.1") + layout_serve.add_argument("--port", type=int, default=0) + layout_serve.add_argument("--parent-pid", type=int, required=True) + layout_serve.add_argument( + "--model", + choices=("onnx", "fake"), + default="onnx", + help=argparse.SUPPRESS, + ) return parser def cli(argv: Sequence[str] | None = None) -> int: """Run the lightweight Gloss integration CLI.""" args = create_parser().parse_args(argv) + if args.command == "layout-serve": + from babeldoc.tools.executor.layout_server import serve as serve_layout + + serve_layout( + args.host, + args.port, + parent_pid=args.parent_pid, + model_name=args.model, + ) + return 0 if args.command == "serve": from babeldoc.tools.executor.server import serve @@ -104,6 +136,10 @@ def cli(argv: Sequence[str] | None = None) -> int: parent_start_time=args.parent_start_time, ) return 0 + if args.command == "package-smoke": + _verify_packaged_dependencies() + print('{"ok":true,"schema_version":1}') + return 0 if args.command != "runtime-info": # pragma: no cover - argparse owns routing raise AssertionError(f"Unexpected command: {args.command}") @@ -125,5 +161,16 @@ def cli(argv: Sequence[str] | None = None) -> int: return 0 +def _verify_packaged_dependencies() -> None: + """Import the native modules required by real PDF and layout execution.""" + import bitstring + import onnx + import onnxruntime + import pymupdf + + del bitstring, onnx, onnxruntime, pymupdf + + if __name__ == "__main__": + multiprocessing.freeze_support() raise SystemExit(cli()) diff --git a/babeldoc/tools/executor/layout_server.py b/babeldoc/tools/executor/layout_server.py new file mode 100644 index 00000000..048c6790 --- /dev/null +++ b/babeldoc/tools/executor/layout_server.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +import base64 +import json +import logging +import os +import threading +import time +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler +from http.server import ThreadingHTTPServer +from typing import Any +from urllib.parse import urlparse + +import cv2 +import msgpack +import numpy as np + +logger = logging.getLogger(__name__) + +READY_PREFIX = "__GLOSS_BABELDOC_LAYOUT_READY__" +MAX_REQUEST_BYTES = 64 * 1024 * 1024 +PARENT_WATCHDOG_INTERVAL_SECONDS = 1.0 +ALLOW_FAKE_MODEL_ENV = "BABELDOC_LAYOUT_ALLOW_FAKE" + + +class LayoutServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, server_address, model): + super().__init__(server_address, LayoutHandler) + self.model = model + self.model_lock = threading.Lock() + + +class LayoutHandler(BaseHTTPRequestHandler): + server: LayoutServer + + def do_GET(self) -> None: + if urlparse(self.path).path != "/healthz": + self._write_json(HTTPStatus.NOT_FOUND, {"error": "not_found"}) + return + self._write_json( + HTTPStatus.OK, + { + "status": "ok", + "service": "gloss-babeldoc-layout", + "schema_version": 1, + }, + ) + + def do_POST(self) -> None: + if urlparse(self.path).path != "/inference": + self._write_json(HTTPStatus.NOT_FOUND, {"error": "not_found"}) + return + try: + raw, content_type = self._read_request() + request = ( + json.loads(raw) + if content_type == "application/json" + else msgpack.unpackb(raw, raw=False) + ) + if not isinstance(request, dict): + raise ValueError("request must be an object") + images = _decode_images(request, is_json=content_type == "application/json") + image_size = request.get("imgsz", 1024) + if ( + not isinstance(image_size, int) + or isinstance(image_size, bool) + or not 1 <= image_size <= 4096 + ): + raise ValueError("imgsz must be an integer between 1 and 4096") + with self.server.model_lock: + results = self.server.model.predict(images, imgsz=image_size) + if content_type == "application/json": + if len(results) != 1: + raise ValueError("rpc_doclayout8 requires one image") + payload = _rpc_v1_result_payload(results[0]) + self._write_json(HTTPStatus.OK, payload) + else: + self._write_bytes( + HTTPStatus.OK, + msgpack.packb( + [_legacy_result_payload(result) for result in results], + use_bin_type=True, + ), + "application/msgpack", + ) + except Exception: + logger.warning("layout inference request rejected", exc_info=True) + self._write_json(HTTPStatus.BAD_REQUEST, {"error": "invalid_request"}) + + def _read_request(self) -> tuple[bytes, str]: + if self.headers.get("Transfer-Encoding") is not None: + raise ValueError("chunked requests are not accepted") + content_type = self.headers.get("Content-Type", "").split(";", 1)[0] + if content_type not in {"application/json", "application/msgpack"}: + raise ValueError("unsupported content type") + try: + length = int(self.headers.get("Content-Length", "0")) + except ValueError as exc: + raise ValueError("invalid content length") from exc + if not 0 < length <= MAX_REQUEST_BYTES: + raise ValueError("invalid content length") + raw = self.rfile.read(length) + if len(raw) != length: + raise ValueError("request body ended early") + return raw, content_type + + def _write_json(self, status: HTTPStatus, payload: dict[str, Any]) -> None: + self._write_bytes( + status, + json.dumps(payload, separators=(",", ":"), sort_keys=True).encode(), + "application/json", + ) + + def _write_bytes( + self, + status: HTTPStatus, + body: bytes, + content_type: str, + ) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def log_message(self, _format: str, *_args: object) -> None: + return + + +def serve( + host: str = "127.0.0.1", + port: int = 0, + *, + parent_pid: int, + model_name: str = "onnx", +) -> None: + if host != "127.0.0.1": + raise ValueError("layout service must bind to 127.0.0.1") + if not 0 <= port <= 65_535: + raise ValueError("port must be between 0 and 65535") + if parent_pid <= 0 or os.getppid() != parent_pid: + raise ValueError("layout service parent identity does not match") + model = _load_model(model_name) + server = LayoutServer((host, port), model) + watchdog = threading.Thread( + target=_monitor_parent, + args=(server, parent_pid), + name="layout-parent-watchdog", + daemon=True, + ) + watchdog.start() + print(f"{READY_PREFIX}{server.server_address[1]}", flush=True) + try: + server.serve_forever() + finally: + server.server_close() + + +def _load_model(model_name: str): + if model_name == "fake": + if os.environ.get(ALLOW_FAKE_MODEL_ENV) != "1": + raise ValueError("fake layout model is disabled outside explicit tests") + return _FakeModel() + if model_name != "onnx": + raise ValueError(f"unknown layout model: {model_name}") + from babeldoc.docvision.doclayout import OnnxModel + + return OnnxModel.from_pretrained() + + +def _monitor_parent(server: LayoutServer, expected_parent_pid: int) -> None: + while True: + if os.getppid() != expected_parent_pid: + server.shutdown() + return + time.sleep(PARENT_WATCHDOG_INTERVAL_SECONDS) + + +def _decode_images(request: dict[str, Any], *, is_json: bool) -> list[np.ndarray]: + raw_images: object + if is_json: + encoded = request.get("image") + if not isinstance(encoded, str) or not encoded: + raise ValueError("image is required") + raw_images = [base64.b64decode(encoded, validate=True)] + else: + raw_images = request.get("image", []) + if not isinstance(raw_images, list) or not raw_images: + raise ValueError("image is required") + + images: list[np.ndarray] = [] + for encoded in raw_images: + if not isinstance(encoded, bytes | bytearray): + raise ValueError("encoded image must be bytes") + image = cv2.imdecode( + np.frombuffer(encoded, dtype=np.uint8), + cv2.IMREAD_COLOR, + ) + if image is None: + raise ValueError("invalid image") + images.append(image) + return images + + +def _legacy_result_payload(result) -> dict[str, Any]: + return { + "boxes": [ + { + "xyxy": [float(value) for value in box.xyxy], + "conf": float(box.conf), + "cls": int(box.cls), + } + for box in result.boxes + ], + "names": {str(key): str(value) for key, value in dict(result.names).items()}, + } + + +def _rpc_v1_result_payload(result) -> dict[str, Any]: + converted = _legacy_result_payload(result) + boxes = [] + for box in converted["boxes"]: + class_id = int(box["cls"]) + boxes.append( + { + "class_id": class_id, + "label": converted["names"].get(str(class_id), str(class_id)), + "score": float(box["conf"]), + "box": [float(value) for value in box["xyxy"]], + } + ) + return {"schema_version": 1, "boxes": boxes} + + +class _FakeModel: + def predict(self, images, *, imgsz: int): + del imgsz + return [ + _FakeResult( + boxes=[ + _FakeBox( + xyxy=[0.0, 0.0, float(image.shape[1]), float(image.shape[0])], + conf=1.0, + cls=0, + ) + ], + names={0: "text"}, + ) + for image in images + ] + + +class _FakeBox: + def __init__(self, *, xyxy, conf: float, cls: int): + self.xyxy = xyxy + self.conf = conf + self.cls = cls + + +class _FakeResult: + def __init__(self, *, boxes, names): + self.boxes = boxes + self.names = names diff --git a/babeldoc/tools/executor/server.py b/babeldoc/tools/executor/server.py index 25eca824..f7c005ba 100644 --- a/babeldoc/tools/executor/server.py +++ b/babeldoc/tools/executor/server.py @@ -50,6 +50,7 @@ SHUTDOWN_CANCEL_TIMEOUT_SECONDS = 5.0 SHUTDOWN_DRAIN_POLL_SECONDS = 0.25 PARENT_WATCHDOG_INTERVAL_SECONDS = 1.0 +EVENT_HEARTBEAT_INTERVAL_SECONDS = 5.0 _TOKEN_CHARACTERS = frozenset(string.ascii_letters + string.digits + "-_") ALLOW_FAKE_RUNNER_ENV = "BABELDOC_EXECUTOR_ALLOW_FAKE" @@ -221,7 +222,15 @@ def __init__( parent_start_time: float | None = None, runner_name: str = "injected", workroot: Path | None = None, + event_heartbeat_interval_seconds: float = (EVENT_HEARTBEAT_INTERVAL_SECONDS), ): + if not ( + 0 < event_heartbeat_interval_seconds <= EVENT_HEARTBEAT_INTERVAL_SECONDS + ): + raise ValueError( + "event_heartbeat_interval_seconds must be greater than zero " + "and no more than five seconds" + ) super().__init__(server_address, ExecutorHandler) self.store = store self.token = token @@ -230,6 +239,7 @@ def __init__( self.parent_start_time = parent_start_time self.runner_name = runner_name self.workroot = workroot + self.event_heartbeat_interval_seconds = event_heartbeat_interval_seconds self.started_at = time.time() self._stopping = threading.Event() self._lifecycle_lock = threading.RLock() @@ -895,7 +905,22 @@ def _stream_events(self, execution_id: str, after_seq: int): ) cursor = after_seq try: - for event in self.server.store.stream(execution_id, after_seq): + for event in self.server.store.stream( + execution_id, + after_seq, + heartbeat_interval_seconds=( + self.server.event_heartbeat_interval_seconds + ), + ): + if event is None: + self.wfile.write( + _heartbeat_json_line( + execution_id=execution_id, + instance_id=self.server.instance_id, + ) + ) + self.wfile.flush() + continue cursor = event.sequence self.wfile.write( _event_json_line(event, instance_id=self.server.instance_id) @@ -1033,6 +1058,22 @@ def _event_json_line(event, *, instance_id: str) -> bytes: ).encode("utf-8") +def _heartbeat_json_line(*, execution_id: str, instance_id: str) -> bytes: + payload = { + "schema_version": SCHEMA_VERSION, + "service_id": SERVICE_ID, + "instance_id": instance_id, + "type": "heartbeat", + "execution_id": execution_id, + "sequence": None, + "emitted_at": time.time(), + "payload": {}, + } + return ( + json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n" + ).encode("utf-8") + + def _stream_error_json_line( *, execution_id: str, diff --git a/babeldoc/tools/executor/state.py b/babeldoc/tools/executor/state.py index 22f5ef85..6c39b211 100644 --- a/babeldoc/tools/executor/state.py +++ b/babeldoc/tools/executor/state.py @@ -271,9 +271,14 @@ def stream( execution_id: str, after_seq: int, wait_seconds: float = 0.25, - ) -> Iterable[EventEnvelope]: + heartbeat_interval_seconds: float = 5.0, + ) -> Iterable[EventEnvelope | None]: + if heartbeat_interval_seconds <= 0: + raise ValueError("heartbeat_interval_seconds must be positive") cursor = after_seq + next_heartbeat_at = time.monotonic() + heartbeat_interval_seconds while True: + heartbeat_due = False with self._condition: record = self._require_record_locked(execution_id) self._raise_if_gap_locked(record, cursor) @@ -285,13 +290,29 @@ def stream( and status in ACTIVE_EXECUTION_STATUSES and not worker_finished ): - self._condition.wait(timeout=wait_seconds) - continue + now = time.monotonic() + if now >= next_heartbeat_at: + heartbeat_due = True + else: + self._condition.wait( + timeout=min( + wait_seconds, + next_heartbeat_at - now, + ) + ) + continue + + if heartbeat_due: + yield None + next_heartbeat_at = time.monotonic() + heartbeat_interval_seconds + continue for event in pending: cursor = event.sequence yield event + if pending: + next_heartbeat_at = time.monotonic() + heartbeat_interval_seconds if pending and pending[-1].type in TERMINAL_EVENT_TYPES: return if not pending and ( diff --git a/docs/gloss-service.md b/docs/gloss-service.md index b93f88c1..28edcd84 100644 --- a/docs/gloss-service.md +++ b/docs/gloss-service.md @@ -137,7 +137,10 @@ shape is: } }, "assets": { - "glossaries": [] + "glossaries": [], + "layout_ir_cache": { + "enabled": true + } }, "metadata": { "metadata_extra_data": null @@ -152,6 +155,12 @@ in snapshots or events. `qps`, `max_pages_per_part`, `pool_max_workers`, and integers). `report_interval_seconds` must be a positive number. `no_dual` and `no_mono` cannot both be true. +`assets.layout_ir_cache.enabled` is optional and defaults to `false` for +protocol compatibility. When enabled, the runtime chooses a private cache +directory and derives the cache key itself. It is eligible only for +non-debug, unsplit, full-document executions that skip scanned-document +detection. No client-controlled cache path or key is accepted. + ## State, events, and output validation Version 1 deliberately permits one active PDF execution per service instance. @@ -186,7 +195,21 @@ Normal event lines have this envelope: "stage_total": 20, "overall_progress": 42, "part_index": 1, - "total_parts": 1 + "total_parts": 1, + "performance": { + "schema_version": 1, + "phase": "translating", + "elapsed_milliseconds": 12401, + "phase_timings_milliseconds": { + "launching": 320, + "parsing": 3100, + "translating": 8981, + "typesetting": 0, + "saving": 0, + "finalizing": 0 + }, + "layout_ir_cache_status": "hit" + } } } ``` @@ -199,6 +222,12 @@ successful first-page loading with MuPDF. The requested mono and/or dual outputs must be present. Auto-extracted glossary output is likewise confined to the execution output directory. +The terminal result contains a final top-level `performance` object with +`phase` set to `completed`. Phase durations use a monotonic clock and +accumulate when split parts revisit a phase. Cache status is one of +`disabled`, `ineligible`, `miss`, `hit`, `stored`, `invalidated`, +`input_changed`, `unsafe_directory`, `read_error`, or `write_error`. + A cancelled payload is stable across cooperative and forced cancellation: ```json @@ -219,6 +248,12 @@ Gloss stores `instance_id`, `execution_id`, and the last consumed sequence. Disconnecting an event stream does not cancel its execution. For the same instance, reconnect with `after_sequence=`. +While an execution is active and no normal events arrive, the stream emits a +`heartbeat` control record at least once every five seconds. It carries the +schema, service, instance, and execution identity, with `sequence: null` and an +empty payload. Heartbeats are not retained or replayed and do not advance the +client's sequence cursor. + The service retains the 16 most recent executions and up to 1,000 events per execution. An already-trimmed cursor returns `410 replay_gap` with a snapshot; resume from `first_available_sequence - 1`. A cursor ahead of @@ -250,7 +285,23 @@ SIGINT, SIGTERM, and parent disappearance use the same controlled path. ## Compatibility boundary The protocol is advertised by `gloss-babeldoc runtime-info` as -`executor.http.v1` and `executor.events.ndjson.v1`. The ordinary `babeldoc` CLI -remains available while Gloss migrates to the service. Font caching, layout-IR -caching, and PDF-generation performance changes remain separate downstream -patches. +`executor.http.v1` and `executor.events.ndjson.v1`. Performance support is +advertised independently as `font-assets.memory-cache.v1`, +`layout-ir-cache.v1`, and `performance.telemetry.v1`. + +The same executable provides the persistent layout gateway used by +`rpc_doclayout8`: + +```text +gloss-babeldoc layout-serve \ + --host 127.0.0.1 \ + --port 0 \ + --parent-pid 1234 +``` + +It reports `__GLOSS_BABELDOC_LAYOUT_READY__` only after the ONNX model is +ready, exposes `GET /healthz` and `POST /inference`, and exits when its owning +Gloss process disappears. This is advertised as +`layout.rpc-doclayout8.v1`, allowing the self-contained runtime to operate +without a system Python installation. The ordinary `babeldoc` CLI remains +available for debugging and upstream compatibility. diff --git a/tests/test_gloss_cli.py b/tests/test_gloss_cli.py index 62849d92..bf903932 100644 --- a/tests/test_gloss_cli.py +++ b/tests/test_gloss_cli.py @@ -40,6 +40,10 @@ def test_build_runtime_info_contract() -> None: assert payload["capabilities"] == [ "executor.events.ndjson.v1", "executor.http.v1", + "font-assets.memory-cache.v1", + "layout-ir-cache.v1", + "layout.rpc-doclayout8.v1", + "performance.telemetry.v1", "runtime-info.v1", ] @@ -107,6 +111,24 @@ def test_unknown_command_exits_with_argparse_error() -> None: assert error.value.code == 2 +def test_package_smoke_runs_dependency_check( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + calls = [] + monkeypatch.setattr( + "babeldoc.gloss_cli._verify_packaged_dependencies", + lambda: calls.append("checked"), + ) + + assert cli(["package-smoke"]) == 0 + assert calls == ["checked"] + assert json.loads(capsys.readouterr().out) == { + "ok": True, + "schema_version": 1, + } + + def test_serve_forwards_service_options( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -168,3 +190,35 @@ def fake_serve(host: str, port: int, **kwargs: object) -> None: assert cli(["serve", "--runner", "fake"]) == 0 assert received["host"] == "127.0.0.1" assert received["port"] == 0 + + +def test_layout_serve_forwards_service_options( + monkeypatch: pytest.MonkeyPatch, +) -> None: + received: dict[str, object] = {} + + def fake_serve(host: str, port: int, **kwargs: object) -> None: + received.update({"host": host, "port": port, **kwargs}) + + monkeypatch.setattr("babeldoc.tools.executor.layout_server.serve", fake_serve) + + assert ( + cli( + [ + "layout-serve", + "--host", + "127.0.0.1", + "--port", + "49153", + "--parent-pid", + "42", + ] + ) + == 0 + ) + assert received == { + "host": "127.0.0.1", + "port": 49153, + "parent_pid": 42, + "model_name": "onnx", + } diff --git a/tests/tools/executor/test_layout_server.py b/tests/tools/executor/test_layout_server.py new file mode 100644 index 00000000..eafdb256 --- /dev/null +++ b/tests/tools/executor/test_layout_server.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import base64 +import http.client +import json +import threading +from types import SimpleNamespace + +import cv2 +import numpy as np +import pytest +from babeldoc.tools.executor import layout_server +from babeldoc.tools.executor.layout_server import LayoutServer + + +class _Model: + def predict(self, images, *, imgsz: int): + assert len(images) == 1 + assert imgsz == 1024 + return [ + SimpleNamespace( + boxes=[ + SimpleNamespace( + xyxy=[1.0, 2.0, 3.0, 4.0], + conf=0.75, + cls=2, + ) + ], + names={2: "title"}, + ) + ] + + +@pytest.fixture +def server(): + instance = LayoutServer(("127.0.0.1", 0), _Model()) + thread = threading.Thread(target=instance.serve_forever, daemon=True) + thread.start() + try: + yield instance + finally: + instance.shutdown() + instance.server_close() + thread.join(timeout=2) + + +def _request(server: LayoutServer, method: str, path: str, **kwargs): + connection = http.client.HTTPConnection( + server.server_address[0], + server.server_address[1], + timeout=2, + ) + connection.request(method, path, **kwargs) + response = connection.getresponse() + body = response.read() + connection.close() + return response.status, response.getheader("Content-Type"), body + + +def test_layout_health_and_rpc_doclayout8_inference(server: LayoutServer) -> None: + status, content_type, body = _request(server, "GET", "/healthz") + assert status == 200 + assert content_type == "application/json" + assert json.loads(body) == { + "schema_version": 1, + "service": "gloss-babeldoc-layout", + "status": "ok", + } + + ok, encoded = cv2.imencode( + ".jpg", + np.zeros((8, 12, 3), dtype=np.uint8), + ) + assert ok + request = json.dumps( + { + "schema_version": 1, + "image": base64.b64encode(encoded.tobytes()).decode(), + } + ).encode() + status, content_type, body = _request( + server, + "POST", + "/inference", + body=request, + headers={"Content-Type": "application/json"}, + ) + + assert status == 200 + assert content_type == "application/json" + assert json.loads(body) == { + "schema_version": 1, + "boxes": [ + { + "box": [1.0, 2.0, 3.0, 4.0], + "class_id": 2, + "label": "title", + "score": 0.75, + } + ], + } + + +def test_layout_server_rejects_invalid_or_oversized_requests( + server: LayoutServer, +) -> None: + status, _content_type, body = _request( + server, + "POST", + "/inference", + body=b"{}", + headers={"Content-Type": "application/json"}, + ) + assert status == 400 + assert json.loads(body) == {"error": "invalid_request"} + + status, _content_type, body = _request( + server, + "POST", + "/inference", + body=b"", + headers={ + "Content-Type": "application/json", + "Content-Length": str(layout_server.MAX_REQUEST_BYTES + 1), + }, + ) + assert status == 400 + assert json.loads(body) == {"error": "invalid_request"} + + +def test_layout_server_restricts_host_parent_and_fake_model( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with pytest.raises(ValueError, match="127.0.0.1"): + layout_server.serve( + "0.0.0.0", # noqa: S104 - the test verifies this is rejected + 0, + parent_pid=1, + ) + with pytest.raises(ValueError, match="parent identity"): + layout_server.serve( + "127.0.0.1", + 0, + parent_pid=2**30, + ) + monkeypatch.delenv(layout_server.ALLOW_FAKE_MODEL_ENV, raising=False) + with pytest.raises(ValueError, match="fake layout model is disabled"): + layout_server._load_model("fake") diff --git a/tests/tools/executor/test_server.py b/tests/tools/executor/test_server.py index b0be845f..bec10083 100644 --- a/tests/tools/executor/test_server.py +++ b/tests/tools/executor/test_server.py @@ -20,6 +20,7 @@ from babeldoc.tools.executor.protocol import EventEnvelope from babeldoc.tools.executor.runner import FakeExecutionRunner from babeldoc.tools.executor.server import ALLOW_FAKE_RUNNER_ENV +from babeldoc.tools.executor.server import EVENT_HEARTBEAT_INTERVAL_SECONDS from babeldoc.tools.executor.server import MAX_JSON_BODY_BYTES from babeldoc.tools.executor.server import READY_PREFIX from babeldoc.tools.executor.server import ExecutorHandler @@ -98,7 +99,7 @@ class GapDuringStreamStore: def replay(self, _execution_id: str, _after_sequence: int) -> list: return [] - def stream(self, execution_id: str, _after_sequence: int): + def stream(self, execution_id: str, _after_sequence: int, **_kwargs): yield EventEnvelope( type="progress", execution_id=execution_id, @@ -126,7 +127,7 @@ class EvictedDuringStreamStore: def replay(self, _execution_id: str, _after_sequence: int) -> list: return [] - def stream(self, execution_id: str, _after_sequence: int): + def stream(self, execution_id: str, _after_sequence: int, **_kwargs): yield EventEnvelope( type="progress", execution_id=execution_id, @@ -198,6 +199,7 @@ def running_server( workroot: Path | None = None, parent_pid: int | None = None, parent_start_time: float | None = None, + event_heartbeat_interval_seconds: float = (EVENT_HEARTBEAT_INTERVAL_SECONDS), ) -> Iterator[tuple[ExecutorServer, threading.Thread]]: previous_workroot = os.environ.get(WORKROOT_ENV) os.environ[WORKROOT_ENV] = str(tmp_path) @@ -212,6 +214,7 @@ def running_server( workroot=workroot, parent_pid=parent_pid, parent_start_time=parent_start_time, + event_heartbeat_interval_seconds=event_heartbeat_interval_seconds, ) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -445,6 +448,67 @@ def test_stream_gap_emits_an_unsequenced_control_record(tmp_path: Path) -> None: assert records[1]["payload"]["after_sequence"] == 11 +def test_quiet_stream_emits_repeated_unsequenced_heartbeats( + tmp_path: Path, +) -> None: + with running_server( + tmp_path, + event_heartbeat_interval_seconds=0.02, + ) as (server, _thread): + status, _headers, created = json_request( + server, + "POST", + "/v1/executions", + body={"task_id": "quiet-heartbeat", "mode": "block"}, + ) + assert status == 201 + execution_id = created["execution_id"] + connection = http.client.HTTPConnection( + *server.server_address[:2], + timeout=1, + ) + try: + connection.request( + "GET", + f"/v1/executions/{execution_id}/events" + f"?after_sequence={created['initial_sequence']}", + headers={"Authorization": f"Bearer {server.token}"}, + ) + response = connection.getresponse() + assert response.status == 200 + heartbeats = [ + json.loads(response.readline()), + json.loads(response.readline()), + ] + + assert [record["type"] for record in heartbeats] == [ + "heartbeat", + "heartbeat", + ] + assert all(record["schema_version"] == 1 for record in heartbeats) + assert all( + record["service_id"] == "gloss-babeldoc" for record in heartbeats + ) + assert all( + record["instance_id"] == "test-instance" for record in heartbeats + ) + assert all(record["execution_id"] == execution_id for record in heartbeats) + assert all(record["sequence"] is None for record in heartbeats) + assert all(record["payload"] == {} for record in heartbeats) + assert all(isinstance(record["emitted_at"], float) for record in heartbeats) + assert heartbeats[1]["emitted_at"] >= heartbeats[0]["emitted_at"] + assert ( + server.store.replay( + execution_id, + created["initial_sequence"], + ) + == [] + ) + finally: + connection.close() + server.store.cancel(execution_id) + + def test_stream_eviction_emits_control_record_with_nullable_snapshot( tmp_path: Path, ) -> None: From 646463ede77702fe04ae7b293cf764dfa0f13901 Mon Sep 17 00:00:00 2001 From: SamsonCJ Date: Thu, 23 Jul 2026 01:22:37 -0700 Subject: [PATCH 4/8] ci: release signed self-contained Gloss runtimes --- .github/workflows/release-runtime.yml | 221 +++++++++++++++++++++ DOWNSTREAM.md | 4 + DOWNSTREAM_PATCHES.md | 6 +- UPSTREAM_BASE.toml | 2 +- babeldoc/__init__.py | 2 +- babeldoc/const.py | 2 +- babeldoc/main.py | 2 +- docs/release-notes/v0.6.4-gloss.3.md | 38 ++++ pyproject.toml | 7 +- scripts/__init__.py | 1 + scripts/release/__init__.py | 1 + scripts/release/archive_runtime.py | 91 +++++++++ scripts/release/build_macos_runtime.sh | 74 +++++++ scripts/release/runtime_manifest.py | 173 +++++++++++++++++ scripts/release/sign_manifest.py | 76 ++++++++ scripts/release/smoke_runtime.py | 195 +++++++++++++++++++ tests/test_release_tools.py | 258 +++++++++++++++++++++++++ uv.lock | 94 ++++++++- 18 files changed, 1238 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/release-runtime.yml create mode 100644 docs/release-notes/v0.6.4-gloss.3.md create mode 100644 scripts/__init__.py create mode 100644 scripts/release/__init__.py create mode 100644 scripts/release/archive_runtime.py create mode 100644 scripts/release/build_macos_runtime.sh create mode 100644 scripts/release/runtime_manifest.py create mode 100644 scripts/release/sign_manifest.py create mode 100644 scripts/release/smoke_runtime.py create mode 100644 tests/test_release_tools.py diff --git a/.github/workflows/release-runtime.yml b/.github/workflows/release-runtime.yml new file mode 100644 index 00000000..361010fc --- /dev/null +++ b/.github/workflows/release-runtime.yml @@ -0,0 +1,221 @@ +name: Release self-contained Gloss runtime + +on: + push: + tags: + - "v*" + workflow_dispatch: + +permissions: + contents: read + +jobs: + macos-runtime: + name: macOS runtime (${{ matrix.architecture }}) + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - runner: macos-15 + architecture: arm64 + - runner: macos-15-intel + architecture: x86_64 + steps: + - name: Checkout release source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + + - name: Setup uv with Python 3.12 + uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2 + with: + python-version: "3.12" + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Install locked build dependencies + run: uv sync --frozen --no-default-groups --group release + + - name: Validate release tag + id: release + shell: bash + run: | + version=$(uv run --frozen --no-default-groups --group release python -c \ + 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])') + expected_tag="v${version/+/-}" + if [[ "$GITHUB_EVENT_NAME" = "push" ]]; then + actual_tag=$(git describe --tags --exact-match) + test "$actual_tag" = "$expected_tag" + test "$GITHUB_REF_TYPE" = "tag" + test "$GITHUB_REF_NAME" = "$expected_tag" + fi + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "source_date_epoch=$(git show -s --format=%ct HEAD)" >> "$GITHUB_OUTPUT" + + - name: Build self-contained runtime + shell: bash + run: | + mkdir -p release + test "$(uname -m)" = "${{ matrix.architecture }}" + bash scripts/release/build_macos_runtime.sh \ + "${{ steps.release.outputs.version }}" \ + "${{ steps.release.outputs.source_date_epoch }}" \ + "$PWD/release" + + - name: Smoke packaged services without system Python + shell: bash + run: | + tag_version=${{ steps.release.outputs.version }} + tag_version=${tag_version/+/-} + archive="release/gloss-babeldoc-${tag_version}-macos-${{ matrix.architecture }}.tar.gz" + test -s "$archive" + mkdir -p "$RUNNER_TEMP/runtime-smoke" + tar -xzf "$archive" -C "$RUNNER_TEMP/runtime-smoke" + uv run --frozen --no-default-groups --group release python \ + scripts/release/smoke_runtime.py \ + "$RUNNER_TEMP/runtime-smoke/gloss-babeldoc-runtime/gloss-babeldoc" + + - name: Upload native runtime + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: gloss-babeldoc-macos-${{ matrix.architecture }} + path: release/*.tar.gz + if-no-files-found: error + retention-days: 7 + + publish: + name: Sign, attest, and publish + needs: macos-runtime + if: github.event_name == 'push' + runs-on: ubuntu-latest + permissions: + attestations: write + contents: write + id-token: write + steps: + - name: Checkout release source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + + - name: Setup uv with Python 3.12 + uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2 + with: + python-version: "3.12" + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Validate release tag + id: release + shell: bash + run: | + version=$(uv run --frozen --no-default-groups python -c \ + 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])') + expected_tag="v${version/+/-}" + actual_tag=$(git describe --tags --exact-match) + test "$actual_tag" = "$expected_tag" + test "$GITHUB_REF_TYPE" = "tag" + test "$GITHUB_REF_NAME" = "$expected_tag" + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "tag=$expected_tag" >> "$GITHUB_OUTPUT" + echo "tag_version=${version/+/-}" >> "$GITHUB_OUTPUT" + echo "source_date_epoch=$(git show -s --format=%ct HEAD)" >> "$GITHUB_OUTPUT" + + - name: Install locked release dependencies + run: uv sync --frozen --no-default-groups + + - name: Download native runtimes + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: gloss-babeldoc-macos-* + path: release + merge-multiple: true + + - name: Verify native runtime set + shell: bash + run: | + test -s "release/gloss-babeldoc-${{ steps.release.outputs.tag_version }}-macos-arm64.tar.gz" + test -s "release/gloss-babeldoc-${{ steps.release.outputs.tag_version }}-macos-x86_64.tar.gz" + + - name: Build reproducible wheel and source distribution + shell: bash + env: + SOURCE_DATE_EPOCH: ${{ steps.release.outputs.source_date_epoch }} + run: | + mkdir -p "$RUNNER_TEMP/dist-first" "$RUNNER_TEMP/dist-second" + uv build --wheel --sdist --out-dir "$RUNNER_TEMP/dist-first" + uv build --wheel --sdist --out-dir "$RUNNER_TEMP/dist-second" + for first in "$RUNNER_TEMP/dist-first"/*; do + name=$(basename "$first") + cmp "$first" "$RUNNER_TEMP/dist-second/$name" + done + cp "$RUNNER_TEMP/dist-first"/* release/ + + - name: Build deterministic source archive + shell: bash + run: | + prefix="BabelDOC-${{ steps.release.outputs.tag_version }}/" + git archive --format=tar --prefix="$prefix" HEAD > "$RUNNER_TEMP/source.tar" + gzip -n -9 "$RUNNER_TEMP/source.tar" + mv "$RUNNER_TEMP/source.tar.gz" \ + "release/BabelDOC-${{ steps.release.outputs.tag_version }}-source.tar.gz" + cp LICENSE NOTICE release/ + + - name: Generate runtime manifest and provenance + shell: bash + run: | + uv run --frozen --no-default-groups python \ + -m scripts.release.runtime_manifest \ + --release-dir release \ + --repository "$GITHUB_REPOSITORY" \ + --commit "$GITHUB_SHA" \ + --source-date-epoch "${{ steps.release.outputs.source_date_epoch }}" + + - name: Sign and verify runtime manifest + shell: bash + env: + BABELDOC_RUNTIME_SIGNING_KEY_BASE64: ${{ secrets.BABELDOC_RUNTIME_SIGNING_KEY_BASE64 }} + run: | + test -n "$BABELDOC_RUNTIME_SIGNING_KEY_BASE64" + uv run --frozen --no-default-groups python \ + -m scripts.release.sign_manifest \ + --manifest release/gloss-runtime-manifest.json \ + --signature release/gloss-runtime-manifest.json.sig + uv run --frozen --no-default-groups python \ + -m scripts.release.sign_manifest \ + --manifest release/gloss-runtime-manifest.json \ + --signature release/gloss-runtime-manifest.json.sig \ + --verify + cp release/gloss-runtime-manifest.json.sig \ + release/gloss-runtime-manifest.sig + + - name: Generate and verify release checksums + shell: bash + run: | + ( + cd release + while IFS= read -r file; do + sha256sum "$file" + done < <(find . -maxdepth 1 -type f ! -name SHA256SUMS -printf '%f\n' | LC_ALL=C sort) \ + > SHA256SUMS + sha256sum --check SHA256SUMS + ) + + - name: Attest release artifacts + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 + with: + subject-path: release/* + + - name: Publish GitHub release + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + tag_name: ${{ steps.release.outputs.tag }} + name: BabelDOC ${{ steps.release.outputs.version }} + body_path: docs/release-notes/v${{ steps.release.outputs.tag_version }}.md + files: release/* + fail_on_unmatched_files: true + generate_release_notes: false + draft: false + prerelease: false + make_latest: true diff --git a/DOWNSTREAM.md b/DOWNSTREAM.md index 8616c81c..494f3726 100644 --- a/DOWNSTREAM.md +++ b/DOWNSTREAM.md @@ -38,6 +38,10 @@ Downstream builds are distributed through GitHub Releases or a Gloss-managed runtime manifest. They are not published to the public PyPI project owned by upstream. Every binary release must link to its exact source tag and include artifact checksums, dependency/build inputs, license notices, and provenance. +The runtime manifest is signed with the dedicated Ed25519 release key; Gloss +pins the corresponding public key and rejects missing or invalid signatures. +macOS releases include native arm64 and x86_64 self-contained archives, so an +end user does not need to install Python or BabelDOC separately. ## Upstream synchronization diff --git a/DOWNSTREAM_PATCHES.md b/DOWNSTREAM_PATCHES.md index c76923f6..a215f1d4 100644 --- a/DOWNSTREAM_PATCHES.md +++ b/DOWNSTREAM_PATCHES.md @@ -9,6 +9,8 @@ upstream, and the condition under which it can be removed. | `gloss-0002` | Active | Add a lightweight, versioned runtime capability handshake for Gloss. | Downstream integration boundary. | Upstream exposes an equivalent stable runtime protocol. | | `gloss-0003` | Active | Add reproducible dependency locking and Linux/macOS package validation. | Candidate upstream CI improvement. | Upstream adopts equivalent locked cross-platform release checks. | | `gloss-0004` | Active | Add an authenticated, reconnectable single-worker service boundary with targeted cancellation and process isolation. | Candidate upstream executor lifecycle and correctness fixes. | Upstream exposes an equivalent authenticated service contract adopted by Gloss. | +| `gloss-0005` | Active | Move font lookup caching, bounded layout-IR session reuse, and phase telemetry into the service runtime. | Candidate upstream PDF pipeline performance work. | Upstream exposes equivalent cache and telemetry contracts adopted by Gloss. | +| `gloss-0006` | Active | Publish signed, self-contained macOS runtimes plus reproducible Python/source artifacts and provenance. | Downstream release integration. | Gloss no longer installs BabelDOC as a managed runtime. | -Runtime, performance, and compatibility patches will be added in separate, -focused commits and PRs. +Compatibility patches will continue to be added in separate, focused commits +and PRs. diff --git a/UPSTREAM_BASE.toml b/UPSTREAM_BASE.toml index e32227e5..c9f70be5 100644 --- a/UPSTREAM_BASE.toml +++ b/UPSTREAM_BASE.toml @@ -1,7 +1,7 @@ # Gloss downstream provenance. Update downstream.version for every downstream # release; update the upstream block only in a reviewed upstream sync PR. [downstream] -version = "0.6.4+gloss.2" +version = "0.6.4+gloss.3" runtime_api_version = 1 [upstream] diff --git a/babeldoc/__init__.py b/babeldoc/__init__.py index 2ed073ca..b8b43ea2 100644 --- a/babeldoc/__init__.py +++ b/babeldoc/__init__.py @@ -1,2 +1,2 @@ # Gloss downstream release identity; see DOWNSTREAM.md. -__version__ = "0.6.4+gloss.2" +__version__ = "0.6.4+gloss.3" diff --git a/babeldoc/const.py b/babeldoc/const.py index e96013d6..e59ba0cd 100644 --- a/babeldoc/const.py +++ b/babeldoc/const.py @@ -7,7 +7,7 @@ from pathlib import Path # Gloss downstream release identity; see DOWNSTREAM.md. -__version__ = "0.6.4+gloss.2" +__version__ = "0.6.4+gloss.3" CACHE_FOLDER = Path.home() / ".cache" / "babeldoc" diff --git a/babeldoc/main.py b/babeldoc/main.py index a2fd4d93..3421035f 100644 --- a/babeldoc/main.py +++ b/babeldoc/main.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) # Gloss downstream release identity; see DOWNSTREAM.md. -__version__ = "0.6.4+gloss.2" +__version__ = "0.6.4+gloss.3" def create_parser(): diff --git a/docs/release-notes/v0.6.4-gloss.3.md b/docs/release-notes/v0.6.4-gloss.3.md new file mode 100644 index 00000000..5638b376 --- /dev/null +++ b/docs/release-notes/v0.6.4-gloss.3.md @@ -0,0 +1,38 @@ +# BabelDOC 0.6.4+gloss.3 + +This downstream release completes the managed runtime path used by the Gloss +macOS application. + +## Performance and observability + +- Repeated fallback-font resource lookups are cached for the lifetime of each + isolated PDF worker. +- Eligible single-part PDFs can reuse a private, bounded layout-IR session + cache. The cache key is derived inside BabelDOC from stable input bytes, + runtime version, page count, and parser-affecting options; clients cannot + choose cache paths or keys. +- Progress and terminal results expose monotonic phase timings for launch, + parse, translation, typesetting, save, and final validation, together with a + stable layout-cache status. + +## Distribution + +- Release automation builds reproducible wheel, sdist, and exact-source + archives. +- Native arm64 and x86_64 macOS archives contain a self-contained + `gloss-babeldoc` runtime and are smoke-tested after extraction. +- The runtime manifest lists verified artifact digests and the executable path + for each architecture, and requires Gloss 0.8.0 or newer. Archives contain + only regular files and directories, including a real mode-0755 executable, + LICENSE, NOTICE, a runtime README, and its machine-readable version record. + A detached Ed25519 signature is mandatory, and the workflow fails closed if + the configured key is absent or does not match the pinned release public key. +- Checksums, AGPL source, notices, machine-readable provenance, and GitHub + artifact attestations accompany the release. + +## Security + +- The XML control-character range is expressed unambiguously. +- Reviewed CodeQL annotations document the MD5 and SHA-2 transforms mandated + by ISO 32000 encrypted-PDF compatibility without disabling those queries for + other application code. diff --git a/pyproject.toml b/pyproject.toml index 499f8e31..db3c146d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ # Gloss downstream modification: release identity and runtime handshake, 2026-07-21. [project] name = "BabelDOC" -version = "0.6.4+gloss.2" +version = "0.6.4+gloss.3" description = "Gloss-maintained downstream of the BabelDOC document translator" license = "AGPL-3.0" readme = "README.md" @@ -163,12 +163,15 @@ dev = [ "pylance>=0.29.0", "py-spy>=0.4.0", ] +release = [ + "pyinstaller==6.21.0", +] [tool.pytest.ini_options] pythonpath = [".", "src"] testpaths = ["tests"] [bumpver] -current_version = "0.6.4+gloss.2" +current_version = "0.6.4+gloss.3" version_pattern = "MAJOR.MINOR.PATCH+gloss.NUM" [bumpver.file_patterns] diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..c4712a90 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Repository automation helpers.""" diff --git a/scripts/release/__init__.py b/scripts/release/__init__.py new file mode 100644 index 00000000..c4712a90 --- /dev/null +++ b/scripts/release/__init__.py @@ -0,0 +1 @@ +"""Repository automation helpers.""" diff --git a/scripts/release/archive_runtime.py b/scripts/release/archive_runtime.py new file mode 100644 index 00000000..45439b33 --- /dev/null +++ b/scripts/release/archive_runtime.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import argparse +import gzip +import os +import tarfile +from pathlib import Path + + +def create_archive(source: Path, output: Path, *, epoch: int) -> None: + source = source.resolve(strict=True) + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("wb") as raw_output: + with gzip.GzipFile( + filename="", + mode="wb", + fileobj=raw_output, + mtime=epoch, + ) as compressed: + with tarfile.open( + fileobj=compressed, + mode="w", + format=tarfile.PAX_FORMAT, + dereference=False, + ) as archive: + _add_path( + archive, + source, + source.name, + source_root=source, + epoch=epoch, + ) + + +def _add_path( + archive: tarfile.TarFile, + path: Path, + archive_name: str, + *, + source_root: Path, + epoch: int, +) -> None: + if path.is_symlink(): + resolved_path = path.resolve(strict=True) + try: + resolved_path.relative_to(source_root) + except ValueError as exc: + raise ValueError(f"runtime symlink escapes the bundle: {path}") from exc + if not resolved_path.is_file(): + raise ValueError(f"runtime symlink must resolve to a regular file: {path}") + path = resolved_path + info = archive.gettarinfo(os.fspath(path), arcname=archive_name) + if info.issym() or info.islnk(): + raise ValueError(f"runtime archive cannot contain links: {archive_name}") + info.uid = 0 + info.gid = 0 + info.uname = "" + info.gname = "" + info.mtime = epoch + if info.isfile(): + with path.open("rb") as handle: + archive.addfile(info, handle) + else: + archive.addfile(info) + if info.isdir(): + for child in sorted(path.iterdir(), key=lambda item: item.name): + _add_path( + archive, + child, + f"{archive_name}/{child.name}", + source_root=source_root, + epoch=epoch, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--source-date-epoch", type=int, required=True) + args = parser.parse_args() + create_archive( + args.source, + args.output, + epoch=args.source_date_epoch, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release/build_macos_runtime.sh b/scripts/release/build_macos_runtime.sh new file mode 100644 index 00000000..dcee5859 --- /dev/null +++ b/scripts/release/build_macos_runtime.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "usage: build_macos_runtime.sh VERSION SOURCE_DATE_EPOCH OUTPUT_DIR" >&2 + exit 2 +fi + +version=$1 +source_date_epoch=$2 +output_dir=$3 +machine_arch=$(uname -m) + +case "$machine_arch" in + arm64) + release_arch=arm64 + ;; + x86_64) + release_arch=x86_64 + ;; + *) + echo "unsupported macOS architecture: $machine_arch" >&2 + exit 1 + ;; +esac + +tag_version=${version/+/-} +build_root=${RUNNER_TEMP:-"$PWD/.release-build"}/gloss-babeldoc-"$release_arch" +pyinstaller_dist=$build_root/dist +pyinstaller_work=$build_root/work +bundle_root=$build_root/gloss-babeldoc-runtime + +mkdir -p "$output_dir" +mkdir -p "$build_root" + +uv sync --frozen --no-default-groups --group release +uv run --no-sync pyinstaller \ + --clean \ + --noconfirm \ + --onedir \ + --console \ + --name gloss-babeldoc \ + --distpath "$pyinstaller_dist" \ + --workpath "$pyinstaller_work" \ + --specpath "$build_root" \ + --copy-metadata BabelDOC \ + --collect-all babeldoc \ + --collect-all bitstring \ + --collect-all pymupdf \ + --hidden-import babeldoc.tools.executor.babeldoc_adapter \ + --hidden-import babeldoc.tools.executor.layout_server \ + babeldoc/gloss_cli.py + +mv "$pyinstaller_dist/gloss-babeldoc" "$bundle_root" +cp LICENSE NOTICE "$bundle_root/" +printf '%s\n' \ + "Gloss BabelDOC managed runtime" \ + "Version: $version" \ + "Exact source: https://github.com/SunChJ/BabelDOC/tree/v$tag_version" \ + "Release provenance: https://github.com/SunChJ/BabelDOC/releases/tag/v$tag_version" \ + "Run ./gloss-babeldoc runtime-info --json for the runtime and upstream identity." \ + "License and attribution are included in LICENSE and NOTICE." \ + > "$bundle_root/README.txt" +"$bundle_root/gloss-babeldoc" runtime-info --json \ + > "$bundle_root/RUNTIME_INFO.json" + +archive_path=$output_dir/gloss-babeldoc-"$tag_version"-macos-"$release_arch".tar.gz +uv run --no-sync python \ + scripts/release/archive_runtime.py \ + --source "$bundle_root" \ + --output "$archive_path" \ + --source-date-epoch "$source_date_epoch" + +echo "$archive_path" diff --git a/scripts/release/runtime_manifest.py b/scripts/release/runtime_manifest.py new file mode 100644 index 00000000..3abfb296 --- /dev/null +++ b/scripts/release/runtime_manifest.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +from datetime import UTC +from datetime import datetime +from pathlib import Path +from urllib.parse import quote + +import tomllib +from babeldoc.gloss_cli import CAPABILITIES +from babeldoc.gloss_cli import UPSTREAM_COMMIT +from babeldoc.gloss_cli import UPSTREAM_REPOSITORY +from babeldoc.gloss_cli import UPSTREAM_VERSION + +PINNED_PUBLIC_KEY_BASE64 = "0lgbX+CkmBjf4BnH9JO66I7Krd1DYM8lTOjIt+7zWEE=" +RUNTIME_ARCHITECTURES = ("arm64", "x86_64") + + +def build_release_metadata( + release_dir: Path, + *, + repository: str, + commit: str, + source_date_epoch: int, +) -> tuple[dict, dict]: + project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) + version = project["project"]["version"] + release_tag = f"v{version.replace('+', '-')}" + release_base_url = ( + f"https://github.com/{repository}/releases/download/{release_tag}" + ) + published_at = ( + datetime.fromtimestamp(source_date_epoch, tz=UTC) + .isoformat(timespec="seconds") + .replace("+00:00", "Z") + ) + + runtime_assets = [] + for architecture in RUNTIME_ARCHITECTURES: + filename = ( + f"gloss-babeldoc-{version.replace('+', '-')}-macos-{architecture}.tar.gz" + ) + path = _required_file(release_dir / filename) + runtime_assets.append( + { + "operatingSystem": "macos", + "architecture": architecture, + "url": f"{release_base_url}/{quote(filename)}", + "sha256": _sha256(path), + "size": path.stat().st_size, + "archiveFormat": "tar.gz", + "executablePath": "gloss-babeldoc-runtime/gloss-babeldoc", + } + ) + + notes_name = f"v{version.replace('+', '-')}.md" + signing_key = base64.b64decode(PINNED_PUBLIC_KEY_BASE64, validate=True) + manifest = { + "schemaVersion": 1, + "channel": "stable", + "version": version, + "releaseTag": release_tag, + "publishedAt": published_at, + "minimumGlossVersion": "0.8.0", + "releaseNotesURL": ( + f"https://github.com/{repository}/blob/{release_tag}/" + f"docs/release-notes/{notes_name}" + ), + "assets": runtime_assets, + "capabilities": sorted(CAPABILITIES), + "upstream": { + "repository": UPSTREAM_REPOSITORY, + "version": UPSTREAM_VERSION, + "commit": UPSTREAM_COMMIT, + }, + "signature": { + "algorithm": "Ed25519", + "encoding": "raw", + "asset": "gloss-runtime-manifest.json.sig", + "publicKeySha256": hashlib.sha256(signing_key).hexdigest(), + }, + "source": { + "repository": f"https://github.com/{repository}", + "commit": commit, + "tag": release_tag, + }, + } + + subjects = [] + for path in sorted(release_dir.iterdir(), key=lambda item: item.name): + if path.is_file() and path.name not in { + "gloss-runtime-manifest.json", + "gloss-runtime-manifest.json.sig", + "gloss-runtime-manifest.sig", + "SHA256SUMS", + "PROVENANCE.json", + }: + subjects.append( + { + "name": path.name, + "sha256": _sha256(path), + "size": path.stat().st_size, + } + ) + provenance = { + "schemaVersion": 1, + "builder": "github-actions", + "source": { + "repository": f"https://github.com/{repository}", + "commit": commit, + "tag": release_tag, + "sourceDateEpoch": source_date_epoch, + }, + "upstream": manifest["upstream"], + "dependencyLock": { + "name": "uv.lock", + "sha256": _sha256(Path("uv.lock")), + }, + "subjects": subjects, + } + return manifest, provenance + + +def write_json(path: Path, value: dict) -> None: + path.write_text( + json.dumps( + value, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + + +def _required_file(path: Path) -> Path: + if not path.is_file() or path.stat().st_size <= 0: + raise FileNotFoundError(f"required release artifact is missing: {path}") + return path + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--release-dir", type=Path, required=True) + parser.add_argument("--repository", required=True) + parser.add_argument("--commit", required=True) + parser.add_argument("--source-date-epoch", type=int, required=True) + args = parser.parse_args() + manifest, provenance = build_release_metadata( + args.release_dir, + repository=args.repository, + commit=args.commit, + source_date_epoch=args.source_date_epoch, + ) + write_json(args.release_dir / "gloss-runtime-manifest.json", manifest) + write_json(args.release_dir / "PROVENANCE.json", provenance) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release/sign_manifest.py b/scripts/release/sign_manifest.py new file mode 100644 index 00000000..3c88b494 --- /dev/null +++ b/scripts/release/sign_manifest.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import argparse +import base64 +import os +from pathlib import Path + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + +from scripts.release.runtime_manifest import PINNED_PUBLIC_KEY_BASE64 + +SIGNING_KEY_ENV = "BABELDOC_RUNTIME_SIGNING_KEY_BASE64" + + +def sign_manifest( + manifest: bytes, + *, + encoded_private_key: str, + expected_public_key_base64: str = PINNED_PUBLIC_KEY_BASE64, +) -> bytes: + if not encoded_private_key: + raise ValueError(f"{SIGNING_KEY_ENV} is required") + try: + pem = base64.b64decode(encoded_private_key, validate=True) + private_key = serialization.load_pem_private_key(pem, password=None) + except Exception as exc: + raise ValueError("runtime signing key is not valid base64 PKCS8 PEM") from exc + if not isinstance(private_key, Ed25519PrivateKey): + raise ValueError("runtime signing key must be Ed25519") + expected_public_key = base64.b64decode( + expected_public_key_base64, + validate=True, + ) + actual_public_key = private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + if actual_public_key != expected_public_key: + raise ValueError("runtime signing key does not match the pinned public key") + return private_key.sign(manifest) + + +def verify_manifest( + manifest: bytes, + signature: bytes, + *, + public_key_base64: str = PINNED_PUBLIC_KEY_BASE64, +) -> None: + public_key = Ed25519PublicKey.from_public_bytes( + base64.b64decode(public_key_base64, validate=True) + ) + public_key.verify(signature, manifest) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--signature", type=Path, required=True) + parser.add_argument("--verify", action="store_true") + args = parser.parse_args() + manifest = args.manifest.read_bytes() + if args.verify: + verify_manifest(manifest, args.signature.read_bytes()) + return 0 + signature = sign_manifest( + manifest, + encoded_private_key=os.environ.get(SIGNING_KEY_ENV, ""), + ) + args.signature.write_bytes(signature) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release/smoke_runtime.py b/scripts/release/smoke_runtime.py new file mode 100644 index 00000000..4fc1b4c2 --- /dev/null +++ b/scripts/release/smoke_runtime.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import argparse +import http.client +import json +import os +import secrets +import select +import subprocess +import tempfile +from pathlib import Path +from urllib.parse import urlparse + +from babeldoc import __version__ +from babeldoc.tools.executor.layout_server import READY_PREFIX as LAYOUT_READY_PREFIX +from babeldoc.tools.executor.server import READY_PREFIX as EXECUTOR_READY_PREFIX + + +def smoke(executable: Path) -> None: + runtime_info = subprocess.run( # noqa: S603 + [executable, "runtime-info", "--json"], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + payload = json.loads(runtime_info.stdout) + if payload["runtime"]["version"] != __version__: + raise RuntimeError("packaged runtime version does not match") + bundle_root = executable.parent + for name in ("LICENSE", "NOTICE", "README.txt", "RUNTIME_INFO.json"): + packaged_file = bundle_root / name + if not packaged_file.is_file() or packaged_file.stat().st_size == 0: + raise RuntimeError(f"packaged runtime is missing {name}") + if json.loads((bundle_root / "RUNTIME_INFO.json").read_bytes()) != payload: + raise RuntimeError("packaged runtime identity record does not match") + readme = (bundle_root / "README.txt").read_text(encoding="utf-8") + if payload["runtime"]["version"] not in readme or "Exact source:" not in readme: + raise RuntimeError("packaged runtime README lacks provenance") + dependency_check = subprocess.run( # noqa: S603 + [executable, "package-smoke"], + check=True, + capture_output=True, + text=True, + timeout=180, + ) + if json.loads(dependency_check.stdout) != { + "ok": True, + "schema_version": 1, + }: + raise RuntimeError("packaged native dependency check failed") + _smoke_layout(executable) + _smoke_executor(executable, runner_name="fake") + _smoke_executor(executable, runner_name="babeldoc") + + +def _smoke_layout(executable: Path) -> None: + environment = os.environ.copy() + environment["BABELDOC_LAYOUT_ALLOW_FAKE"] = "1" + process = subprocess.Popen( # noqa: S603 + [ + executable, + "layout-serve", + "--host", + "127.0.0.1", + "--port", + "0", + "--parent-pid", + str(os.getpid()), + "--model", + "fake", + ], + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + port = int(_ready_line(process, LAYOUT_READY_PREFIX)) + status, body = _json_request(f"http://127.0.0.1:{port}", "GET", "/healthz") + if status != 200 or body.get("status") != "ok": + raise RuntimeError("packaged layout service health check failed") + finally: + process.terminate() + process.wait(timeout=10) + + +def _smoke_executor(executable: Path, *, runner_name: str) -> None: + with tempfile.TemporaryDirectory(prefix="gloss-runtime-smoke-") as temporary: + workroot = Path(temporary) + workroot.chmod(0o700) + marker = workroot / ".executor-workroot-ready" + marker.write_text("ready\n", encoding="utf-8") + marker.chmod(0o600) + token = secrets.token_urlsafe(32) + token_file = workroot / "token" + token_file.write_text(token + "\n", encoding="utf-8") + token_file.chmod(0o600) + environment = os.environ.copy() + if runner_name == "fake": + environment["BABELDOC_EXECUTOR_ALLOW_FAKE"] = "1" + else: + environment.pop("BABELDOC_EXECUTOR_ALLOW_FAKE", None) + process = subprocess.Popen( # noqa: S603 + [ + executable, + "serve", + "--host", + "127.0.0.1", + "--port", + "0", + "--runner", + runner_name, + "--token-file", + token_file, + "--work-dir", + workroot, + "--parent-pid", + str(os.getpid()), + ], + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + ready = json.loads(_ready_line(process, EXECUTOR_READY_PREFIX)) + headers = {"Authorization": f"Bearer {token}"} + status, body = _json_request( + ready["endpoint"], + "GET", + "/healthz", + headers=headers, + ) + if status != 200 or body.get("ok") is not True: + raise RuntimeError("packaged executor health check failed") + status, _body = _json_request( + ready["endpoint"], + "POST", + "/v1/shutdown", + headers={**headers, "Content-Type": "application/json"}, + body=b"{}", + ) + if status != 202: + raise RuntimeError("packaged executor shutdown failed") + if process.wait(timeout=10) != 0: + raise RuntimeError("packaged executor exited unsuccessfully") + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + + +def _ready_line(process: subprocess.Popen, prefix: str) -> str: + if process.stdout is None: + raise RuntimeError("service stdout is unavailable") + readable, _, _ = select.select([process.stdout], [], [], 120) + if not readable: + raise TimeoutError("service did not report ready") + line = process.stdout.readline().strip() + if not line.startswith(prefix): + error = process.stderr.read() if process.stderr is not None else "" + raise RuntimeError( + f"invalid service ready line: {line}; stderr={error[-2000:]}" + ) + return line.removeprefix(prefix) + + +def _json_request( + endpoint: str, + method: str, + path: str, + *, + headers: dict[str, str] | None = None, + body: bytes | None = None, +) -> tuple[int, dict]: + parsed = urlparse(endpoint) + connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=5) + connection.request(method, path, headers=headers or {}, body=body) + response = connection.getresponse() + payload = json.loads(response.read()) + connection.close() + return response.status, payload + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("executable", type=Path) + args = parser.parse_args() + smoke(args.executable.resolve(strict=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_release_tools.py b/tests/test_release_tools.py new file mode 100644 index 00000000..731ab4c0 --- /dev/null +++ b/tests/test_release_tools.py @@ -0,0 +1,258 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +import tarfile +from pathlib import Path + +import pytest +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from scripts.release import smoke_runtime +from scripts.release.archive_runtime import create_archive +from scripts.release.runtime_manifest import build_release_metadata +from scripts.release.runtime_manifest import write_json +from scripts.release.sign_manifest import sign_manifest +from scripts.release.sign_manifest import verify_manifest + +VERSION = "0.6.4+gloss.3" +TAG_VERSION = "0.6.4-gloss.3" +SOURCE_DATE_EPOCH = 1_750_000_000 + + +def test_packaged_runtime_smoke_starts_fake_and_production_runners( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + bundle = tmp_path / "gloss-babeldoc-runtime" + bundle.mkdir() + executable = bundle / "gloss-babeldoc" + executable.write_bytes(b"packaged executable") + payload = {"runtime": {"version": VERSION}} + for name in ("LICENSE", "NOTICE"): + (bundle / name).write_text(f"{name}\n", encoding="utf-8") + (bundle / "README.txt").write_text( + f"Version: {VERSION}\nExact source: test\n", + encoding="utf-8", + ) + (bundle / "RUNTIME_INFO.json").write_text( + json.dumps(payload), + encoding="utf-8", + ) + + def fake_run(arguments, **_kwargs): + if arguments[1] == "runtime-info": + stdout = json.dumps(payload) + elif arguments[1] == "package-smoke": + stdout = '{"ok":true,"schema_version":1}' + else: # pragma: no cover - guards the smoke command contract + raise AssertionError(arguments) + return type("Completed", (), {"stdout": stdout})() + + layout_calls: list[Path] = [] + runner_calls: list[str] = [] + monkeypatch.setattr(smoke_runtime.subprocess, "run", fake_run) + monkeypatch.setattr( + smoke_runtime, + "_smoke_layout", + lambda path: layout_calls.append(path), + ) + monkeypatch.setattr( + smoke_runtime, + "_smoke_executor", + lambda _path, *, runner_name: runner_calls.append(runner_name), + ) + + smoke_runtime.smoke(executable) + + assert layout_calls == [executable] + assert runner_calls == ["fake", "babeldoc"] + + +def test_runtime_archive_is_reproducible_and_preserves_executable( + tmp_path: Path, +) -> None: + bundle = tmp_path / "gloss-babeldoc-runtime" + bundle.mkdir() + executable = bundle / "gloss-babeldoc" + executable.write_bytes(b"#!/bin/sh\nexit 0\n") + executable.chmod(0o755) + (bundle / "payload.txt").write_text("runtime payload\n", encoding="utf-8") + (bundle / "runtime-link").symlink_to("payload.txt") + for name in ("LICENSE", "NOTICE", "README.txt", "RUNTIME_INFO.json"): + (bundle / name).write_text(f"{name}\n", encoding="utf-8") + + first = tmp_path / "first.tar.gz" + second = tmp_path / "second.tar.gz" + create_archive(bundle, first, epoch=SOURCE_DATE_EPOCH) + create_archive(bundle, second, epoch=SOURCE_DATE_EPOCH) + + assert first.read_bytes() == second.read_bytes() + with tarfile.open(first, "r:gz") as archive: + members = archive.getmembers() + assert [member.name for member in members] == [ + "gloss-babeldoc-runtime", + "gloss-babeldoc-runtime/LICENSE", + "gloss-babeldoc-runtime/NOTICE", + "gloss-babeldoc-runtime/README.txt", + "gloss-babeldoc-runtime/RUNTIME_INFO.json", + "gloss-babeldoc-runtime/gloss-babeldoc", + "gloss-babeldoc-runtime/payload.txt", + "gloss-babeldoc-runtime/runtime-link", + ] + packaged_executable = archive.getmember("gloss-babeldoc-runtime/gloss-babeldoc") + assert packaged_executable.mode & 0o111 + assert archive.getmember("gloss-babeldoc-runtime/runtime-link").isfile() + assert not any(member.issym() or member.islnk() for member in members) + assert all(member.mtime == SOURCE_DATE_EPOCH for member in members) + assert all(member.uid == member.gid == 0 for member in members) + + +def test_runtime_archive_rejects_symlinks_outside_bundle(tmp_path: Path) -> None: + bundle = tmp_path / "gloss-babeldoc-runtime" + bundle.mkdir() + outside = tmp_path / "outside" + outside.write_text("not part of the runtime\n", encoding="utf-8") + (bundle / "escape").symlink_to(outside) + + with pytest.raises(ValueError, match="escapes the bundle"): + create_archive( + bundle, + tmp_path / "runtime.tar.gz", + epoch=SOURCE_DATE_EPOCH, + ) + + +def test_release_manifest_has_strict_gloss_asset_contract(tmp_path: Path) -> None: + for architecture in ("arm64", "x86_64"): + filename = f"gloss-babeldoc-{TAG_VERSION}-macos-{architecture}.tar.gz" + (tmp_path / filename).write_bytes(f"archive-{architecture}".encode()) + (tmp_path / f"BabelDOC-{VERSION}-py3-none-any.whl").write_bytes(b"wheel") + (tmp_path / f"babeldoc-{VERSION}.tar.gz").write_bytes(b"sdist") + + manifest, provenance = build_release_metadata( + tmp_path, + repository="SunChJ/BabelDOC", + commit="a" * 40, + source_date_epoch=SOURCE_DATE_EPOCH, + ) + + assert { + "schemaVersion", + "channel", + "version", + "releaseTag", + "publishedAt", + "minimumGlossVersion", + "releaseNotesURL", + "assets", + } <= manifest.keys() + assert manifest["version"] == VERSION + assert manifest["releaseTag"] == f"v{TAG_VERSION}" + assert manifest["publishedAt"] == "2025-06-15T15:06:40Z" + assert manifest["minimumGlossVersion"] == "0.8.0" + assert manifest["signature"]["asset"] == "gloss-runtime-manifest.json.sig" + assert len(manifest["assets"]) == 2 + assert {asset["architecture"] for asset in manifest["assets"]} == { + "arm64", + "x86_64", + } + for asset in manifest["assets"]: + path = tmp_path / Path(asset["url"]).name + assert asset == { + "operatingSystem": "macos", + "architecture": asset["architecture"], + "url": ( + "https://github.com/SunChJ/BabelDOC/releases/download/" + f"v{TAG_VERSION}/{path.name}" + ), + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "size": path.stat().st_size, + "archiveFormat": "tar.gz", + "executablePath": "gloss-babeldoc-runtime/gloss-babeldoc", + } + assert provenance["source"]["commit"] == "a" * 40 + assert {subject["name"] for subject in provenance["subjects"]} == { + f"BabelDOC-{VERSION}-py3-none-any.whl", + f"babeldoc-{VERSION}.tar.gz", + f"gloss-babeldoc-{TAG_VERSION}-macos-arm64.tar.gz", + f"gloss-babeldoc-{TAG_VERSION}-macos-x86_64.tar.gz", + } + + output = tmp_path / "gloss-runtime-manifest.json" + write_json(output, manifest) + assert output.read_bytes().endswith(b"\n") + assert json.loads(output.read_bytes()) == manifest + + +def test_release_manifest_requires_both_macos_architectures( + tmp_path: Path, +) -> None: + (tmp_path / f"gloss-babeldoc-{TAG_VERSION}-macos-arm64.tar.gz").write_bytes( + b"archive" + ) + with pytest.raises(FileNotFoundError, match="x86_64"): + build_release_metadata( + tmp_path, + repository="SunChJ/BabelDOC", + commit="a" * 40, + source_date_epoch=SOURCE_DATE_EPOCH, + ) + + +def test_manifest_signature_is_raw_ed25519_and_fails_closed() -> None: + private_key = Ed25519PrivateKey.generate() + private_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + encoded_private_key = base64.b64encode(private_pem).decode() + public_key_base64 = base64.b64encode( + private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + ).decode() + manifest = b'{"schemaVersion":1}\n' + + signature = sign_manifest( + manifest, + encoded_private_key=encoded_private_key, + expected_public_key_base64=public_key_base64, + ) + + assert len(signature) == 64 + verify_manifest( + manifest, + signature, + public_key_base64=public_key_base64, + ) + with pytest.raises(InvalidSignature): + verify_manifest( + manifest + b" ", + signature, + public_key_base64=public_key_base64, + ) + with pytest.raises(ValueError, match="required"): + sign_manifest( + manifest, + encoded_private_key="", + expected_public_key_base64=public_key_base64, + ) + other_public_key = base64.b64encode( + Ed25519PrivateKey.generate() + .public_key() + .public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + ).decode() + with pytest.raises(ValueError, match="pinned public key"): + sign_manifest( + manifest, + encoded_private_key=encoded_private_key, + expected_public_key_base64=other_public_key, + ) diff --git a/uv.lock b/uv.lock index 0bc01167..d9d54621 100644 --- a/uv.lock +++ b/uv.lock @@ -8,6 +8,15 @@ resolution-markers = [ "python_full_version < '3.11'", ] +[[package]] +name = "altgraph" +version = "0.17.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/f8/97fdf103f38fed6792a1601dbc16cc8aac56e7459a9fff08c812d8ae177a/altgraph-0.17.5.tar.gz", hash = "sha256:c87b395dd12fabde9c99573a9749d67da8d29ef9de0125c7f536699b4a9bc9e7", size = 48428, upload-time = "2025-11-21T20:35:50.583Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/ba/000a1996d4308bc65120167c21241a3b205464a2e0b58deda26ae8ac21d1/altgraph-0.17.5-py2.py3-none-any.whl", hash = "sha256:f3a22400bce1b0c701683820ac4f3b159cd301acab067c51c653e06961600597", size = 21228, upload-time = "2025-11-21T20:35:49.444Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -42,7 +51,7 @@ wheels = [ [[package]] name = "babeldoc" -version = "0.6.4+gloss.2" +version = "0.6.4+gloss.3" source = { editable = "." } dependencies = [ { name = "bitstring" }, @@ -116,6 +125,9 @@ dev = [ { name = "pytest" }, { name = "ruff" }, ] +release = [ + { name = "pyinstaller" }, +] [package.metadata] requires-dist = [ @@ -175,6 +187,7 @@ dev = [ { name = "pytest", specifier = ">=8.3.4" }, { name = "ruff", specifier = ">=0.9.2" }, ] +release = [{ name = "pyinstaller", specifier = "==6.21.0" }] [[package]] name = "backports-zstd" @@ -1248,6 +1261,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/a4/441aee36c6f6b249823d20fd91f9be9ab89d7c5a8ae542a4a4ca6d342d56/lxml-6.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:ed21202aec73cda4d55d1ce57b389aadb90ffb044e6cd1080b8347efe1b1ec84", size = 3508989, upload-time = "2026-05-18T19:18:38.158Z" }, ] +[[package]] +name = "macholib" +version = "1.16.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "altgraph" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/10/2f/97589876ea967487978071c9042518d28b958d87b17dceb7cdc1d881f963/macholib-1.16.4.tar.gz", hash = "sha256:f408c93ab2e995cd2c46e34fe328b130404be143469e41bc366c807448979362", size = 59427, upload-time = "2025-11-22T08:28:38.373Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/d1/a9f36f8ecdf0fb7c9b1e78c8d7af12b8c8754e74851ac7b94a8305540fc7/macholib-1.16.4-py2.py3-none-any.whl", hash = "sha256:da1a3fa8266e30f0ce7e97c6a54eefaae8edd1e5f86f3eb8b95457cae90265ea", size = 38117, upload-time = "2025-11-22T08:28:36.939Z" }, +] + [[package]] name = "markdown" version = "3.10.2" @@ -2236,6 +2261,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/63/26ec789e68e994d64ed957dd6ee5a08bb78c2f401c40e46fc142669cd8b0/peewee-4.2.6-py3-none-any.whl", hash = "sha256:b54c0f6e09c987465f8268bb2b4c1cba2e1b2788fcc0974c8e2233af1fd5af8f", size = 173774, upload-time = "2026-07-17T18:11:19.787Z" }, ] +[[package]] +name = "pefile" +version = "2024.8.26" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/4f/2750f7f6f025a1507cd3b7218691671eecfd0bbebebe8b39aa0fe1d360b8/pefile-2024.8.26.tar.gz", hash = "sha256:3ff6c5d8b43e8c37bb6e6dd5085658d658a7a0bdcd20b6a07b1fcfc1c4e9d632", size = 76008, upload-time = "2024-08-26T20:58:38.155Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/16/12b82f791c7f50ddec566873d5bdd245baa1491bac11d15ffb98aecc8f8b/pefile-2024.8.26-py3-none-any.whl", hash = "sha256:76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f", size = 74766, upload-time = "2024-08-26T21:01:02.632Z" }, +] + [[package]] name = "pillow" version = "12.3.0" @@ -2552,6 +2586,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pyinstaller" +version = "6.21.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "altgraph" }, + { name = "macholib", marker = "sys_platform == 'darwin'" }, + { name = "packaging" }, + { name = "pefile", marker = "sys_platform == 'win32'" }, + { name = "pyinstaller-hooks-contrib" }, + { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" }, + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d5/4d/ec706c3fcf39e26888c35b39615ff4d5865d184069666c47492cff1fbe50/pyinstaller-6.21.0.tar.gz", hash = "sha256:bb9fab705983e393a2d1cac77d6972513057ad800215fd861dc15ff5272e98fd", size = 4061519, upload-time = "2026-06-13T14:15:06.25Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/4a/53cf98bf66daed012dc9cd78c8203f19a675d696f2fc12afcf8c5049a0e0/pyinstaller-6.21.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:327d132389f37912609e01be62810cf96b5aa95b613903e4b8692e0d12fb0eda", size = 1052350, upload-time = "2026-06-13T14:13:55.88Z" }, + { url = "https://files.pythonhosted.org/packages/30/83/b591295c352ef464c50b4c6ffff1c4f771d875c9e833f578d1b9f564f6b3/pyinstaller-6.21.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7071d4b094d5b40deeef5fa3d3b98a1b846087f7562b49209663d5f9281fe251", size = 748477, upload-time = "2026-06-13T14:14:00.327Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8f/88fff4e403873b1e22286911350e75ff00db014aa08e57045da9d4328993/pyinstaller-6.21.0-py3-none-manylinux2014_i686.whl", hash = "sha256:6b6374d652107dd4a2eeece903ff82bb4045bb5e1006c5a158a6dcdbefe84bf2", size = 760877, upload-time = "2026-06-13T14:14:04.836Z" }, + { url = "https://files.pythonhosted.org/packages/8a/13/f0e48fbdfd1d05d948157121cea8b1b823dcb89efe6934b71fdd8bdb3f0f/pyinstaller-6.21.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:4e3108b3f02384560da70e39b8bf22b0ad597d02bd68a40d76ea91c1cfa00cad", size = 759194, upload-time = "2026-06-13T14:14:10.61Z" }, + { url = "https://files.pythonhosted.org/packages/dd/d5/ea7878cf9924ed30d946d8288777424e6d069d94f5bde56b4d0890069664/pyinstaller-6.21.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:697532279f535ad572bda613db4f821540e235c7854ca6da4d3bf0373f4415ee", size = 754979, upload-time = "2026-06-13T14:14:15.226Z" }, + { url = "https://files.pythonhosted.org/packages/9f/09/51b8905714b733bac66dbc041a7821372d70d888d273ae474c4037d4202d/pyinstaller-6.21.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:605169523a6b5ace39f13dfbff21add9f2bc43df99c7daf9394fefb2c45e8b6f", size = 754812, upload-time = "2026-06-13T14:14:20.264Z" }, + { url = "https://files.pythonhosted.org/packages/4b/43/d77779439d8c6c2e27a77bcfbd1d5cc0f568ebb611bb472b11af81b5f177/pyinstaller-6.21.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:5fa56746c1e76f93634d018502301378a2d0c382553d37d8c3c34ff436c12dd1", size = 753887, upload-time = "2026-06-13T14:14:25.268Z" }, + { url = "https://files.pythonhosted.org/packages/51/8f/c22df1f6837784ac349057ba693f08e7b1ca7a0e06f9c33c63bc6280007b/pyinstaller-6.21.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:42395ec76df8e8120c36b13339d9db8cab83e316a12839ee303cc00fc941bb74", size = 753779, upload-time = "2026-06-13T14:14:29.445Z" }, + { url = "https://files.pythonhosted.org/packages/c9/76/1ce8a27ce62ba8cf3a87c9ce6d575610f4e55d7cb0123e7512fc3f4b921a/pyinstaller-6.21.0-py3-none-win32.whl", hash = "sha256:c6b28d30d8fd99ce162ff3aab5013ed44dbfb747566b1f01b9bed7964d7c14e9", size = 1336462, upload-time = "2026-06-13T14:14:35.785Z" }, + { url = "https://files.pythonhosted.org/packages/c1/fa/ca1d7e5257dd8566a9dfc0dfb02f8a8075eeb53d4b2d3c579f1276759042/pyinstaller-6.21.0-py3-none-win_amd64.whl", hash = "sha256:7fae06c494ce0ebfe6bd3055c0e409def884f63af2e3705d06bd431ad9237fc7", size = 1397487, upload-time = "2026-06-13T14:14:42.328Z" }, + { url = "https://files.pythonhosted.org/packages/dc/75/21b51523ce8d96629b71311775a0a65f5f5a872124ab0de33e5c848f8bff/pyinstaller-6.21.0-py3-none-win_arm64.whl", hash = "sha256:f13c95c9c03fb567217135919f93815c305813126780b0ed6e0123cb8acaf025", size = 1346094, upload-time = "2026-06-13T14:14:48.914Z" }, +] + +[[package]] +name = "pyinstaller-hooks-contrib" +version = "2026.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/94/5b/c9fe0db5e83ee1c39b2258fa21d23b15e1a60786b6c5990ee5074ead8bb6/pyinstaller_hooks_contrib-2026.6.tar.gz", hash = "sha256:bef5002c32f4f50bd55b005da12cff64eca8783e7eaf86a06a62410164bab725", size = 173354, upload-time = "2026-06-08T22:37:16.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/31/f2d7343d8ed5f7c4678377886f6ce533e6eaaa131b252ce950114c2a7efa/pyinstaller_hooks_contrib-2026.6-py3-none-any.whl", hash = "sha256:fd13b8ac126b35361175edacd41a0d97080b75dd5f4b594ecefefff969509dd3", size = 457159, upload-time = "2026-06-08T22:37:14.722Z" }, +] + [[package]] name = "pylance" version = "8.0.0" @@ -2653,6 +2727,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/7b/14882602ddee241d7984a742fcb423cb4a30fb0d6efc546ac3129fba475a/python_discovery-1.5.0-py3-none-any.whl", hash = "sha256:70c4fc61b4e7404e44f01d6fc44a715c4d685ca6cea83d295922f05891877c98", size = 34205, upload-time = "2026-07-21T13:14:13.398Z" }, ] +[[package]] +name = "pywin32-ctypes" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", size = 29471, upload-time = "2024-08-14T10:15:34.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -3280,6 +3363,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/3a/21154e2d54eb3639c6bf4dbae2e531c68356bfe95990daa30df33b30d556/scipy-1.18.0-cp313-cp313-win_arm64.whl", hash = "sha256:c5dbddf60e58c2312316d097271a8e73d40eaf2eabfa4d95ed7d3695bbf2ce7b", size = 24350911, upload-time = "2026-06-19T15:00:40.062Z" }, ] +[[package]] +name = "setuptools" +version = "83.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, +] + [[package]] name = "six" version = "1.17.0" From ada97c09cee42231f768a06e9ecb4febf8b0ab56 Mon Sep 17 00:00:00 2001 From: SamsonCJ Date: Thu, 23 Jul 2026 01:39:24 -0700 Subject: [PATCH 5/8] fix: scope CodeQL PDF crypto suppressions --- SECURITY.md | 2 ++ babeldoc/pdfminer/pdfdocument.py | 28 ++++++++++++++++------------ tests/test_pdfminer_security.py | 19 +++++++++++++++++++ 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 014edcfe..977b432f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -18,6 +18,8 @@ would make valid PDFs unreadable. The narrow `codeql[py/weak-sensitive-data-hashing]` comments on those mandated operations record that reviewed exception without disabling the query for other code. +Each suppression is an otherwise-empty comment line immediately before the +mandated hash operation, which is the source-level form recognized by CodeQL. Control characters removed by the XML converter are expressed with explicit raw-string hexadecimal ranges so scanners and reviewers see the intended XML 1.0 character set unambiguously. diff --git a/babeldoc/pdfminer/pdfdocument.py b/babeldoc/pdfminer/pdfdocument.py index 5b4bb32c..5bceea6e 100644 --- a/babeldoc/pdfminer/pdfdocument.py +++ b/babeldoc/pdfminer/pdfdocument.py @@ -386,7 +386,8 @@ def compute_u(self, key: bytes) -> bytes: # Algorithm 3.5 # ISO 32000-1 Algorithm 3.5 mandates MD5 for this legacy PDF format; # this is file-format compatibility, not application password storage. - hash = md5(self.PASSWORD_PADDING) # codeql[py/weak-sensitive-data-hashing] + # codeql[py/weak-sensitive-data-hashing] + hash = md5(self.PASSWORD_PADDING) hash.update(self.docid[0]) # 3 result = Arcfour(key).encrypt(hash.digest()) # 4 for i in range(1, 20): # 5 @@ -399,7 +400,8 @@ def compute_encryption_key(self, password: bytes) -> bytes: # Algorithm 3.2 password = (password + self.PASSWORD_PADDING)[:32] # 1 # ISO 32000-1 Algorithm 3.2 mandates MD5 for legacy PDF key derivation. - hash = md5(password) # codeql[py/weak-sensitive-data-hashing] + # codeql[py/weak-sensitive-data-hashing] + hash = md5(password) hash.update(self.o) # 3 # See https://github.com/pdfminer/pdfminer.six/issues/186 hash.update(struct.pack(" bytes: if self.r >= 3: n = self.length // 8 for _ in range(50): - result = md5( # codeql[py/weak-sensitive-data-hashing] - result[:n] - ).digest() + # codeql[py/weak-sensitive-data-hashing] + result = md5(result[:n]).digest() return result[:n] def authenticate(self, password: str) -> bytes | None: @@ -442,12 +443,12 @@ def authenticate_owner_password(self, password: bytes) -> bytes | None: # Algorithm 3.7 password = (password + self.PASSWORD_PADDING)[:32] # ISO 32000-1 Algorithm 3.7 mandates MD5 for owner-password recovery. - hash = md5(password) # codeql[py/weak-sensitive-data-hashing] + # codeql[py/weak-sensitive-data-hashing] + hash = md5(password) if self.r >= 3: for _ in range(50): - hash = md5( # codeql[py/weak-sensitive-data-hashing] - hash.digest() - ) + # codeql[py/weak-sensitive-data-hashing] + hash = md5(hash.digest()) n = 5 if self.r >= 3: n = self.length // 8 @@ -622,7 +623,8 @@ def _r5_password( ) -> bytes: """Compute the password for revision 5""" # ISO 32000-2 Algorithm 2.B mandates this exact revision-5 transform. - hash = sha256(password) # codeql[py/weak-sensitive-data-hashing] + # codeql[py/weak-sensitive-data-hashing] + hash = sha256(password) hash.update(salt) if vector is not None: hash.update(vector) @@ -636,7 +638,8 @@ def _r6_password( ) -> bytes: """Compute the password for revision 6""" # ISO 32000-2 Algorithm 2.B mandates this revision-6 iterative transform. - initial_hash = sha256(password) # codeql[py/weak-sensitive-data-hashing] + # codeql[py/weak-sensitive-data-hashing] + initial_hash = sha256(password) initial_hash.update(salt) if vector is not None: initial_hash.update(vector) @@ -649,7 +652,8 @@ def _r6_password( # compute the first 16 bytes of e, # interpreted as an unsigned integer mod 3 next_hash = hashes[self._bytes_mod_3(e[:16])] - k = next_hash(e).digest() # codeql[py/weak-sensitive-data-hashing] + # codeql[py/weak-sensitive-data-hashing] + k = next_hash(e).digest() last_byte_val = e[len(e) - 1] round_no += 1 return k[:32] diff --git a/tests/test_pdfminer_security.py b/tests/test_pdfminer_security.py index a94905ee..ed3fb252 100644 --- a/tests/test_pdfminer_security.py +++ b/tests/test_pdfminer_security.py @@ -1,8 +1,27 @@ from __future__ import annotations +from pathlib import Path + from babeldoc.pdfminer.converter import XMLConverter +def test_codeql_pdf_crypto_suppressions_are_line_scoped() -> None: + source = ( + Path(__file__).parents[1] / "babeldoc" / "pdfminer" / "pdfdocument.py" + ).read_text(encoding="utf-8") + lines = source.splitlines() + marker = "# codeql[py/weak-sensitive-data-hashing]" + marker_indexes = [ + index for index, line in enumerate(lines) if line.strip() == marker + ] + + assert len(marker_indexes) == 8 + assert all( + any(call in lines[index + 1] for call in ("md5(", "sha256(", "next_hash(")) + for index in marker_indexes + ) + + def test_xml_control_filter_removes_only_disallowed_xml_controls() -> None: allowed = "\t\n\r visible" disallowed = "".join(chr(value) for value in [0, 1, 8, 11, 12, 14, 31]) From 3196ee2b04115814cecb156411be0cbb8466f013 Mon Sep 17 00:00:00 2001 From: SamsonCJ Date: Thu, 23 Jul 2026 01:51:38 -0700 Subject: [PATCH 6/8] fix: suppress mandated PDF hash findings --- SECURITY.md | 11 ++++++----- babeldoc/pdfminer/pdfdocument.py | 8 ++++---- tests/test_pdfminer_security.py | 14 +++++++++----- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 977b432f..7e2294d0 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -14,12 +14,13 @@ The vendored `babeldoc/pdfminer` implementation must decode encrypted PDF files according to ISO 32000. Legacy revisions require MD5, while revisions 5 and 6 prescribe exact SHA-2 transforms. Those calls are format parsers, not password-storage functions, and replacing them with Argon2, bcrypt, or PBKDF2 -would make valid PDFs unreadable. The narrow -`codeql[py/weak-sensitive-data-hashing]` comments on those mandated operations -record that reviewed exception without disabling the query for other code. +would make valid PDFs unreadable. Narrow, query-specific suppression annotations +on those mandated operations record that reviewed exception without disabling +the query for other code. -Each suppression is an otherwise-empty comment line immediately before the -mandated hash operation, which is the source-level form recognized by CodeQL. +Suppressions use CodeQL's otherwise-empty preceding-line form. The two legacy +MD5 results that CodeQL reports with column-scoped locations use its supported +inline `lgtm[py/weak-sensitive-data-hashing]` form instead. Control characters removed by the XML converter are expressed with explicit raw-string hexadecimal ranges so scanners and reviewers see the intended XML 1.0 character set unambiguously. diff --git a/babeldoc/pdfminer/pdfdocument.py b/babeldoc/pdfminer/pdfdocument.py index 5bceea6e..4826bbe6 100644 --- a/babeldoc/pdfminer/pdfdocument.py +++ b/babeldoc/pdfminer/pdfdocument.py @@ -386,8 +386,9 @@ def compute_u(self, key: bytes) -> bytes: # Algorithm 3.5 # ISO 32000-1 Algorithm 3.5 mandates MD5 for this legacy PDF format; # this is file-format compatibility, not application password storage. - # codeql[py/weak-sensitive-data-hashing] - hash = md5(self.PASSWORD_PADDING) + hash = md5( # lgtm[py/weak-sensitive-data-hashing] + self.PASSWORD_PADDING + ) hash.update(self.docid[0]) # 3 result = Arcfour(key).encrypt(hash.digest()) # 4 for i in range(1, 20): # 5 @@ -400,8 +401,7 @@ def compute_encryption_key(self, password: bytes) -> bytes: # Algorithm 3.2 password = (password + self.PASSWORD_PADDING)[:32] # 1 # ISO 32000-1 Algorithm 3.2 mandates MD5 for legacy PDF key derivation. - # codeql[py/weak-sensitive-data-hashing] - hash = md5(password) + hash = md5(password) # lgtm[py/weak-sensitive-data-hashing] hash.update(self.o) # 3 # See https://github.com/pdfminer/pdfminer.six/issues/186 hash.update(struct.pack(" None: Path(__file__).parents[1] / "babeldoc" / "pdfminer" / "pdfdocument.py" ).read_text(encoding="utf-8") lines = source.splitlines() - marker = "# codeql[py/weak-sensitive-data-hashing]" - marker_indexes = [ - index for index, line in enumerate(lines) if line.strip() == marker + codeql_marker = "# codeql[py/weak-sensitive-data-hashing]" + codeql_marker_indexes = [ + index for index, line in enumerate(lines) if line.strip() == codeql_marker ] + lgtm_marker = "# lgtm[py/weak-sensitive-data-hashing]" + lgtm_marker_lines = [line for line in lines if lgtm_marker in line] - assert len(marker_indexes) == 8 + assert len(codeql_marker_indexes) == 6 assert all( any(call in lines[index + 1] for call in ("md5(", "sha256(", "next_hash(")) - for index in marker_indexes + for index in codeql_marker_indexes ) + assert len(lgtm_marker_lines) == 2 + assert all("md5(" in line for line in lgtm_marker_lines) def test_xml_control_filter_removes_only_disallowed_xml_controls() -> None: From ed0913e5e6b64690f42b7031d800c0da7fbba866 Mon Sep 17 00:00:00 2001 From: SamsonCJ Date: Thu, 23 Jul 2026 01:58:50 -0700 Subject: [PATCH 7/8] fix: mark remaining PDF hash exceptions --- SECURITY.md | 4 ++-- babeldoc/pdfminer/pdfdocument.py | 6 +++--- tests/test_pdfminer_security.py | 10 ++++++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 7e2294d0..a3f50210 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -19,8 +19,8 @@ on those mandated operations record that reviewed exception without disabling the query for other code. Suppressions use CodeQL's otherwise-empty preceding-line form. The two legacy -MD5 results that CodeQL reports with column-scoped locations use its supported -inline `lgtm[py/weak-sensitive-data-hashing]` form instead. +MD5 results that CodeQL reports with column-scoped locations use its supported, +line-scoped `noqa` form instead. Control characters removed by the XML converter are expressed with explicit raw-string hexadecimal ranges so scanners and reviewers see the intended XML 1.0 character set unambiguously. diff --git a/babeldoc/pdfminer/pdfdocument.py b/babeldoc/pdfminer/pdfdocument.py index 4826bbe6..b4779709 100644 --- a/babeldoc/pdfminer/pdfdocument.py +++ b/babeldoc/pdfminer/pdfdocument.py @@ -386,8 +386,8 @@ def compute_u(self, key: bytes) -> bytes: # Algorithm 3.5 # ISO 32000-1 Algorithm 3.5 mandates MD5 for this legacy PDF format; # this is file-format compatibility, not application password storage. - hash = md5( # lgtm[py/weak-sensitive-data-hashing] - self.PASSWORD_PADDING + hash = md5( + self.PASSWORD_PADDING # noqa -- mandated PDF MD5 transform ) hash.update(self.docid[0]) # 3 result = Arcfour(key).encrypt(hash.digest()) # 4 @@ -401,7 +401,7 @@ def compute_encryption_key(self, password: bytes) -> bytes: # Algorithm 3.2 password = (password + self.PASSWORD_PADDING)[:32] # 1 # ISO 32000-1 Algorithm 3.2 mandates MD5 for legacy PDF key derivation. - hash = md5(password) # lgtm[py/weak-sensitive-data-hashing] + hash = md5(password) # noqa -- mandated PDF MD5 transform hash.update(self.o) # 3 # See https://github.com/pdfminer/pdfminer.six/issues/186 hash.update(struct.pack(" None: codeql_marker_indexes = [ index for index, line in enumerate(lines) if line.strip() == codeql_marker ] - lgtm_marker = "# lgtm[py/weak-sensitive-data-hashing]" - lgtm_marker_lines = [line for line in lines if lgtm_marker in line] + noqa_marker = "# noqa -- mandated PDF MD5 transform" + noqa_marker_lines = [line for line in lines if noqa_marker in line] assert len(codeql_marker_indexes) == 6 assert all( any(call in lines[index + 1] for call in ("md5(", "sha256(", "next_hash(")) for index in codeql_marker_indexes ) - assert len(lgtm_marker_lines) == 2 - assert all("md5(" in line for line in lgtm_marker_lines) + assert len(noqa_marker_lines) == 2 + assert all( + "md5(" in line or "PASSWORD_PADDING" in line for line in noqa_marker_lines + ) def test_xml_control_filter_removes_only_disallowed_xml_controls() -> None: From ffc0de03159b1a32b1b6d3810d1a74eef8acdfc9 Mon Sep 17 00:00:00 2001 From: SamsonCJ Date: Thu, 23 Jul 2026 02:15:37 -0700 Subject: [PATCH 8/8] fix: mark PDF MD5 as non-security use --- SECURITY.md | 7 ++++--- babeldoc/pdfminer/pdfdocument.py | 25 +++++++++++++------------ tests/test_pdfminer_security.py | 13 ++++--------- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index a3f50210..0ff0ed81 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -18,9 +18,10 @@ would make valid PDFs unreadable. Narrow, query-specific suppression annotations on those mandated operations record that reviewed exception without disabling the query for other code. -Suppressions use CodeQL's otherwise-empty preceding-line form. The two legacy -MD5 results that CodeQL reports with column-scoped locations use its supported, -line-scoped `noqa` form instead. +Legacy MD5 calls go through a dedicated constructor with Python's +`usedforsecurity=False` flag. The remaining SHA-2 suppressions use CodeQL's +otherwise-empty preceding-line form and apply only to the exact mandated +operations. Control characters removed by the XML converter are expressed with explicit raw-string hexadecimal ranges so scanners and reviewers see the intended XML 1.0 character set unambiguously. diff --git a/babeldoc/pdfminer/pdfdocument.py b/babeldoc/pdfminer/pdfdocument.py index b4779709..96344dee 100644 --- a/babeldoc/pdfminer/pdfdocument.py +++ b/babeldoc/pdfminer/pdfdocument.py @@ -7,6 +7,7 @@ from collections.abc import Iterator from collections.abc import KeysView from collections.abc import Sequence +from functools import partial from hashlib import md5 from hashlib import sha256 from hashlib import sha384 @@ -51,6 +52,11 @@ log = logging.getLogger(__name__) +# ISO 32000 requires MD5 for legacy PDF encryption revisions. The +# usedforsecurity flag makes this file-format compatibility use explicit and +# keeps it available on Python builds that restrict security-sensitive MD5. +_pdf_spec_md5 = partial(md5, usedforsecurity=False) + class PDFNoValidXRef(PDFSyntaxError): pass @@ -386,9 +392,7 @@ def compute_u(self, key: bytes) -> bytes: # Algorithm 3.5 # ISO 32000-1 Algorithm 3.5 mandates MD5 for this legacy PDF format; # this is file-format compatibility, not application password storage. - hash = md5( - self.PASSWORD_PADDING # noqa -- mandated PDF MD5 transform - ) + hash = _pdf_spec_md5(self.PASSWORD_PADDING) hash.update(self.docid[0]) # 3 result = Arcfour(key).encrypt(hash.digest()) # 4 for i in range(1, 20): # 5 @@ -401,7 +405,7 @@ def compute_encryption_key(self, password: bytes) -> bytes: # Algorithm 3.2 password = (password + self.PASSWORD_PADDING)[:32] # 1 # ISO 32000-1 Algorithm 3.2 mandates MD5 for legacy PDF key derivation. - hash = md5(password) # noqa -- mandated PDF MD5 transform + hash = _pdf_spec_md5(password) hash.update(self.o) # 3 # See https://github.com/pdfminer/pdfminer.six/issues/186 hash.update(struct.pack(" bytes: if self.r >= 3: n = self.length // 8 for _ in range(50): - # codeql[py/weak-sensitive-data-hashing] - result = md5(result[:n]).digest() + result = _pdf_spec_md5(result[:n]).digest() return result[:n] def authenticate(self, password: str) -> bytes | None: @@ -443,12 +446,10 @@ def authenticate_owner_password(self, password: bytes) -> bytes | None: # Algorithm 3.7 password = (password + self.PASSWORD_PADDING)[:32] # ISO 32000-1 Algorithm 3.7 mandates MD5 for owner-password recovery. - # codeql[py/weak-sensitive-data-hashing] - hash = md5(password) + hash = _pdf_spec_md5(password) if self.r >= 3: for _ in range(50): - # codeql[py/weak-sensitive-data-hashing] - hash = md5(hash.digest()) + hash = _pdf_spec_md5(hash.digest()) n = 5 if self.r >= 3: n = self.length // 8 @@ -474,7 +475,7 @@ def decrypt( def decrypt_rc4(self, objid: int, genno: int, data: bytes) -> bytes: assert self.key is not None key = self.key + struct.pack(" bytes: + struct.pack(" None: codeql_marker_indexes = [ index for index, line in enumerate(lines) if line.strip() == codeql_marker ] - noqa_marker = "# noqa -- mandated PDF MD5 transform" - noqa_marker_lines = [line for line in lines if noqa_marker in line] - - assert len(codeql_marker_indexes) == 6 + assert len(codeql_marker_indexes) == 3 assert all( - any(call in lines[index + 1] for call in ("md5(", "sha256(", "next_hash(")) + any(call in lines[index + 1] for call in ("sha256(", "next_hash(")) for index in codeql_marker_indexes ) - assert len(noqa_marker_lines) == 2 - assert all( - "md5(" in line or "PASSWORD_PADDING" in line for line in noqa_marker_lines - ) + assert "partial(md5, usedforsecurity=False)" in source + assert source.count("_pdf_spec_md5(") == 7 def test_xml_control_filter_removes_only_disallowed_xml_controls() -> None: