diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a1d8943d..4a512911 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -88,8 +88,9 @@ jobs: - name: Validate runtime handshake shell: bash run: | - .wheel-venv/bin/gloss-babeldoc runtime-info --json \ - | .wheel-venv/bin/python -c ' + cd "$RUNNER_TEMP" + "$GITHUB_WORKSPACE/.wheel-venv/bin/gloss-babeldoc" runtime-info --json \ + | "$GITHUB_WORKSPACE/.wheel-venv/bin/python" -c ' import json import sys from importlib.metadata import version @@ -99,3 +100,87 @@ jobs: assert payload["runtime_api_version"] == 1 assert payload["upstream"]["version"] == "0.6.4" ' + + - name: Validate packaged executor service + shell: bash + env: + BABELDOC_EXECUTOR_ALLOW_FAKE: "1" + run: | + cd "$RUNNER_TEMP" + "$GITHUB_WORKSPACE/.wheel-venv/bin/python" - <<'PY' + import http.client + import json + import os + import secrets + import select + import subprocess + import sys + import tempfile + from pathlib import Path + from urllib.parse import urlparse + + from babeldoc.tools.executor.babeldoc_adapter import run_babeldoc_request + + del run_babeldoc_request + workroot = Path(tempfile.mkdtemp(prefix="gloss-babeldoc-smoke-")) + workroot.chmod(0o700) + marker = workroot / ".executor-workroot-ready" + marker.write_text("ready\n", encoding="utf-8") + marker.chmod(0o600) + token = secrets.token_urlsafe(32) + token_file = workroot / "token" + token_file.write_text(token + "\n", encoding="utf-8") + token_file.chmod(0o600) + executable = Path(sys.executable).with_name("gloss-babeldoc") + process = subprocess.Popen( + [ + str(executable), + "serve", + "--runner", + "fake", + "--work-dir", + str(workroot), + "--token-file", + str(token_file), + ], + env=os.environ.copy(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + assert process.stdout is not None + readable, _, _ = select.select([process.stdout], [], [], 20) + assert readable, "executor service did not report ready" + line = process.stdout.readline().strip() + prefix = "__GLOSS_BABELDOC_SERVICE_READY__" + assert line.startswith(prefix), line + ready = json.loads(line.removeprefix(prefix)) + assert "auth_token" not in ready + endpoint = urlparse(ready["endpoint"]) + connection = http.client.HTTPConnection( + endpoint.hostname, + endpoint.port, + timeout=5, + ) + headers = {"Authorization": f"Bearer {token}"} + connection.request("GET", "/healthz", headers=headers) + response = connection.getresponse() + health = json.loads(response.read()) + assert response.status == 200 and health["ok"] is True + connection.request( + "POST", + "/v1/shutdown", + body=b"{}", + headers={**headers, "Content-Type": "application/json"}, + ) + response = connection.getresponse() + response.read() + connection.close() + assert response.status == 202 + assert process.wait(timeout=10) == 0 + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + PY diff --git a/DOWNSTREAM_PATCHES.md b/DOWNSTREAM_PATCHES.md index bd32fcde..c76923f6 100644 --- a/DOWNSTREAM_PATCHES.md +++ b/DOWNSTREAM_PATCHES.md @@ -8,6 +8,7 @@ upstream, and the condition under which it can be removed. | `gloss-0001` | Active | Establish downstream identity, provenance, safe automation, and maintenance policy. | Not applicable: repository governance. | The Gloss downstream is retired. | | `gloss-0002` | Active | Add a lightweight, versioned runtime capability handshake for Gloss. | Downstream integration boundary. | Upstream exposes an equivalent stable runtime protocol. | | `gloss-0003` | Active | Add reproducible dependency locking and Linux/macOS package validation. | Candidate upstream CI improvement. | Upstream adopts equivalent locked cross-platform release checks. | +| `gloss-0004` | Active | Add an authenticated, reconnectable single-worker service boundary with targeted cancellation and process isolation. | Candidate upstream executor lifecycle and correctness fixes. | Upstream exposes an equivalent authenticated service contract adopted by Gloss. | Runtime, performance, and compatibility patches will be added in separate, focused commits and PRs. diff --git a/README.md b/README.md index 481ac793..a6ec46ab 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ > This is the **Gloss-maintained downstream** of > [funstory-ai/BabelDOC](https://github.com/funstory-ai/BabelDOC). It is not an > upstream release and is not endorsed by the upstream maintainers. See -> [DOWNSTREAM.md](./DOWNSTREAM.md) for support scope, versioning, and the exact -> upstream base. +> [DOWNSTREAM.md](https://github.com/SunChJ/BabelDOC/blob/main/DOWNSTREAM.md) +> for support scope, versioning, and the exact upstream base.
diff --git a/UPSTREAM_BASE.toml b/UPSTREAM_BASE.toml index d2d5e90b..e32227e5 100644 --- a/UPSTREAM_BASE.toml +++ b/UPSTREAM_BASE.toml @@ -1,6 +1,7 @@ -# Gloss downstream provenance. Update this file only in an upstream sync PR. +# Gloss downstream provenance. Update downstream.version for every downstream +# release; update the upstream block only in a reviewed upstream sync PR. [downstream] -version = "0.6.4+gloss.1" +version = "0.6.4+gloss.2" runtime_api_version = 1 [upstream] diff --git a/babeldoc/__init__.py b/babeldoc/__init__.py index e4e9d2a5..2ed073ca 100644 --- a/babeldoc/__init__.py +++ b/babeldoc/__init__.py @@ -1,2 +1,2 @@ # Gloss downstream release identity; see DOWNSTREAM.md. -__version__ = "0.6.4+gloss.1" +__version__ = "0.6.4+gloss.2" diff --git a/babeldoc/const.py b/babeldoc/const.py index be4b3364..e96013d6 100644 --- a/babeldoc/const.py +++ b/babeldoc/const.py @@ -7,7 +7,7 @@ from pathlib import Path # Gloss downstream release identity; see DOWNSTREAM.md. -__version__ = "0.6.4+gloss.1" +__version__ = "0.6.4+gloss.2" CACHE_FOLDER = Path.home() / ".cache" / "babeldoc" diff --git a/babeldoc/gloss_cli.py b/babeldoc/gloss_cli.py index 81233304..b0ad21d0 100644 --- a/babeldoc/gloss_cli.py +++ b/babeldoc/gloss_cli.py @@ -19,7 +19,11 @@ UPSTREAM_REPOSITORY = "https://github.com/funstory-ai/BabelDOC" UPSTREAM_VERSION = "0.6.4" UPSTREAM_COMMIT = "17480db9df92ddcb37349ce34b312335226e8ec9" -CAPABILITIES = ("runtime-info.v1",) +CAPABILITIES = ( + "executor.events.ndjson.v1", + "executor.http.v1", + "runtime-info.v1", +) def package_version() -> str: @@ -64,12 +68,42 @@ def create_parser() -> argparse.ArgumentParser: action="store_true", help="Emit one machine-readable JSON object.", ) + serve = commands.add_parser( + "serve", + help="Run the authenticated loopback PDF executor service.", + ) + serve.add_argument("--host", default="127.0.0.1") + serve.add_argument("--port", type=int, default=0) + serve.add_argument( + "--runner", + choices=("babeldoc", "fake"), + default="babeldoc", + ) + serve.add_argument("--token-file") + serve.add_argument("--work-dir") + serve.add_argument("--instance-id") + serve.add_argument("--parent-pid", type=int) + serve.add_argument("--parent-start-time", type=float) return parser def cli(argv: Sequence[str] | None = None) -> int: """Run the lightweight Gloss integration CLI.""" args = create_parser().parse_args(argv) + if args.command == "serve": + from babeldoc.tools.executor.server import serve + + serve( + args.host, + args.port, + runner_name=args.runner, + token_file=args.token_file, + work_dir=args.work_dir, + instance_id=args.instance_id, + parent_pid=args.parent_pid, + parent_start_time=args.parent_start_time, + ) + return 0 if args.command != "runtime-info": # pragma: no cover - argparse owns routing raise AssertionError(f"Unexpected command: {args.command}") diff --git a/babeldoc/main.py b/babeldoc/main.py index c270f589..a2fd4d93 100644 --- a/babeldoc/main.py +++ b/babeldoc/main.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) # Gloss downstream release identity; see DOWNSTREAM.md. -__version__ = "0.6.4+gloss.1" +__version__ = "0.6.4+gloss.2" def create_parser(): diff --git a/babeldoc/tools/executor/babeldoc_adapter.py b/babeldoc/tools/executor/babeldoc_adapter.py index c4c1401d..ab608c96 100644 --- a/babeldoc/tools/executor/babeldoc_adapter.py +++ b/babeldoc/tools/executor/babeldoc_adapter.py @@ -2,12 +2,13 @@ import asyncio import logging +import os +import signal import threading import time from pathlib import Path from typing import Any -import pyarrow from babeldoc import __version__ as babeldoc_version from babeldoc.format.pdf.translation_config import TranslateResult from babeldoc.format.pdf.translation_config import TranslationConfig @@ -59,22 +60,30 @@ def emit(event_type: str, payload: dict[str, Any]) -> None: cancel_watcher.start() result = _run_async_translate(config, emit) - logger.info("BabelDOC execution finished: task_id=%s", task_id) - emit("result", translate_result_to_payload(result, config)) + if cancel_state.cancel_requested: + emit("cancelled", {"reason": "client_request"}) + else: + logger.info("BabelDOC execution finished: task_id=%s", task_id) + emit("result", translate_result_to_payload(result, config)) + except asyncio.CancelledError: + emit("cancelled", {"reason": "client_request"}) except Exception as exc: - logger.exception("BabelDOC execution failed: task_id=%s", task_id) - user_message = ( - exc.message_for_user if isinstance(exc, BabelDocReportedError) else None - ) - emit( - "error", - { - "code": _error_code(exc), - "message": _safe_message(exc), - "message_for_user": user_message, - "details": {"exception_type": exc.__class__.__name__}, - }, - ) + if cancel_state.cancel_requested: + emit("cancelled", {"reason": "client_request"}) + else: + logger.exception("BabelDOC execution failed: task_id=%s", task_id) + user_message = ( + exc.message_for_user if isinstance(exc, BabelDocReportedError) else None + ) + emit( + "error", + { + "code": _error_code(exc), + "message": _safe_message(exc), + "message_for_user": user_message, + "details": {"exception_type": exc.__class__.__name__}, + }, + ) finally: if stop_cancel_watcher is not None: stop_cancel_watcher.set() @@ -96,6 +105,19 @@ def request_cancel(self) -> None: if config is not None: config.cancel_translation() + def reapply_cancel(self) -> None: + with self._lock: + if not self._cancel_requested: + return + config = self._config + if config is not None: + config.cancel_translation() + + @property + def cancel_requested(self) -> bool: + with self._lock: + return self._cancel_requested + def attach_config(self, config: TranslationConfig) -> None: with self._lock: self._config = config @@ -105,14 +127,40 @@ def attach_config(self, config: TranslationConfig) -> None: def _watch_cancel_pipe(cancel_recv, cancel_state: _CancelState, stop_event) -> None: + parent_lost = False while not stop_event.is_set(): try: if cancel_recv.poll(0.05): cancel_recv.recv() cancel_state.request_cancel() - return + break except (BrokenPipeError, EOFError, OSError): + parent_lost = True + cancel_state.request_cancel() + break + hard_exit_at = time.monotonic() + 2.0 if parent_lost else None + while True: + if stop_event.wait(0.05): + if parent_lost: + _hard_exit_after_parent_loss() return + cancel_state.reapply_cancel() + if hard_exit_at is not None and time.monotonic() >= hard_exit_at: + _hard_exit_after_parent_loss() + + +def _hard_exit_after_parent_loss() -> None: + process_id = os.getpid() + kill_signal = getattr(signal, "SIGKILL", None) + if kill_signal is not None and hasattr(os, "getpgrp") and hasattr(os, "killpg"): + try: + if os.getpgrp() == process_id: + os.killpg(process_id, kill_signal) + except OSError: + logger.exception( + "failed to kill executor worker process group after parent loss" + ) + os._exit(130) def build_translation_config( @@ -133,6 +181,10 @@ def mark(name: str, started_at: float) -> None: gateways = _required_object(request, "gateways") assets = _optional_object(request, "assets") metadata = _optional_object(request, "metadata") + no_dual = _required_bool(translation, "no_dual") + no_mono = _required_bool(translation, "no_mono") + if no_dual and no_mono: + raise ValueError("no_dual and no_mono cannot both be true") started_at = time.monotonic() input_file = resolve_file(workroot, _required_str(paths, "input_file")) @@ -145,11 +197,17 @@ def mark(name: str, started_at: float) -> None: mark("paths", started_at) started_at = time.monotonic() - qps = _required_int(runtime_limits, "qps") - report_interval = _required_number(runtime_limits, "report_interval_seconds") + qps = _required_positive_int(runtime_limits, "qps") + report_interval = _required_positive_number( + runtime_limits, + "report_interval_seconds", + ) set_translate_rate_limiter(qps) - max_pages_per_part = _required_int(runtime_limits, "max_pages_per_part") + max_pages_per_part = _required_positive_int( + runtime_limits, + "max_pages_per_part", + ) split_strategy = TranslationConfig.create_max_pages_per_part_split_strategy( max_pages_per_part ) @@ -196,8 +254,8 @@ def mark(name: str, started_at: float) -> None: lang_in=_required_str(translation, "lang_in"), lang_out=_required_str(translation, "lang_out"), pages=_optional_str(translation, "pages"), - no_dual=_required_bool(translation, "no_dual"), - no_mono=_required_bool(translation, "no_mono"), + no_dual=no_dual, + no_mono=no_mono, qps=qps, doc_layout_model=doc_layout_model, skip_clean=_required_bool(translation, "skip_clean"), @@ -221,7 +279,10 @@ def mark(name: str, started_at: float) -> None: ocr_workaround=_required_bool(translation, "ocr_workaround"), custom_system_prompt=_optional_str(translation, "custom_system_prompt"), glossaries=glossaries, - pool_max_workers=_required_int(runtime_limits, "pool_max_workers"), + pool_max_workers=_required_positive_int( + runtime_limits, + "pool_max_workers", + ), auto_extract_glossary=_required_bool(translation, "auto_extract_glossary"), auto_enable_ocr_workaround=_required_bool( translation, "auto_enable_ocr_workaround" @@ -238,7 +299,10 @@ def mark(name: str, started_at: float) -> None: translation, "remove_non_formula_lines" ), metadata_extra_data=_optional_str(metadata, "metadata_extra_data"), - term_pool_max_workers=_required_int(runtime_limits, "term_pool_max_workers"), + term_pool_max_workers=_required_positive_int( + runtime_limits, + "term_pool_max_workers", + ), ) mark("translation_config", started_at) @@ -273,21 +337,44 @@ def translate_result_to_payload( result: TranslateResult, config: TranslationConfig ) -> dict[str, Any]: workroot = get_workroot() + output_dir = resolve_dir(workroot, str(config.output_dir)) + if config.no_mono and config.no_dual: + raise ValueError("no_dual and no_mono cannot both be true") + pdf_paths = { + "mono_pdf": result.mono_pdf_path, + "dual_pdf": result.dual_pdf_path, + "mono_no_watermark_pdf": result.no_watermark_mono_pdf_path, + "dual_no_watermark_pdf": result.no_watermark_dual_pdf_path, + } files = { - "mono_pdf": relative_to_workroot(workroot, result.mono_pdf_path), - "dual_pdf": relative_to_workroot(workroot, result.dual_pdf_path), - "mono_no_watermark_pdf": relative_to_workroot( - workroot, result.no_watermark_mono_pdf_path - ), - "dual_no_watermark_pdf": relative_to_workroot( - workroot, result.no_watermark_dual_pdf_path - ), - "auto_extracted_glossary_csv": relative_to_workroot( - workroot, result.auto_extracted_glossary_path - ), + key: _validated_pdf_result_path(workroot, output_dir, path, key) + for key, path in pdf_paths.items() + if path is not None } + if not files: + raise ValueError("BabelDOC result did not contain a PDF") + if not config.no_mono and not { + "mono_pdf", + "mono_no_watermark_pdf", + }.intersection(files): + raise ValueError("BabelDOC result did not contain the requested mono PDF") + if not config.no_dual and not { + "dual_pdf", + "dual_no_watermark_pdf", + }.intersection(files): + raise ValueError("BabelDOC result did not contain the requested dual PDF") + glossary_path = relative_to_workroot( + workroot, + result.auto_extracted_glossary_path, + ) + if glossary_path: + glossary_file = resolve_file(workroot, glossary_path) + _require_inside_output_dir(glossary_file, output_dir, "auto glossary") + if glossary_file.suffix.lower() != ".csv": + raise ValueError("auto_extracted_glossary_csv is not a CSV") + files["auto_extracted_glossary_csv"] = glossary_path return { - "files": {key: value for key, value in files.items() if value}, + "files": files, "metrics": { "time_consume_seconds": _number_or_zero( getattr(result, "total_seconds", None) @@ -306,6 +393,52 @@ def translate_result_to_payload( } +def _validated_pdf_result_path( + workroot: Path, + output_dir: Path, + value: str | Path, + field_name: str, +) -> str: + try: + path = Path(value).resolve(strict=True) + relative_path = relative_to_workroot(workroot, path) + except (OSError, ValueError) as exc: + raise ValueError(f"{field_name} is outside the workroot or missing") from exc + if relative_path is None or not path.is_file(): + raise ValueError(f"{field_name} is not a regular file") + _require_inside_output_dir(path, output_dir, field_name) + if path.suffix.lower() != ".pdf": + raise ValueError(f"{field_name} is not a PDF") + try: + with path.open("rb") as handle: + header = handle.read(1024) + except OSError as exc: + raise ValueError(f"{field_name} is not readable") from exc + if b"%PDF-" not in header: + raise ValueError(f"{field_name} does not contain a PDF header") + try: + import pymupdf + + with pymupdf.open(path) as document: + if document.page_count < 1: + raise ValueError(f"{field_name} contains no pages") + document.load_page(0) + except ValueError: + raise + except Exception as exc: + raise ValueError(f"{field_name} is not a readable PDF") from exc + return relative_path + + +def _require_inside_output_dir(path: Path, output_dir: Path, field_name: str) -> None: + try: + path.relative_to(output_dir) + except ValueError as exc: + raise ValueError( + f"{field_name} is outside the execution output directory" + ) from exc + + def _run_async_translate(config: TranslationConfig, emit) -> TranslateResult: from babeldoc.format.pdf import high_level @@ -461,15 +594,15 @@ def _required_bool(root: dict[str, Any], key: str) -> bool: raise ValueError(f"{key} must be a boolean") -def _required_int(root: dict[str, Any], key: str) -> int: +def _required_positive_int(root: dict[str, Any], key: str) -> int: value = root.get(key) - if isinstance(value, int): + if isinstance(value, int) and not isinstance(value, bool) and value > 0: return value - raise ValueError(f"{key} must be an integer") + raise ValueError(f"{key} must be a positive integer") -def _required_number(root: dict[str, Any], key: str) -> float: +def _required_positive_number(root: dict[str, Any], key: str) -> float: value = root.get(key) - if isinstance(value, int | float): + if isinstance(value, int | float) and not isinstance(value, bool) and value > 0: return float(value) - raise ValueError(f"{key} must be a number") + raise ValueError(f"{key} must be a positive number") diff --git a/babeldoc/tools/executor/protocol.py b/babeldoc/tools/executor/protocol.py index 02e5abac..424859ac 100644 --- a/babeldoc/tools/executor/protocol.py +++ b/babeldoc/tools/executor/protocol.py @@ -5,10 +5,23 @@ from typing import Any MAX_EVENT_LOG_SIZE = 1000 +MAX_EXECUTION_HISTORY_SIZE = 16 MAX_SEQUENCE = 2**63 - 1 SEQUENCE_HEADROOM = 100_000_000 MAX_INITIAL_SEQUENCE = MAX_SEQUENCE - SEQUENCE_HEADROOM +EXECUTION_STATUSES = frozenset( + {"running", "cancelling", "succeeded", "failed", "cancelled"} +) +ACTIVE_EXECUTION_STATUSES = frozenset({"running", "cancelling"}) +TERMINAL_EXECUTION_STATUSES = frozenset({"succeeded", "failed", "cancelled"}) +TERMINAL_EVENT_TYPES = frozenset({"result", "error", "cancelled"}) +TERMINAL_EVENT_STATUS = { + "result": "succeeded", + "error": "failed", + "cancelled": "cancelled", +} + @dataclass(frozen=True) class WorkerEvent: @@ -21,6 +34,7 @@ class EventEnvelope: type: str execution_id: str sequence: int + emitted_at: float payload: dict[str, Any] def to_json_line(self) -> bytes: @@ -28,6 +42,7 @@ def to_json_line(self) -> bytes: "type": self.type, "execution_id": self.execution_id, "sequence": self.sequence, + "emitted_at": self.emitted_at, "payload": self.payload, } return ( diff --git a/babeldoc/tools/executor/runner.py b/babeldoc/tools/executor/runner.py index c18216a1..2c2b5714 100644 --- a/babeldoc/tools/executor/runner.py +++ b/babeldoc/tools/executor/runner.py @@ -2,7 +2,9 @@ import logging import multiprocessing +import os import shutil +import signal import threading import time from collections.abc import Callable @@ -10,10 +12,13 @@ from pathlib import Path from typing import Any +import psutil +from babeldoc.tools.executor.protocol import TERMINAL_EVENT_TYPES from babeldoc.tools.executor.protocol import WorkerEvent ProcessTarget = Callable[[dict[str, Any], Any, Any], None] logger = logging.getLogger(__name__) +KILL_SIGNAL = getattr(signal, "SIGKILL", signal.SIGTERM) def _configure_child_logging() -> None: @@ -36,6 +41,7 @@ def _run_process_target( progress_send, cancel_recv, ) -> None: + _isolate_process_group() _configure_child_logging() task_id = _task_id(request) started_at = time.monotonic() @@ -55,6 +61,15 @@ def _forkserver_warmup_target() -> None: return +def _isolate_process_group() -> None: + if not hasattr(os, "setsid"): + return + try: + os.setsid() + except OSError: + logger.warning("executor subprocess could not create a private process group") + + class ExecutionRunner: def run( self, @@ -173,6 +188,10 @@ def run( @staticmethod def _resolve_io(request: dict[str, Any]) -> tuple[Path, Path]: + from babeldoc.tools.executor.workroot import get_workroot + from babeldoc.tools.executor.workroot import resolve_dir + from babeldoc.tools.executor.workroot import resolve_file + paths = request.get("paths") if not isinstance(paths, dict): raise ValueError("paths must be an object") @@ -184,14 +203,9 @@ def _resolve_io(request: dict[str, Any]) -> tuple[Path, Path]: if not isinstance(output_value, str) or not output_value: raise ValueError("paths.output_dir is required") - input_path = Path(input_value) - output_dir = Path(output_value) - if not input_path.is_absolute(): - input_path = Path.cwd() / input_path - if not output_dir.is_absolute(): - output_dir = Path.cwd() / output_dir - if not input_path.is_file(): - raise FileNotFoundError(str(input_path)) + workroot = get_workroot(require_ready_file=True) + input_path = resolve_file(workroot, input_value) + output_dir = resolve_dir(workroot, output_value, create=True) return input_path, output_dir @@ -247,7 +261,14 @@ def run( args=(self._target, request, progress_send, cancel_recv), ) started_at = time.monotonic() - process.start() + try: + process.start() + except BaseException: + progress_recv.close() + progress_send.close() + cancel_recv.close() + cancel_send.close() + raise logger.info( "executor subprocess started: task_id=%s pid=%s start_elapsed=%.3fs", task_id, @@ -258,15 +279,25 @@ def run( cancel_recv.close() terminal_seen = False + cancellation_requested = False + cancellation_cleanup_completed = False try: while True: if abort_event.is_set(): + cancellation_requested = True logger.warning( "executor subprocess cancellation requested: task_id=%s pid=%s", task_id, process.pid, ) - self._send_cancel(cancel_send) + terminal_seen = self._cancel_process( + process, + progress_recv, + cancel_send, + emit, + terminal_seen, + ) + cancellation_cleanup_completed = True return if progress_recv.poll(self._poll_seconds): @@ -282,6 +313,7 @@ def run( process.exitcode, ) self._emit_missing_terminal_error(process.exitcode, emit) + terminal_seen = True return if item is None: if not terminal_seen: @@ -293,19 +325,23 @@ def run( process.exitcode, ) self._emit_missing_terminal_error(process.exitcode, emit) + terminal_seen = True return event = self._coerce_event(item) - emit(event) - if event.type in {"result", "error"}: - terminal_seen = True + terminal_seen = self._emit_unless_terminal_seen( + event, + emit, + terminal_seen, + ) + if terminal_seen: if event.type == "result": logger.info( "executor subprocess emitted terminal result: task_id=%s pid=%s", task_id, process.pid, ) - else: + elif event.type == "error": logger.warning( "executor subprocess emitted terminal error: task_id=%s pid=%s code=%s message=%s", task_id, @@ -313,6 +349,13 @@ def run( event.payload.get("code"), event.payload.get("message"), ) + else: + logger.info( + "executor subprocess emitted terminal cancelled: task_id=%s pid=%s reason=%s", + task_id, + process.pid, + event.payload.get("reason"), + ) return continue @@ -329,12 +372,19 @@ def run( process.exitcode, ) self._emit_missing_terminal_error(process.exitcode, emit) + terminal_seen = True return finally: - self._send_cancel(cancel_send) - cancel_send.close() - progress_recv.close() - self._stop_process(process) + try: + if cancellation_requested and not cancellation_cleanup_completed: + self._stop_process(process) + elif not cancellation_requested and terminal_seen: + self._wait_for_terminal_exit(process) + elif not cancellation_requested: + self._stop_process(process) + finally: + cancel_send.close() + progress_recv.close() @staticmethod def _coerce_event(item: Any) -> WorkerEvent: @@ -361,10 +411,91 @@ def _drain_progress( if item is None: return terminal_seen event = self._coerce_event(item) - emit(event) - terminal_seen = event.type in {"result", "error"} + terminal_seen = self._emit_unless_terminal_seen( + event, + emit, + terminal_seen, + ) + return terminal_seen + + def _cancel_process( + self, + process: multiprocessing.Process, + progress_recv: Any, + cancel_send: Any, + emit: Callable[[WorkerEvent], None], + terminal_seen: bool, + ) -> bool: + process_group_id = process.pid + self._send_cancel(cancel_send) + deadline = time.monotonic() + self._join_timeout_seconds + while process.is_alive() and time.monotonic() < deadline: + remaining = max(0.0, deadline - time.monotonic()) + if progress_recv.poll(min(self._poll_seconds, remaining)): + try: + item = progress_recv.recv() + except EOFError: + break + if item is None: + break + event = self._coerce_event(item) + if event.type == "cancelled": + terminal_seen = self._emit_unless_terminal_seen( + event, + emit, + terminal_seen, + ) + elif event.type not in TERMINAL_EVENT_TYPES and not terminal_seen: + emit(event) + process.join(timeout=0) + + if not process.is_alive(): + terminal_seen = self._drain_cancelled_event( + progress_recv, + emit, + terminal_seen, + ) + self._stop_process(process, process_group_id=process_group_id) + if not terminal_seen: + emit(WorkerEvent("cancelled", {"reason": "client_request"})) + terminal_seen = True + return terminal_seen + + def _drain_cancelled_event( + self, + progress_recv: Any, + emit: Callable[[WorkerEvent], None], + terminal_seen: bool, + ) -> bool: + while progress_recv.poll(): + try: + item = progress_recv.recv() + except EOFError: + return terminal_seen + if item is None: + return terminal_seen + event = self._coerce_event(item) + if event.type == "cancelled": + return self._emit_unless_terminal_seen(event, emit, terminal_seen) + if event.type not in TERMINAL_EVENT_TYPES and not terminal_seen: + emit(event) return terminal_seen + @staticmethod + def _emit_unless_terminal_seen( + event: WorkerEvent, + emit: Callable[[WorkerEvent], None], + terminal_seen: bool, + ) -> bool: + if terminal_seen: + logger.warning( + "executor ignored event after terminal event: type=%s", + event.type, + ) + return True + emit(event) + return event.type in TERMINAL_EVENT_TYPES + @staticmethod def _send_cancel(cancel_send: Any) -> None: try: @@ -392,24 +523,112 @@ def _emit_missing_terminal_error( ) ) - def _stop_process(self, process: multiprocessing.Process) -> None: - if not process.is_alive(): + def _wait_for_terminal_exit(self, process: multiprocessing.Process) -> None: + process_group_id = process.pid + process.join(timeout=self._join_timeout_seconds) + if process.is_alive() or self._process_group_exists(process_group_id): + logger.warning( + "executor subprocess group did not exit after terminal event: pid=%s", + process.pid, + ) + self._stop_process(process, process_group_id=process_group_id) + + def _stop_process( + self, + process: multiprocessing.Process, + *, + process_group_id: int | None = None, + ) -> None: + process_group_id = process_group_id or process.pid + group_exists = self._process_group_exists(process_group_id) + if not process.is_alive() and not group_exists: process.join(timeout=0) return - logger.warning( - "executor subprocess still alive during cleanup; terminating pid=%s", + "executor subprocess group still alive during cleanup; terminating pid=%s", process.pid, ) - process.terminate() - process.join(timeout=self._join_timeout_seconds) - if process.is_alive(): + self._signal_process( + process, + signal.SIGTERM, + process_group_id=process_group_id, + ) + deadline = time.monotonic() + self._join_timeout_seconds + while time.monotonic() < deadline: + process.join(timeout=0) + if not process.is_alive() and not self._process_group_exists( + process_group_id + ): + return + time.sleep(min(self._poll_seconds, max(0.0, deadline - time.monotonic()))) + if process.is_alive() or self._process_group_exists(process_group_id): logger.error( - "executor subprocess did not terminate; killing pid=%s", + "executor subprocess group did not terminate; killing pid=%s", process.pid, ) - process.kill() - process.join() + self._signal_process( + process, + KILL_SIGNAL, + process_group_id=process_group_id, + force=True, + ) + process.join(timeout=self._join_timeout_seconds) + if process.is_alive() or self._process_group_exists(process_group_id): + logger.critical( + "executor subprocess group survived SIGKILL deadline: pid=%s", + process.pid, + ) + + @staticmethod + def _signal_process( + process: multiprocessing.Process, + signal_number: int, + *, + process_group_id: int | None = None, + force: bool = False, + ) -> None: + process_group_id = process_group_id or process.pid + if process_group_id is None: + return + if hasattr(os, "killpg"): + try: + os.killpg(process_group_id, signal_number) + return + except ProcessLookupError: + if not process.is_alive(): + return + except OSError: + pass + try: + if force: + process.kill() + else: + process.terminate() + except ProcessLookupError: + return + + @staticmethod + def _process_group_exists(process_group_id: int | None) -> bool: + if process_group_id is None or not hasattr(os, "killpg"): + return False + try: + os.killpg(process_group_id, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return False + for member in psutil.process_iter(["pid", "status"]): + try: + if ( + os.getpgid(member.pid) == process_group_id + and member.info["status"] != psutil.STATUS_ZOMBIE + ): + return True + except (OSError, psutil.Error): + continue + return False def _task_id(request: dict[str, Any]) -> str: diff --git a/babeldoc/tools/executor/server.py b/babeldoc/tools/executor/server.py index 61afb0a6..25eca824 100644 --- a/babeldoc/tools/executor/server.py +++ b/babeldoc/tools/executor/server.py @@ -1,12 +1,18 @@ from __future__ import annotations import argparse -import ipaddress +import hmac import json import logging -import subprocess -import sys +import multiprocessing +import os +import secrets +import signal +import stat +import string +import threading import time +import uuid from http import HTTPStatus from http.server import BaseHTTPRequestHandler from http.server import ThreadingHTTPServer @@ -14,12 +20,18 @@ from urllib.parse import parse_qs from urllib.parse import urlparse +import psutil +from babeldoc.tools.executor.protocol import MAX_SEQUENCE from babeldoc.tools.executor.runner import FakeExecutionRunner from babeldoc.tools.executor.runner import MultiprocessExecutionRunner +from babeldoc.tools.executor.state import CursorAheadError from babeldoc.tools.executor.state import ExecutionBusyError +from babeldoc.tools.executor.state import ExecutionConflictError from babeldoc.tools.executor.state import ExecutionNotFoundError from babeldoc.tools.executor.state import ExecutionStore from babeldoc.tools.executor.state import ReplayGapError +from babeldoc.tools.executor.workroot import WORKROOT_ENV +from babeldoc.tools.executor.workroot import WORKROOT_READY_FILE from babeldoc.tools.executor.workroot import get_workroot from babeldoc.tools.executor.workroot import relative_to_workroot from babeldoc.tools.executor.workroot import resolve_file @@ -27,36 +39,378 @@ logger = logging.getLogger(__name__) WATERMARK_TIMEOUT_SECONDS = 600 +WATERMARK_POLL_SECONDS = 0.05 +WATERMARK_TERMINATE_TIMEOUT_SECONDS = 1.0 +WATERMARK_KILL_TIMEOUT_SECONDS = 5.0 +MAX_JSON_BODY_BYTES = 1024 * 1024 +READY_PREFIX = "__GLOSS_BABELDOC_SERVICE_READY__" +SERVICE_ID = "gloss-babeldoc" +SCHEMA_VERSION = 1 +REQUEST_TIMEOUT_SECONDS = 10.0 +SHUTDOWN_CANCEL_TIMEOUT_SECONDS = 5.0 +SHUTDOWN_DRAIN_POLL_SECONDS = 0.25 +PARENT_WATCHDOG_INTERVAL_SECONDS = 1.0 +_TOKEN_CHARACTERS = frozenset(string.ascii_letters + string.digits + "-_") +ALLOW_FAKE_RUNNER_ENV = "BABELDOC_EXECUTOR_ALLOW_FAKE" + + +def _dispatch_watermark_operation( + operation: str, + input_file: Path, + output_file: Path, + asset_files: tuple[Path, ...], +) -> None: + from babeldoc.tools.executor.watermark_transform import add_corner_watermark + from babeldoc.tools.executor.watermark_transform import add_tiled_watermark + + if operation == "watermark1": + if len(asset_files) != 1: + raise ValueError("watermark1 requires exactly one asset") + add_tiled_watermark(input_file, output_file, asset_files[0]) + return + if operation == "watermark2": + if len(asset_files) != 2: + raise ValueError("watermark2 requires exactly two assets") + add_corner_watermark( + input_file, + output_file, + asset_files[0], + asset_files[1], + ) + return + raise ValueError("unsupported watermark operation") + + +def _run_watermark_process_target( + operation: str, + input_file: Path, + output_file: Path, + asset_files: tuple[Path, ...], +) -> None: + if hasattr(os, "setsid"): + try: + os.setsid() + except OSError: + pass + try: + _dispatch_watermark_operation( + operation, + input_file, + output_file, + asset_files, + ) + except BaseException: + # Keep request-derived paths out of child tracebacks while preserving a + # non-zero exit status for the supervising service. + raise SystemExit(1) from None + + +def _watermark_process_group_exists(process_group_id: int | None) -> bool: + if process_group_id is None or not hasattr(os, "killpg"): + return False + try: + os.killpg(process_group_id, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return False + for member in psutil.process_iter(["pid", "status"]): + try: + if ( + os.getpgid(member.pid) == process_group_id + and member.info["status"] != psutil.STATUS_ZOMBIE + ): + return True + except (OSError, psutil.Error): + continue + return False + + +def _signal_watermark_process( + process: multiprocessing.Process, + signal_number: int, + *, + force: bool = False, +) -> None: + process_group_id = process.pid + if process_group_id is None: + return + if hasattr(os, "killpg"): + try: + os.killpg(process_group_id, signal_number) + return + except ProcessLookupError: + if not process.is_alive(): + return + except OSError: + pass + try: + if force: + process.kill() + else: + process.terminate() + except ProcessLookupError: + return + + +def _stop_watermark_process(process: multiprocessing.Process) -> None: + process_group_id = process.pid + try: + if not process.is_alive() and not _watermark_process_group_exists( + process_group_id + ): + process.join(timeout=0) + return + + _signal_watermark_process(process, signal.SIGTERM) + deadline = time.monotonic() + WATERMARK_TERMINATE_TIMEOUT_SECONDS + while time.monotonic() < deadline: + process.join(timeout=0) + if not process.is_alive() and not _watermark_process_group_exists( + process_group_id + ): + return + time.sleep( + min( + WATERMARK_POLL_SECONDS, + max(0.0, deadline - time.monotonic()), + ) + ) + + if process.is_alive() or _watermark_process_group_exists(process_group_id): + _signal_watermark_process( + process, + getattr(signal, "SIGKILL", signal.SIGTERM), + force=True, + ) + process.join(timeout=WATERMARK_KILL_TIMEOUT_SECONDS) + if process.is_alive() or _watermark_process_group_exists(process_group_id): + logger.critical( + "watermark subprocess group survived cleanup: pid=%s", + process.pid, + ) + finally: + if not process.is_alive(): + process.close() + + +class RequestBodyError(ValueError): + def __init__(self, message: str, status: HTTPStatus = HTTPStatus.BAD_REQUEST): + super().__init__(message) + self.status = status + + +class ServiceStoppingError(RuntimeError): + pass class ExecutorServer(ThreadingHTTPServer): - def __init__(self, server_address, store: ExecutionStore): + daemon_threads = True + allow_reuse_address = False + + def __init__( + self, + server_address, + store: ExecutionStore, + *, + token: str, + instance_id: str | None = None, + parent_pid: int | None = None, + parent_start_time: float | None = None, + runner_name: str = "injected", + workroot: Path | None = None, + ): super().__init__(server_address, ExecutorHandler) self.store = store + self.token = token + self.instance_id = instance_id or str(uuid.uuid4()) + self.parent_pid = parent_pid + self.parent_start_time = parent_start_time + self.runner_name = runner_name + self.workroot = workroot + self.started_at = time.time() + self._stopping = threading.Event() + self._lifecycle_lock = threading.RLock() + self._shutdown_worker_started = False + self._shutdown_cancel_active = False + self._watchdog_stop = threading.Event() + self._watchdog: threading.Thread | None = None + + @property + def stopping(self) -> bool: + return self._stopping.is_set() + + def identity_payload(self) -> dict: + host, port = self.server_address[:2] + return { + "schema_version": SCHEMA_VERSION, + "protocol_version": 1, + "service_id": SERVICE_ID, + "instance_id": self.instance_id, + "pid": os.getpid(), + "process_start_time": _process_start_time(os.getpid()), + "endpoint": f"http://{host}:{port}", + "started_at": self.started_at, + "runner": self.runner_name, + "parent_pid": self.parent_pid, + "parent_start_time": self.parent_start_time, + } + + def get_request(self): + request, client_address = super().get_request() + request.settimeout(REQUEST_TIMEOUT_SECONDS) + return request, client_address + + def admit_execution(self, request: dict) -> dict: + with self._lifecycle_lock: + if self._stopping.is_set(): + raise ServiceStoppingError + return self.store.create(request) + + def begin_heavy_operation(self, operation_id: str) -> threading.Event: + with self._lifecycle_lock: + if self._stopping.is_set(): + raise ServiceStoppingError + return self.store.begin_heavy_operation(operation_id) + + def start_parent_watchdog(self) -> None: + if self.parent_pid is None or self._watchdog is not None: + return + self._watchdog = threading.Thread( + target=self._watch_parent, + name="gloss-babeldoc-parent-watchdog", + daemon=True, + ) + self._watchdog.start() + + def request_shutdown( + self, + *, + cancel_active: bool = True, + cancel_reason: str = "service_shutdown", + defer_server_stop: bool = False, + ) -> bool: + with self._lifecycle_lock: + self._shutdown_cancel_active = self._shutdown_cancel_active or cancel_active + if cancel_active: + self._watchdog_stop.set() + if self._stopping.is_set(): + if cancel_active: + self.store.abort_current(reason=cancel_reason) + started = False + else: + self._stopping.set() + if cancel_active: + self.store.abort_current(reason=cancel_reason) + started = True + if not defer_server_stop: + self._ensure_shutdown_worker() + return started + + def _ensure_shutdown_worker(self) -> None: + with self._lifecycle_lock: + if self._shutdown_worker_started: + return + self._shutdown_worker_started = True + threading.Thread( + target=self._finish_shutdown, + name="gloss-babeldoc-shutdown", + daemon=True, + ).start() + + def _finish_shutdown(self) -> None: + wait_until_idle = getattr(self.store, "wait_until_idle", None) + if callable(wait_until_idle): + while True: + with self._lifecycle_lock: + cancel_active = self._shutdown_cancel_active + timeout_seconds = ( + SHUTDOWN_CANCEL_TIMEOUT_SECONDS + if cancel_active + else SHUTDOWN_DRAIN_POLL_SECONDS + ) + if wait_until_idle(timeout_seconds=timeout_seconds): + break + if cancel_active: + logger.warning( + "executor shutdown timed out waiting for active worker" + ) + break + try: + self.shutdown() + finally: + self._watchdog_stop.set() + + def _watch_parent(self) -> None: + while not self._watchdog_stop.wait(PARENT_WATCHDOG_INTERVAL_SECONDS): + if _process_matches(self.parent_pid, self.parent_start_time): + continue + logger.warning("executor parent disappeared; shutting down") + self.request_shutdown( + cancel_active=True, + cancel_reason="parent_exit", + ) + return class ExecutorHandler(BaseHTTPRequestHandler): server: ExecutorServer def do_GET(self): - if self.path == "/healthz": - try: - workroot = get_workroot() - proof_file = workroot / ".executor-healthz-write-proof" - proof_file.write_text("ok", encoding="utf-8") - proof_file.unlink() - except ValueError as exc: - self._write_json( - HTTPStatus.SERVICE_UNAVAILABLE, - {"status": "error", "code": "workroot_unavailable"}, - ) - logger.warning("executor healthz failed: %s", exc) - return - self._write_json(HTTPStatus.OK, {"status": "ok"}) + if not self._authenticate(): return parsed = urlparse(self.path) parts = parsed.path.strip("/").split("/") + if parsed.path == "/healthz": + payload = self.server.identity_payload() + workroot_error = _workroot_health_error(self.server.workroot) + payload.update( + { + "status": ( + "stopping" + if self.server.stopping + else "error" + if workroot_error + else "ok" + ), + "ok": not self.server.stopping and workroot_error is None, + } + ) + if workroot_error is not None: + payload["code"] = "workroot_unavailable" + self._write_json( + HTTPStatus.SERVICE_UNAVAILABLE + if self.server.stopping or workroot_error is not None + else HTTPStatus.OK, + payload, + ) + return + + if parsed.path == "/v1/runtime": + from babeldoc.gloss_cli import build_runtime_info + + payload = build_runtime_info() + payload["service"] = self.server.identity_payload() + self._write_json(HTTPStatus.OK, payload) + return + + if parsed.path == "/v1/executions/current": + self._write_json( + HTTPStatus.OK, + {"execution": _active_snapshot(self.server.store)}, + ) + return + + if parsed.path == "/v1/executions/latest": + self._write_json( + HTTPStatus.OK, + {"execution": _latest_snapshot(self.server.store)}, + ) + return + if ( len(parts) == 4 and parts[:2] == ["v1", "executions"] @@ -79,20 +433,42 @@ def do_GET(self): "after_sequence is required", ) return + if not 0 <= after_seq <= MAX_SEQUENCE: + self._write_error( + HTTPStatus.BAD_REQUEST, + "invalid_request", + "after_sequence is outside the supported range", + ) + return self._stream_events(parts[2], after_seq) return + if len(parts) == 3 and parts[:2] == ["v1", "executions"]: + self._execution_snapshot(parts[2]) + return + self._write_error(HTTPStatus.NOT_FOUND, "not_found", "not found") def do_POST(self): + if not self._authenticate(): + return + parsed = urlparse(self.path) parts = parsed.path.strip("/").split("/") if parsed.path == "/v1/executions": self._create_execution() return - if parsed.path == "/v1/abort": - self._abort_current() + if ( + len(parts) == 4 + and parts[:2] == ["v1", "executions"] + and parts[3] == "cancel" + ): + self._cancel_execution(parts[2]) + return + + if parsed.path == "/v1/shutdown": + self._shutdown() return if parsed.path in {"/v1/pdf/watermark1", "/v1/pdf/watermark2"}: @@ -107,17 +483,66 @@ def log_message(self, fmt, *args): return logger.debug(fmt, *args) + def _authenticate(self) -> bool: + authorization = self.headers.get("Authorization", "") + scheme, separator, presented = authorization.partition(" ") + try: + token_matches = hmac.compare_digest( + presented.encode("ascii"), + self.server.token.encode("ascii"), + ) + except UnicodeEncodeError: + token_matches = False + if separator and scheme.lower() == "bearer" and token_matches: + return True + self._write_error( + HTTPStatus.UNAUTHORIZED, + "unauthorized", + "a valid bearer token is required", + headers={"WWW-Authenticate": "Bearer"}, + ) + return False + + def _execution_snapshot(self, execution_id: str) -> None: + try: + snapshot = self.server.store.snapshot(execution_id) + except ExecutionNotFoundError: + self._write_error( + HTTPStatus.NOT_FOUND, + "execution_not_found", + "execution not found", + ) + return + self._write_json(HTTPStatus.OK, snapshot) + def _create_execution(self): request: dict | None = None try: request = self._read_json_body() - response = self.server.store.create(request) - except json.JSONDecodeError: - self._write_error(HTTPStatus.BAD_REQUEST, "invalid_request", "invalid json") + response = self.server.admit_execution(request) + except RequestBodyError as exc: + self._write_error(exc.status, "invalid_request", str(exc)) return except ValueError as exc: self._write_error(HTTPStatus.BAD_REQUEST, "invalid_request", str(exc)) return + except ServiceStoppingError: + self._write_error( + HTTPStatus.SERVICE_UNAVAILABLE, + "service_stopping", + "executor service is stopping", + ) + return + except ExecutionConflictError as exc: + self._write_json( + HTTPStatus.CONFLICT, + { + "code": "idempotency_conflict", + "message": "task_id was already used for a different request", + "snapshot": exc.snapshot, + }, + ) + return except ExecutionBusyError as exc: logger.warning( "executor rejected create because it is busy: requested_task_id=%s active_task_id=%s active_execution_id=%s", @@ -141,22 +566,65 @@ def _create_execution(self): response.get("execution_id"), response.get("initial_sequence"), ) - self._write_json(HTTPStatus.CREATED, response) + self._write_json( + HTTPStatus.OK if response.get("replayed") else HTTPStatus.CREATED, + response, + ) - def _abort_current(self): - logger.warning("executor abort requested") - self.server.store.abort_current() - self._write_json(HTTPStatus.ACCEPTED, {"status": "aborting"}) + def _cancel_execution(self, execution_id: str) -> None: + try: + self._read_json_body() + response = _cancel_execution(self.server.store, execution_id) + except RequestBodyError as exc: + self._write_error(exc.status, "invalid_request", str(exc)) + return + except ExecutionNotFoundError: + self._write_error( + HTTPStatus.NOT_FOUND, + "execution_not_found", + "execution not found", + ) + return + self._write_json(HTTPStatus.ACCEPTED, response) + + def _shutdown(self) -> None: + try: + request = self._read_json_body() + except RequestBodyError as exc: + self._write_error(exc.status, "invalid_request", str(exc)) + return + cancel_active = request.get("cancel_active", True) + if not isinstance(cancel_active, bool): + self._write_error( + HTTPStatus.BAD_REQUEST, + "invalid_request", + "cancel_active must be a boolean", + ) + return + started = self.server.request_shutdown( + cancel_active=cancel_active, + defer_server_stop=True, + ) + try: + self._write_json( + HTTPStatus.ACCEPTED, + {"status": "stopping", "already_stopping": not started}, + ) + finally: + self.server._ensure_shutdown_worker() def _run_watermark(self, operation: str) -> None: try: request = self._read_json_body() + workroot = self.server.workroot + if workroot is None: + raise ValueError("executor workroot is unavailable") operation_id, input_file, output_file, asset_files = ( - self._validate_watermark_request(operation, request) + self._validate_watermark_request(workroot, operation, request) ) - abort_event = self.server.store.begin_heavy_operation(operation_id) - except json.JSONDecodeError: - self._write_error(HTTPStatus.BAD_REQUEST, "invalid_request", "invalid json") + abort_event = self.server.begin_heavy_operation(operation_id) + except RequestBodyError as exc: + self._write_error(exc.status, "invalid_request", str(exc)) return except FileNotFoundError as exc: self._write_error( @@ -168,6 +636,23 @@ def _run_watermark(self, operation: str) -> None: except ValueError as exc: self._write_error(HTTPStatus.BAD_REQUEST, "invalid_request", str(exc)) return + except ServiceStoppingError: + self._write_error( + HTTPStatus.SERVICE_UNAVAILABLE, + "service_stopping", + "executor service is stopping", + ) + return + except ExecutionConflictError as exc: + self._write_json( + HTTPStatus.CONFLICT, + { + "code": "idempotency_conflict", + "message": "operation_id was already used", + "snapshot": exc.snapshot, + }, + ) + return except ExecutionBusyError as exc: self._write_json( HTTPStatus.CONFLICT, @@ -179,62 +664,82 @@ def _run_watermark(self, operation: str) -> None: ) return - logger.info( - "executor watermark operation started: operation=%s operation_id=%s input=%s output=%s assets=%s", - operation, - operation_id, - relative_to_workroot(get_workroot(), input_file), - relative_to_workroot(get_workroot(), output_file), - len(asset_files), - ) + error_code: str | None = None + error_message: str | None = None + error_status = HTTPStatus.INTERNAL_SERVER_ERROR + output_relative: str | None = None try: - self._run_watermark_subprocess( + input_relative = relative_to_workroot(workroot, input_file) + output_relative = relative_to_workroot(workroot, output_file) + logger.info( + "executor watermark operation started: operation=%s operation_id=%s input=%s output=%s assets=%s", + operation, + operation_id, + input_relative, + output_relative, + len(asset_files), + ) + self._run_watermark_process( operation, input_file, output_file, asset_files, abort_event, ) - self._write_json( - HTTPStatus.OK, - { - "operation_id": operation_id, - "output_file": relative_to_workroot(get_workroot(), output_file), - }, - ) - logger.info( - "executor watermark operation finished: operation=%s operation_id=%s output=%s", - operation, - operation_id, - relative_to_workroot(get_workroot(), output_file), - ) except TimeoutError: - logger.exception( - "executor watermark operation timed out: operation=%s operation_id=%s", - operation, - operation_id, - ) - self._write_error( - HTTPStatus.INTERNAL_SERVER_ERROR, - "transform_timeout", - "watermark transform timed out", - ) - except RuntimeError: + if abort_event.is_set(): + error_code = "operation_cancelled" + error_message = "watermark transform was cancelled" + error_status = HTTPStatus.CONFLICT + logger.warning( + "executor watermark operation cancelled: operation=%s operation_id=%s", + operation, + operation_id, + ) + else: + error_code = "transform_timeout" + error_message = "watermark transform timed out" + logger.exception( + "executor watermark operation timed out: operation=%s operation_id=%s", + operation, + operation_id, + ) + except Exception: + error_code = "transform_failed" + error_message = "watermark transform failed" logger.exception( "executor watermark operation failed: operation=%s operation_id=%s", operation, operation_id, ) - self._write_error( - HTTPStatus.INTERNAL_SERVER_ERROR, - "transform_failed", - "watermark transform failed", - ) finally: - self.server.store.finish_heavy_operation(operation_id) + self.server.store.finish_heavy_operation( + operation_id, + error_code=error_code, + error_message=error_message, + ) + if error_code is not None: + self._write_error(error_status, error_code, error_message or error_code) + return + if output_relative is None: # pragma: no cover - guarded by success path + raise AssertionError("watermark output path was not resolved") + self._write_json( + HTTPStatus.OK, + { + "operation_id": operation_id, + "output_file": output_relative, + }, + ) + logger.info( + "executor watermark operation finished: operation=%s operation_id=%s output=%s", + operation, + operation_id, + output_relative, + ) @staticmethod def _validate_watermark_request( + workroot: Path, operation: str, request: dict, ) -> tuple[str, Path, Path, list[Path]]: @@ -245,7 +750,6 @@ def _validate_watermark_request( if options is not None and options != {}: raise ValueError("options must be an empty object") - workroot = get_workroot() input_value = request.get("input_file") output_value = request.get("output_file") if not isinstance(input_value, str) or not input_value: @@ -263,9 +767,11 @@ def _validate_watermark_request( request, ) output_file = resolve_inside_workroot(workroot, output_value) - output_file.parent.mkdir(parents=True, exist_ok=True) if input_file == output_file: raise ValueError("output_file must not overwrite input_file") + if output_file in asset_files: + raise ValueError("output_file must not overwrite an asset file") + output_file.parent.mkdir(parents=True, exist_ok=True) return operation_id, input_file, output_file, asset_files @staticmethod @@ -297,36 +803,34 @@ def _resolve_watermark_assets( return [asset_file_1, asset_file_2] @staticmethod - def _run_watermark_subprocess( + def _run_watermark_process( operation: str, input_file: Path, output_file: Path, asset_files: list[Path], abort_event, ) -> None: - command = [ - sys.executable, - "-m", - "babeldoc.tools.executor.watermark_transform", - operation, - "--input", - str(input_file), - "--output", - str(output_file), - ] - for asset_file in asset_files: - command.extend(["--asset", str(asset_file)]) - process = subprocess.Popen( # noqa: S603 - command, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + context = multiprocessing.get_context("spawn") + process = context.Process( + target=_run_watermark_process_target, + args=( + operation, + input_file, + output_file, + tuple(asset_files), + ), ) + try: + process.start() + except BaseException: + process.close() + raise deadline = time.monotonic() + WATERMARK_TIMEOUT_SECONDS try: while True: - return_code = process.poll() - if return_code is not None: - if return_code != 0: + if not process.is_alive(): + process.join(timeout=0) + if process.exitcode != 0: raise RuntimeError("watermark transform failed") if not output_file.is_file(): raise RuntimeError("watermark transform did not create output") @@ -335,29 +839,36 @@ def _run_watermark_subprocess( raise TimeoutError("watermark transform aborted") if time.monotonic() >= deadline: raise TimeoutError("watermark transform timed out") - time.sleep(0.05) + time.sleep(WATERMARK_POLL_SECONDS) finally: - if process.poll() is None: - process.terminate() - try: - process.wait(timeout=1) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=5) + _stop_watermark_process(process) def _stream_events(self, execution_id: str, after_seq: int): try: self.server.store.replay(execution_id, after_seq) + except CursorAheadError: + self._write_json( + HTTPStatus.CONFLICT, + { + "code": "cursor_ahead", + "message": "requested sequence is ahead of the execution", + "snapshot": self._snapshot_or_none(execution_id), + }, + ) + return except ReplayGapError: logger.error( "executor event stream replay gap: execution_id=%s after_sequence=%s", execution_id, after_seq, ) - self._write_error( + self._write_json( HTTPStatus.GONE, - "replay_gap", - "requested sequence is no longer available", + { + "code": "replay_gap", + "message": "requested sequence is no longer available", + "snapshot": self._snapshot_or_none(execution_id), + }, ) return except ExecutionNotFoundError: @@ -375,86 +886,451 @@ def _stream_events(self, execution_id: str, after_seq: int): self.send_response(HTTPStatus.OK) self.send_header("Content-Type", "application/x-ndjson") + self.send_header("Cache-Control", "no-store") self.end_headers() logger.info( "executor event stream attached: execution_id=%s after_sequence=%s", execution_id, after_seq, ) + cursor = after_seq try: for event in self.server.store.stream(execution_id, after_seq): - self.wfile.write(event.to_json_line()) + cursor = event.sequence + self.wfile.write( + _event_json_line(event, instance_id=self.server.instance_id) + ) self.wfile.flush() - except (BrokenPipeError, ConnectionResetError): + except (ReplayGapError, ExecutionNotFoundError) as exc: + code = ( + "replay_gap" + if isinstance(exc, ReplayGapError) + else "execution_not_found" + ) + message = ( + "event history changed while the stream was attached" + if isinstance(exc, ReplayGapError) + else "execution is no longer retained" + ) + try: + self.wfile.write( + _stream_error_json_line( + execution_id=execution_id, + after_sequence=cursor, + instance_id=self.server.instance_id, + code=code, + message=message, + snapshot=self._snapshot_or_none(execution_id), + ) + ) + self.wfile.flush() + except (BrokenPipeError, ConnectionResetError, TimeoutError, OSError): + pass + except (BrokenPipeError, ConnectionResetError, TimeoutError, OSError): logger.warning( "executor event stream disconnected: execution_id=%s after_sequence=%s", execution_id, after_seq, ) - return logger.info( "executor event stream ended: execution_id=%s after_sequence=%s", execution_id, after_seq, ) - def _read_json_body(self): - length = int(self.headers.get("Content-Length", "0")) - raw = self.rfile.read(length) + def _snapshot_or_none(self, execution_id: str) -> dict | None: + try: + return self.server.store.snapshot(execution_id) + except ExecutionNotFoundError: + return None + + def _read_json_body(self) -> dict: + if self.headers.get("Transfer-Encoding") is not None: + raise RequestBodyError("Transfer-Encoding is not supported") + raw_length = self.headers.get("Content-Length", "0") + try: + length = int(raw_length) + except ValueError as exc: + raise RequestBodyError("Content-Length must be an integer") from exc + if length < 0: + raise RequestBodyError("Content-Length must not be negative") + if length > MAX_JSON_BODY_BYTES: + raise RequestBodyError( + "json body exceeds 1 MiB limit", + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + ) + try: + raw = self.rfile.read(length) + except (TimeoutError, OSError) as exc: + raise RequestBodyError( + "request body timed out", + HTTPStatus.REQUEST_TIMEOUT, + ) from exc + if len(raw) != length: + raise RequestBodyError("request body ended before Content-Length") if not raw: return {} - return json.loads(raw) + try: + payload = json.loads(raw) + except (ValueError, RecursionError) as exc: + raise RequestBodyError("invalid json") from exc + if not isinstance(payload, dict): + raise RequestBodyError("json body must be an object") + return payload - def _write_json(self, status: HTTPStatus, payload: dict): + def _write_json( + self, + status: HTTPStatus, + payload: dict, + *, + headers: dict[str, str] | None = None, + ): data = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode( "utf-8" ) self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(data))) + self.send_header("Cache-Control", "no-store") + for name, value in (headers or {}).items(): + self.send_header(name, value) self.end_headers() self.wfile.write(data) - def _write_error(self, status: HTTPStatus, code: str, message: str): - self._write_json(status, {"code": code, "message": message}) + def _write_error( + self, + status: HTTPStatus, + code: str, + message: str, + *, + headers: dict[str, str] | None = None, + ): + self._write_json( + status, + {"code": code, "message": message}, + headers=headers, + ) def _is_loopback_host(host: str) -> bool: - """Return True iff ``host`` binds to a loopback interface only. + """Keep v1 on an unambiguous IPv4 loopback endpoint.""" + return host == "127.0.0.1" - Used by ``serve`` to warn when the executor sidecar is exposed beyond - its intended trust boundary (loopback / Unix-domain peer). - """ - if host in ("localhost",): - return True + +def _event_json_line(event, *, instance_id: str) -> bytes: + payload = { + "schema_version": SCHEMA_VERSION, + "service_id": SERVICE_ID, + "instance_id": instance_id, + "type": event.type, + "execution_id": event.execution_id, + "sequence": event.sequence, + "emitted_at": event.emitted_at, + "payload": event.payload, + } + return ( + json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n" + ).encode("utf-8") + + +def _stream_error_json_line( + *, + execution_id: str, + after_sequence: int, + instance_id: str, + code: str, + message: str, + snapshot: dict | None, +) -> bytes: + payload = { + "schema_version": SCHEMA_VERSION, + "service_id": SERVICE_ID, + "instance_id": instance_id, + "type": "stream_error", + "execution_id": execution_id, + "sequence": None, + "emitted_at": time.time(), + "payload": { + "code": code, + "message": message, + "after_sequence": after_sequence, + "snapshot": snapshot, + }, + } + return ( + json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n" + ).encode("utf-8") + + +def _active_snapshot(store: ExecutionStore) -> dict | None: + active_snapshot = getattr(store, "active_snapshot", None) + if callable(active_snapshot): + return active_snapshot() + return _latest_snapshot(store) + + +def _latest_snapshot(store: ExecutionStore) -> dict | None: + current_snapshot = getattr(store, "current_snapshot", None) + if callable(current_snapshot): + return current_snapshot() + record = getattr(store, "_current", None) + if record is None: + return None try: - return ipaddress.ip_address(host).is_loopback - except ValueError: + return store.snapshot(record.execution_id) + except ExecutionNotFoundError: + return None + + +def _cancel_execution(store: ExecutionStore, execution_id: str) -> dict: + cancel = getattr(store, "cancel", None) + if callable(cancel): + return cancel(execution_id) + snapshot = store.snapshot(execution_id) + current = _active_snapshot(store) + if current is None or current.get("execution_id") != execution_id: + raise ExecutionNotFoundError(execution_id) + store.abort_current() + return {**snapshot, "status": "aborted"} + + +def _process_matches(process_id: int | None, start_time: float | None) -> bool: + if process_id is None or process_id <= 0: + return False + try: + process = psutil.Process(process_id) + if not process.is_running() or process.status() == psutil.STATUS_ZOMBIE: + return False + if start_time is None: + return True + return abs(process.create_time() - start_time) <= 0.01 + except (psutil.Error, OSError, ValueError): return False +def _process_start_time(process_id: int) -> float | None: + try: + process = psutil.Process(process_id) + if not process.is_running() or process.status() == psutil.STATUS_ZOMBIE: + return None + return process.create_time() + except (psutil.Error, OSError, ValueError): + return None + + +def _resolve_parent_identity( + process_id: int | None, + supplied_start_time: float | None, +) -> tuple[int | None, float | None]: + if process_id is None: + if supplied_start_time is not None: + raise ValueError("parent_start_time requires parent_pid") + return None, None + if process_id <= 0: + raise ValueError("parent_pid must be positive") + actual_start_time = _process_start_time(process_id) + if actual_start_time is None: + raise ValueError("parent process is not running") + if ( + supplied_start_time is not None + and abs(actual_start_time - supplied_start_time) > 0.01 + ): + raise ValueError("parent process identity does not match parent_start_time") + return process_id, actual_start_time + + +def _load_token(token_file: str | Path | None) -> tuple[str, bool]: + if token_file is not None: + path = Path(token_file) + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise ValueError("unable to read executor token file") from exc + try: + file_stat = os.fstat(descriptor) + if not stat.S_ISREG(file_stat.st_mode): + raise ValueError("executor token file must be a regular file") + if hasattr(os, "geteuid") and file_stat.st_uid != os.geteuid(): + raise ValueError("executor token file must be owned by this user") + if os.name != "nt" and stat.S_IMODE(file_stat.st_mode) & 0o077: + raise ValueError("executor token file permissions must be 0600") + raw = os.read(descriptor, 4097) + finally: + os.close(descriptor) + if len(raw) > 4096: + raise ValueError("executor token file is too large") + try: + value = raw.decode("ascii").strip() + except UnicodeDecodeError as exc: + raise ValueError("executor token must be ASCII") from exc + if not 32 <= len(value) <= 256 or any( + character not in _TOKEN_CHARACTERS for character in value + ): + raise ValueError("executor token must be 32-256 URL-safe characters") + return value, False + return secrets.token_urlsafe(32), True + + +def _configure_workroot( + work_dir: str | Path | None, + *, + required: bool, +) -> Path | None: + if work_dir is None and not required: + return None + if work_dir is not None: + path = Path(work_dir).resolve() + os.environ[WORKROOT_ENV] = str(path) + workroot = get_workroot(require_ready_file=True) + home = Path.home().resolve() + if workroot == Path(workroot.anchor) or workroot == home: + raise ValueError("executor work directory is too broad") + _validate_private_path(workroot, expected_directory=True) + _validate_private_path( + workroot / WORKROOT_READY_FILE, + expected_directory=False, + ) + error = _workroot_health_error(workroot) + if error is not None: + raise ValueError("executor work directory is not writable") + return workroot + + +def _validate_private_path(path: Path, *, expected_directory: bool) -> None: + try: + path_stat = path.lstat() + except OSError as exc: + raise ValueError(f"required private path is unavailable: {path.name}") from exc + expected_type = stat.S_ISDIR if expected_directory else stat.S_ISREG + if stat.S_ISLNK(path_stat.st_mode) or not expected_type(path_stat.st_mode): + raise ValueError(f"required private path has an unsafe type: {path.name}") + if hasattr(os, "geteuid") and path_stat.st_uid != os.geteuid(): + raise ValueError(f"required private path has a different owner: {path.name}") + if os.name != "nt" and stat.S_IMODE(path_stat.st_mode) & 0o077: + raise ValueError( + f"required private path permissions must be private: {path.name}" + ) + + +def _workroot_health_error(workroot: Path | None) -> str | None: + if workroot is None: + return None + proof = workroot / f".executor-health-{uuid.uuid4().hex}" + descriptor: int | None = None + try: + descriptor = os.open( + proof, + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, + ) + os.write(descriptor, b"ok") + except OSError as exc: + return exc.__class__.__name__ + finally: + if descriptor is not None: + os.close(descriptor) + try: + proof.unlink(missing_ok=True) + except OSError: + pass + return None + + +def _install_signal_handlers(server: ExecutorServer) -> dict[int, object]: + if threading.current_thread() is not threading.main_thread(): + return {} + previous: dict[int, object] = {} + + def handle_shutdown(_signal_number, _frame) -> None: + server.request_shutdown(cancel_active=True) + + for signal_number in (signal.SIGINT, signal.SIGTERM): + previous[signal_number] = signal.getsignal(signal_number) + signal.signal(signal_number, handle_shutdown) + return previous + + +def _restore_signal_handlers(previous: dict[int, object]) -> None: + for signal_number, handler in previous.items(): + signal.signal(signal_number, handler) + + def serve( - host: str, - port: int, + host: str = "127.0.0.1", + port: int = 0, store: ExecutionStore | None = None, runner_name: str = "babeldoc", -): + *, + token_file: str | Path | None = None, + work_dir: str | Path | None = None, + instance_id: str | None = None, + parent_pid: int | None = None, + parent_start_time: float | None = None, +) -> None: if not _is_loopback_host(host): - logger.warning( - "executor sidecar binding to non-loopback host %r has no built-in " - "authentication; treat the bind interface as a trust boundary and " - "ensure only intended peers can reach it", - host, - ) - runner = _create_runner(runner_name) - if store is None and isinstance(runner, MultiprocessExecutionRunner): - runner.warmup() + raise ValueError("executor service must bind to a loopback host") + if not 0 <= port <= 65_535: + raise ValueError("port must be between 0 and 65535") + parent_pid, parent_start_time = _resolve_parent_identity( + parent_pid, + parent_start_time, + ) + if store is None and runner_name == "babeldoc" and token_file is None: + raise ValueError("babeldoc runner requires a private token file") + if ( + store is None + and runner_name == "fake" + and os.environ.get(ALLOW_FAKE_RUNNER_ENV) != "1" + ): + raise ValueError("fake runner is disabled outside explicit test mode") + + workroot = _configure_workroot( + work_dir, + required=store is None, + ) + if store is None: + runner = _create_runner(runner_name) + if isinstance(runner, MultiprocessExecutionRunner): + runner.warmup() + store = ExecutionStore(runner) + token, generated_token = _load_token(token_file) server = ExecutorServer( (host, port), - store or ExecutionStore(runner), + store, + token=token, + instance_id=instance_id, + parent_pid=parent_pid, + parent_start_time=parent_start_time, + runner_name=runner_name, + workroot=workroot, + ) + previous_signal_handlers = _install_signal_handlers(server) + ready = server.identity_payload() + ready.update({"type": "ready"}) + if generated_token: + ready["auth_token"] = token + print( + READY_PREFIX + + json.dumps( + ready, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ), + flush=True, ) - logger.info("starting executor on %s:%s", host, port) - server.serve_forever() + server.start_parent_watchdog() + logger.info("starting executor on %s:%s", *server.server_address[:2]) + try: + server.serve_forever() + finally: + server.request_shutdown(cancel_active=True) + server.store.wait_until_idle(timeout_seconds=5.0) + server.server_close() + _restore_signal_handlers(previous_signal_handlers) def _create_runner(runner_name: str): @@ -484,7 +1360,21 @@ def main(): ) parser = argparse.ArgumentParser(description="Run HTTP executor.") parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--port", type=int, default=7860) + parser.add_argument("--port", type=int, default=0) parser.add_argument("--runner", choices=["babeldoc", "fake"], default="babeldoc") + parser.add_argument("--token-file") + parser.add_argument("--work-dir") + parser.add_argument("--instance-id") + parser.add_argument("--parent-pid", type=int) + parser.add_argument("--parent-start-time", type=float) args = parser.parse_args() - serve(args.host, args.port, runner_name=args.runner) + serve( + args.host, + args.port, + runner_name=args.runner, + token_file=args.token_file, + work_dir=args.work_dir, + instance_id=args.instance_id, + parent_pid=args.parent_pid, + parent_start_time=args.parent_start_time, + ) diff --git a/babeldoc/tools/executor/state.py b/babeldoc/tools/executor/state.py index a054399b..22f5ef85 100644 --- a/babeldoc/tools/executor/state.py +++ b/babeldoc/tools/executor/state.py @@ -1,18 +1,28 @@ from __future__ import annotations +import hashlib +import json import logging +import re import secrets import threading +import time import uuid +from collections import OrderedDict from collections import deque from collections.abc import Iterable from dataclasses import dataclass from dataclasses import field from typing import Any +from babeldoc.tools.executor.protocol import ACTIVE_EXECUTION_STATUSES from babeldoc.tools.executor.protocol import MAX_EVENT_LOG_SIZE +from babeldoc.tools.executor.protocol import MAX_EXECUTION_HISTORY_SIZE from babeldoc.tools.executor.protocol import MAX_INITIAL_SEQUENCE from babeldoc.tools.executor.protocol import MAX_SEQUENCE +from babeldoc.tools.executor.protocol import TERMINAL_EVENT_STATUS +from babeldoc.tools.executor.protocol import TERMINAL_EVENT_TYPES +from babeldoc.tools.executor.protocol import TERMINAL_EXECUTION_STATUSES from babeldoc.tools.executor.protocol import EventEnvelope from babeldoc.tools.executor.protocol import WorkerEvent from babeldoc.tools.executor.runner import ExecutionRunner @@ -25,17 +35,24 @@ class ReplayGapError(Exception): pass +class CursorAheadError(Exception): + pass + + class ExecutionBusyError(Exception): def __init__(self, snapshot: dict[str, Any]): super().__init__("executor is busy") self.snapshot = snapshot -class ExecutionNotFoundError(Exception): - pass +class ExecutionConflictError(Exception): + def __init__(self, snapshot: dict[str, Any]): + super().__init__("task_id already exists with a different request") + self.snapshot = snapshot -TERMINAL_EVENT_TYPES = {"result", "error"} +class ExecutionNotFoundError(Exception): + pass @dataclass @@ -43,12 +60,17 @@ class ExecutionRecord: execution_id: str task_id: str request: dict[str, Any] + request_fingerprint: str initial_seq: int last_seq: int - status: str = "active" + created_at: float + finished_at: float | None = None + status: str = "running" events: deque[EventEnvelope] = field(default_factory=deque) abort_event: threading.Event = field(default_factory=threading.Event) first_available_seq: int | None = None + worker_finished: bool = False + cancel_reason: str = "client_request" class ExecutionStore: @@ -56,39 +78,51 @@ def __init__( self, runner: ExecutionRunner | None = None, max_event_log_size: int = MAX_EVENT_LOG_SIZE, + max_execution_history_size: int = MAX_EXECUTION_HISTORY_SIZE, ): if max_event_log_size <= 0: raise ValueError("max_event_log_size must be positive") + if max_execution_history_size <= 0: + raise ValueError("max_execution_history_size must be positive") self._runner = runner or UnavailableRunner() self._max_event_log_size = max_event_log_size + self._max_execution_history_size = max_execution_history_size self._lock = threading.RLock() self._condition = threading.Condition(self._lock) - self._current: ExecutionRecord | None = None + self._records: OrderedDict[str, ExecutionRecord] = OrderedDict() + self._task_index: dict[str, str] = {} + self._current_execution_id: str | None = None + self._active_execution_id: str | None = None def create(self, request: dict[str, Any]) -> dict[str, Any]: - task_id = request.get("task_id") - if not isinstance(task_id, str) or not task_id: - raise ValueError("task_id is required") + request_fingerprint, request_copy = self._fingerprint_request(request) + task_id = request_copy.get("task_id") + self._validate_identifier(task_id, "task_id") - with self._lock: - if self._current is not None and self._current.status == "active": + with self._condition: + existing = self._record_for_task_locked(task_id) + if existing is not None: + if existing.request_fingerprint != request_fingerprint: + raise ExecutionConflictError(self._snapshot_locked(existing)) + return self._create_response_locked(existing, replayed=True) + + active = self._active_record_locked() + if active is not None: logger.warning( "executor create rejected because active execution exists: requested_task_id=%s active_task_id=%s active_execution_id=%s", task_id, - self._current.task_id, - self._current.execution_id, + active.task_id, + active.execution_id, ) - raise ExecutionBusyError(self._snapshot_locked(self._current)) + raise ExecutionBusyError(self._snapshot_locked(active)) - initial_seq = secrets.randbelow(MAX_INITIAL_SEQUENCE) + 1 - record = ExecutionRecord( + record = self._new_record( execution_id=str(uuid.uuid4()), task_id=task_id, - request=request, - initial_seq=initial_seq, - last_seq=initial_seq, + request=request_copy, + request_fingerprint=request_fingerprint, ) - self._current = record + self._register_record_locked(record, active=True) logger.info( "executor execution created: task_id=%s execution_id=%s initial_sequence=%s", @@ -96,6 +130,33 @@ def create(self, request: dict[str, Any]) -> dict[str, Any]: record.execution_id, record.initial_seq, ) + try: + self._start_worker(record) + except Exception as exc: + logger.exception( + "executor worker thread failed to start: task_id=%s execution_id=%s", + record.task_id, + record.execution_id, + ) + with self._condition: + self._append_event_locked( + record, + WorkerEvent( + "error", + { + "code": "worker_start_failed", + "message": str(exc), + "message_for_user": None, + "details": {"exception_type": exc.__class__.__name__}, + }, + ), + ) + self._finish_worker_locked(record) + + with self._lock: + return self._create_response_locked(record, replayed=False) + + def _start_worker(self, record: ExecutionRecord) -> None: thread = threading.Thread( target=self._run, args=(record,), @@ -104,68 +165,104 @@ def create(self, request: dict[str, Any]) -> dict[str, Any]: ) thread.start() - return { - "execution_id": record.execution_id, - "status": "started", - "initial_sequence": record.initial_seq, - } - - def abort_current(self) -> None: + def cancel( + self, + execution_id: str, + *, + reason: str = "client_request", + ) -> dict[str, Any]: + if reason not in {"client_request", "service_shutdown", "parent_exit"}: + raise ValueError("unsupported cancellation reason") with self._condition: - record = self._current - if record is None: + record = self._require_record_locked(execution_id) + if record.status == "running": + logger.warning( + "executor execution cancellation requested: task_id=%s execution_id=%s", + record.task_id, + record.execution_id, + ) + record.status = "cancelling" + record.cancel_reason = reason + record.abort_event.set() self._condition.notify_all() - return - logger.warning( - "executor active execution abort requested: task_id=%s execution_id=%s status=%s", - record.task_id, - record.execution_id, - record.status, - ) - record.abort_event.set() - if record.status == "active": - record.status = "aborted" - self._condition.notify_all() + return self._snapshot_locked(record) + + def abort_current(self, *, reason: str = "client_request") -> None: + with self._lock: + execution_id = self._active_execution_id + if execution_id is None: + return + self.cancel(execution_id, reason=reason) def begin_heavy_operation(self, operation_id: str) -> threading.Event: - if not operation_id: - raise ValueError("operation_id is required") + self._validate_identifier(operation_id, "operation_id") with self._condition: - if self._current is not None and self._current.status == "active": - raise ExecutionBusyError(self._snapshot_locked(self._current)) - initial_seq = secrets.randbelow(MAX_INITIAL_SEQUENCE) + 1 - record = ExecutionRecord( + existing = self._records.get(operation_id) + if existing is not None: + raise ExecutionConflictError(self._snapshot_locked(existing)) + indexed = self._record_for_task_locked(operation_id) + if indexed is not None: + raise ExecutionConflictError(self._snapshot_locked(indexed)) + active = self._active_record_locked() + if active is not None: + raise ExecutionBusyError(self._snapshot_locked(active)) + + record = self._new_record( execution_id=operation_id, task_id=operation_id, request={}, - initial_seq=initial_seq, - last_seq=initial_seq, + request_fingerprint=hashlib.sha256(b"{}").hexdigest(), ) - self._current = record + self._register_record_locked(record, active=True) logger.info( "executor heavy operation started: operation_id=%s initial_sequence=%s", operation_id, - initial_seq, + record.initial_seq, ) self._condition.notify_all() return record.abort_event - def finish_heavy_operation(self, operation_id: str) -> None: + def finish_heavy_operation( + self, + operation_id: str, + *, + error_code: str | None = None, + error_message: str | None = None, + ) -> None: with self._condition: - if self._current is None or self._current.execution_id != operation_id: + record = self._records.get(operation_id) + if record is None or record.worker_finished: return - if self._current.status == "active": - self._current.status = "completed" + if record.status == "cancelling": + self._append_cancelled_event_locked(record) + elif record.status == "running" and error_code is not None: + self._append_event_locked( + record, + WorkerEvent( + "error", + { + "code": error_code, + "message": error_message or "heavy operation failed", + "message_for_user": None, + "details": {"operation_id": operation_id}, + }, + ), + ) + elif record.status == "running": + self._append_event_locked( + record, + WorkerEvent("result", {"operation_id": operation_id}), + ) + self._finish_worker_locked(record) logger.info( "executor heavy operation finished: operation_id=%s status=%s", operation_id, - self._current.status, + record.status, ) - self._condition.notify_all() def replay(self, execution_id: str, after_seq: int) -> list[EventEnvelope]: with self._lock: - record = self._require_current_locked(execution_id) + record = self._require_record_locked(execution_id) self._raise_if_gap_locked(record, after_seq) return [event for event in record.events if event.sequence > after_seq] @@ -178,11 +275,16 @@ def stream( cursor = after_seq while True: with self._condition: - record = self._require_current_locked(execution_id) + record = self._require_record_locked(execution_id) self._raise_if_gap_locked(record, cursor) pending = [event for event in record.events if event.sequence > cursor] status = record.status - if not pending and status == "active": + worker_finished = record.worker_finished + if ( + not pending + and status in ACTIVE_EXECUTION_STATUSES + and not worker_finished + ): self._condition.wait(timeout=wait_seconds) continue @@ -192,12 +294,44 @@ def stream( if pending and pending[-1].type in TERMINAL_EVENT_TYPES: return - if not pending and status != "active": + if not pending and ( + status in TERMINAL_EXECUTION_STATUSES or worker_finished + ): return def snapshot(self, execution_id: str) -> dict[str, Any]: with self._lock: - return self._snapshot_locked(self._require_current_locked(execution_id)) + return self._snapshot_locked(self._require_record_locked(execution_id)) + + def current_snapshot(self) -> dict[str, Any] | None: + """Return the most recently created execution, including terminal ones.""" + with self._lock: + if self._current_execution_id is None: + return None + record = self._records.get(self._current_execution_id) + return self._snapshot_locked(record) if record is not None else None + + def active_snapshot(self) -> dict[str, Any] | None: + with self._lock: + record = self._active_record_locked() + return self._snapshot_locked(record) if record is not None else None + + def wait_until_idle(self, timeout_seconds: float | None = None) -> bool: + if timeout_seconds is not None and timeout_seconds < 0: + raise ValueError("timeout_seconds must not be negative") + deadline = ( + None if timeout_seconds is None else time.monotonic() + timeout_seconds + ) + with self._condition: + while self._active_execution_id is not None: + if deadline is None: + self._condition.wait() + continue + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + self._condition.wait(timeout=remaining) + return True def _run(self, record: ExecutionRecord) -> None: def emit(event: WorkerEvent) -> None: @@ -205,8 +339,38 @@ def emit(event: WorkerEvent) -> None: try: self._runner.run(record.request, emit, record.abort_event) + except Exception as exc: + with self._condition: + if record.status == "running": + logger.exception( + "executor runner raised internal error: task_id=%s execution_id=%s", + record.task_id, + record.execution_id, + ) + self._append_event_locked( + record, + WorkerEvent( + "error", + { + "code": "internal_error", + "message": str(exc), + "message_for_user": None, + "details": {"exception_type": exc.__class__.__name__}, + }, + ), + ) + elif record.status in TERMINAL_EXECUTION_STATUSES: + logger.exception( + "executor runner raised after terminal event: task_id=%s execution_id=%s status=%s", + record.task_id, + record.execution_id, + record.status, + ) + finally: with self._condition: - if record.status == "active": + if record.status == "cancelling": + self._append_cancelled_event_locked(record) + elif record.status == "running": logger.error( "executor runner returned without terminal event: task_id=%s execution_id=%s", record.task_id, @@ -226,29 +390,11 @@ def emit(event: WorkerEvent) -> None: }, ), ) - self._condition.notify_all() - except Exception as exc: - logger.exception( - "executor runner raised internal error: task_id=%s execution_id=%s", - record.task_id, - record.execution_id, - ) - self._append_event( - record.execution_id, - WorkerEvent( - "error", - { - "code": "internal_error", - "message": str(exc), - "message_for_user": None, - "details": {"exception_type": exc.__class__.__name__}, - }, - ), - ) + self._finish_worker_locked(record) def _append_event(self, execution_id: str, event: WorkerEvent) -> None: with self._condition: - record = self._require_current_locked(execution_id) + record = self._require_record_locked(execution_id) self._append_event_locked(record, event) self._condition.notify_all() @@ -257,7 +403,9 @@ def _append_event_locked( record: ExecutionRecord, event: WorkerEvent, ) -> None: - if record.status in {"terminal", "aborted"}: + if record.status in TERMINAL_EXECUTION_STATUSES: + return + if record.status == "cancelling" and event.type in {"result", "error"}: return if record.last_seq >= MAX_SEQUENCE: record.abort_event.set() @@ -266,11 +414,17 @@ def _append_event_locked( if record.last_seq <= 0: record.abort_event.set() raise OverflowError("event sequence invariant violated") + payload = ( + self._cancelled_payload(record, event.payload) + if event.type == "cancelled" + else event.payload + ) envelope = EventEnvelope( type=event.type, execution_id=record.execution_id, sequence=record.last_seq, - payload=event.payload, + emitted_at=time.time(), + payload=payload, ) record.events.append(envelope) if record.first_available_seq is None: @@ -278,42 +432,193 @@ def _append_event_locked( while len(record.events) > self._max_event_log_size: record.events.popleft() record.first_available_seq = record.events[0].sequence - if event.type in TERMINAL_EVENT_TYPES and record.status == "active": - record.status = "terminal" - if event.type == "result": - logger.info( - "executor terminal result emitted: task_id=%s execution_id=%s sequence=%s", - record.task_id, - record.execution_id, - envelope.sequence, - ) - else: - logger.warning( - "executor terminal error emitted: task_id=%s execution_id=%s sequence=%s code=%s message=%s", - record.task_id, - record.execution_id, - envelope.sequence, - event.payload.get("code"), - event.payload.get("message"), - ) - def _require_current_locked(self, execution_id: str) -> ExecutionRecord: - if self._current is None or self._current.execution_id != execution_id: + terminal_status = TERMINAL_EVENT_STATUS.get(event.type) + if terminal_status is None: + return + record.status = terminal_status + record.finished_at = envelope.emitted_at + if event.type == "result": + logger.info( + "executor terminal result emitted: task_id=%s execution_id=%s sequence=%s", + record.task_id, + record.execution_id, + envelope.sequence, + ) + else: + logger.warning( + "executor terminal event emitted: task_id=%s execution_id=%s sequence=%s type=%s code=%s message=%s", + record.task_id, + record.execution_id, + envelope.sequence, + event.type, + event.payload.get("code"), + event.payload.get("message"), + ) + + def _append_cancelled_event_locked(self, record: ExecutionRecord) -> None: + self._append_event_locked( + record, + WorkerEvent("cancelled", {}), + ) + + @staticmethod + def _cancelled_payload( + record: ExecutionRecord, + payload: dict[str, Any], + ) -> dict[str, Any]: + reason = ( + record.cancel_reason + if record.status == "cancelling" + else payload.get("reason", "client_request") + ) + details = payload.get("details") + return { + "reason": reason, + "code": "cancelled", + "message": "execution cancelled", + "message_for_user": payload.get("message_for_user"), + "details": details if isinstance(details, dict) else {}, + } + + def _finish_worker_locked(self, record: ExecutionRecord) -> None: + record.worker_finished = True + record.request.clear() + if self._active_execution_id == record.execution_id: + self._active_execution_id = None + self._prune_history_locked() + self._condition.notify_all() + + def _new_record( + self, + *, + execution_id: str, + task_id: str, + request: dict[str, Any], + request_fingerprint: str, + ) -> ExecutionRecord: + initial_seq = secrets.randbelow(MAX_INITIAL_SEQUENCE) + 1 + return ExecutionRecord( + execution_id=execution_id, + task_id=task_id, + request=request, + request_fingerprint=request_fingerprint, + initial_seq=initial_seq, + last_seq=initial_seq, + created_at=time.time(), + ) + + def _register_record_locked( + self, + record: ExecutionRecord, + *, + active: bool, + ) -> None: + self._records[record.execution_id] = record + self._task_index[record.task_id] = record.execution_id + self._current_execution_id = record.execution_id + if active: + self._active_execution_id = record.execution_id + self._prune_history_locked() + + def _prune_history_locked(self) -> None: + while len(self._records) > self._max_execution_history_size: + execution_id, record = next(iter(self._records.items())) + if execution_id == self._active_execution_id: + return + self._records.pop(execution_id) + if self._task_index.get(record.task_id) == execution_id: + self._task_index.pop(record.task_id, None) + if self._current_execution_id == execution_id: + self._current_execution_id = next(reversed(self._records), None) + + def _record_for_task_locked(self, task_id: str) -> ExecutionRecord | None: + execution_id = self._task_index.get(task_id) + if execution_id is None: + return None + record = self._records.get(execution_id) + if record is None: + self._task_index.pop(task_id, None) + return record + + def _active_record_locked(self) -> ExecutionRecord | None: + if self._active_execution_id is None: + return None + record = self._records.get(self._active_execution_id) + if record is None: + self._active_execution_id = None + return record + + def _require_record_locked(self, execution_id: str) -> ExecutionRecord: + record = self._records.get(execution_id) + if record is None: raise ExecutionNotFoundError(execution_id) - return self._current + return record + + def _create_response_locked( + self, + record: ExecutionRecord, + *, + replayed: bool, + ) -> dict[str, Any]: + return { + "execution_id": record.execution_id, + "status": record.status, + "initial_sequence": record.initial_seq, + "replayed": replayed, + } def _snapshot_locked(self, record: ExecutionRecord) -> dict[str, Any]: return { "execution_id": record.execution_id, "task_id": record.task_id, "status": record.status, + "initial_sequence": record.initial_seq, + "first_available_sequence": record.first_available_seq, "last_sequence": record.last_seq, + "worker_finished": record.worker_finished, + "created_at": record.created_at, + "finished_at": record.finished_at, } def _raise_if_gap_locked(self, record: ExecutionRecord, after_seq: int) -> None: + if after_seq > record.last_seq: + raise CursorAheadError("requested sequence is ahead of the execution") if record.first_available_seq is None: if after_seq < record.initial_seq: raise ReplayGapError("requested sequence is no longer available") return if after_seq < record.first_available_seq - 1: raise ReplayGapError("requested sequence is no longer available") + + @staticmethod + def _fingerprint_request( + request: dict[str, Any], + ) -> tuple[str, dict[str, Any]]: + if not isinstance(request, dict): + raise ValueError("request must be an object") + try: + canonical = json.dumps( + request, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ) + request_copy = json.loads(canonical) + except (TypeError, ValueError) as exc: + raise ValueError("request must be JSON-serializable") from exc + if not isinstance(request_copy, dict): # pragma: no cover - defensive + raise ValueError("request must be an object") + fingerprint = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return fingerprint, request_copy + + @staticmethod + def _validate_identifier(value: Any, field_name: str) -> None: + if not isinstance(value, str) or not re.fullmatch( + r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}", + value, + ): + raise ValueError( + f"{field_name} must be 1-128 URL-safe identifier characters" + ) diff --git a/babeldoc/tools/executor/workroot.py b/babeldoc/tools/executor/workroot.py index 7744c744..a86f59ee 100644 --- a/babeldoc/tools/executor/workroot.py +++ b/babeldoc/tools/executor/workroot.py @@ -11,7 +11,9 @@ def get_workroot(*, require_ready_file: bool = False) -> Path: raw = os.environ.get(WORKROOT_ENV) if not raw: raise ValueError(f"{WORKROOT_ENV} is required") - workroot = Path(raw).resolve() + if "\x00" in raw: + raise ValueError(f"{WORKROOT_ENV} contains a NUL byte") + workroot = Path(os.path.normpath(os.path.realpath(raw))) if not workroot.is_dir(): raise ValueError("executor workroot must be an existing directory") if not os.access(workroot, os.R_OK | os.W_OK): @@ -22,17 +24,35 @@ def get_workroot(*, require_ready_file: bool = False) -> Path: def resolve_inside_workroot(workroot: Path, value: str) -> Path: - if not value: + if not isinstance(value, str) or not value: raise ValueError("path must be a non-empty string") - candidate = Path(value) - if not candidate.is_absolute(): - candidate = workroot / candidate - resolved = candidate.resolve(strict=False) - try: - resolved.relative_to(workroot) - except ValueError as exc: - raise ValueError("path escapes executor workroot") from exc - return resolved + if "\x00" in value: + raise ValueError("path contains a NUL byte") + + safe_root = os.path.normcase( + os.path.normpath(os.path.realpath(os.fspath(workroot))), + ) + candidate = value + if not os.path.isabs(candidate): # noqa: PTH117 - CodeQL path sanitizer + candidate = os.path.join( # noqa: PTH118 - CodeQL path sanitizer + safe_root, + candidate, + ) + resolved = os.path.normcase(os.path.normpath(os.path.realpath(candidate))) + + # Keep the normalized value that reaches filesystem APIs behind a direct + # prefix guard. CodeQL recognizes this shape as a safe path access check. + if not resolved.startswith(safe_root): + raise ValueError("path escapes executor workroot") + + # The separator-bounded check prevents prefix confusion such as allowing + # /work/jobs-attacker when the trusted root is /work/jobs. + root_prefix = safe_root + if not root_prefix.endswith(os.sep): + root_prefix += os.sep + if resolved != safe_root and not resolved.startswith(root_prefix): + raise ValueError("path escapes executor workroot") + return Path(resolved) def resolve_file(workroot: Path, value: str) -> Path: @@ -54,8 +74,12 @@ def resolve_dir(workroot: Path, value: str, *, create: bool = False) -> Path: def relative_to_workroot(workroot: Path, value: Path | str | None) -> str | None: if value is None: return None - resolved = Path(value).resolve(strict=False) - try: - return str(resolved.relative_to(workroot)) - except ValueError as exc: - raise ValueError("path escapes executor workroot") from exc + raw = os.fspath(value) + if not raw: + raise ValueError("path must be a non-empty string") + if "\x00" in raw: + raise ValueError("path contains a NUL byte") + raw = os.path.realpath(raw) + resolved = resolve_inside_workroot(workroot, raw) + safe_root = os.path.normpath(os.path.realpath(os.fspath(workroot))) + return os.path.relpath(resolved, safe_root) diff --git a/docs/gloss-service.md b/docs/gloss-service.md new file mode 100644 index 00000000..b93f88c1 --- /dev/null +++ b/docs/gloss-service.md @@ -0,0 +1,256 @@ +# Gloss service boundary + +`gloss-babeldoc serve` exposes the downstream runtime as an authenticated, +loopback-only HTTP service. It is the private integration boundary for the +Gloss desktop application, not a general network API. + +## Bootstrap and ownership + +Gloss creates a new private workroot and bearer token for every service +instance. Before launch: + +- the workroot must be an existing `0700` directory owned by the current user; +- `/.executor-workroot-ready` must be a regular `0600` file owned by + the current user; +- the token file must be a regular `0600` file owned by the current user and + contain 32–256 URL-safe ASCII characters; +- the workroot is canonicalized before use; `/`, the user's home directory, + marker/token symlinks, and group/world-accessible paths are rejected. + +The production runner requires both `--work-dir` and `--token-file`: + +```text +gloss-babeldoc serve \ + --work-dir /private/path/to/runtime \ + --token-file /private/path/to/runtime/service.token \ + --parent-pid 1234 \ + --parent-start-time 1784749200.125 +``` + +`parent-start-time` is the process creation time in Unix epoch seconds, as +reported by `psutil.Process(pid).create_time()`. The service validates the +PID/start-time pair before announcing readiness and monitors it afterward to +prevent PID-reuse mistakes. + +The service binds only to `127.0.0.1`; the default port is `0`, so the kernel +selects a free port. It writes exactly one ready record to stdout, prefixed by +`__GLOSS_BABELDOC_SERVICE_READY__`. Logs go to stderr. A ready payload contains +the schema and protocol versions, instance ID, PID and process start time, +parent identity, runner, and endpoint. Production ready records never contain +the bearer token. A generated token is available only when the explicitly +enabled fake test runner is used. + +All HTTP requests, including health checks, use: + +```text +Authorization: Bearer +``` + +Gloss must authenticate `GET /v1/runtime` and compare the returned +`instance_id`, PID, and process start time before reusing or terminating a +remembered process. A process must never be killed merely because it occupies +a remembered port. + +## HTTP endpoints + +| Method | Path | Purpose | +| --- | --- | --- | +| `GET` | `/healthz` | Liveness, identity, and workroot write probe. | +| `GET` | `/v1/runtime` | Runtime version, capabilities, and service identity. | +| `GET` | `/v1/executions/current` | The active execution, or `null`. | +| `GET` | `/v1/executions/latest` | The most recently created execution, including terminal executions. | +| `GET` | `/v1/executions/{id}` | One retained execution snapshot. | +| `GET` | `/v1/executions/{id}/events?after_sequence=N` | Replay and follow UTF-8 NDJSON events. | +| `POST` | `/v1/executions` | Idempotently start one PDF execution. | +| `POST` | `/v1/executions/{id}/cancel` | Idempotently cancel that execution only. | +| `POST` | `/v1/shutdown` | Stop admission, optionally cancel active work, and exit. | +| `POST` | `/v1/pdf/watermark1` | Apply the retained single-asset watermark transform. | +| `POST` | `/v1/pdf/watermark2` | Apply the retained two-asset watermark transform. | + +The former global `/v1/abort` route is intentionally not part of this +protocol: a delayed client must never cancel a newer execution. + +Requests are JSON objects no larger than 1 MiB. Chunked request bodies are not +accepted. A task ID is 1–128 URL-safe identifier characters. Reusing a task ID +with the same canonical request returns the existing execution; reusing it +with different data returns `409 idempotency_conflict`. + +## Execution request + +Version 1 accepts one input PDF per execution. Paths may be workroot-relative +or absolute paths already contained by the private workroot; they may not +escape it. Returned paths are always workroot-relative. The complete request +shape is: + +```json +{ + "task_id": "pdf-018f0d8f", + "paths": { + "input_file": "inputs/paper.pdf", + "output_dir": "outputs/pdf-018f0d8f", + "working_dir": "working/pdf-018f0d8f" + }, + "translation_config": { + "debug": false, + "lang_in": "en", + "lang_out": "zh-CN", + "pages": null, + "no_dual": true, + "no_mono": false, + "skip_clean": false, + "dual_translate_first": false, + "disable_rich_text_translate": true, + "use_side_by_side_dual": false, + "use_alternating_pages_dual": false, + "skip_scanned_detection": true, + "ocr_workaround": false, + "custom_system_prompt": null, + "primary_font_family": null, + "auto_extract_glossary": false, + "auto_enable_ocr_workaround": false, + "only_include_translated_page": false, + "merge_alternating_line_numbers": true, + "remove_non_formula_lines": false + }, + "runtime_limits": { + "qps": 4, + "report_interval_seconds": 0.5, + "max_pages_per_part": 50, + "pool_max_workers": 4, + "term_pool_max_workers": 4 + }, + "gateways": { + "main_llm": { + "model": "gloss-provider", + "base_url": "http://127.0.0.1:49160/v1", + "api_key": "per-task-bridge-token" + }, + "ate_llm": { + "model": "gloss-provider", + "base_url": "http://127.0.0.1:49160/v1", + "api_key": "per-task-bridge-token" + }, + "layout": { + "adapter": "rpc_doclayout8", + "base_url": "http://127.0.0.1:49161", + "requires_line_extraction": false + } + }, + "assets": { + "glossaries": [] + }, + "metadata": { + "metadata_extra_data": null + } +} +``` + +Gateway credentials are held only for the active worker and are cleared from +the retained execution record after the worker exits. They are never included +in snapshots or events. `qps`, `max_pages_per_part`, `pool_max_workers`, and +`term_pool_max_workers` must be positive integers (JSON booleans are not +integers). `report_interval_seconds` must be a positive number. `no_dual` and +`no_mono` cannot both be true. + +## State, events, and output validation + +Version 1 deliberately permits one active PDF execution per service instance. +Gloss owns the visible multi-file queue and submits the next file only after +the current worker has finished cleanup. Each PDF runs in an isolated process. + +```text +running -> succeeded + -> failed + -> cancelling -> cancelled +``` + +Snapshots contain `execution_id`, `task_id`, `status`, `initial_sequence`, +`first_available_sequence`, `last_sequence`, `worker_finished`, `created_at`, +and optional `finished_at`. Times are Unix epoch seconds. + +Normal event lines have this envelope: + +```json +{ + "schema_version": 1, + "service_id": "gloss-babeldoc", + "instance_id": "6a4e...", + "type": "progress", + "execution_id": "7c8a...", + "sequence": 812345, + "emitted_at": 1784749212.5, + "payload": { + "type": "progress_update", + "stage": "Translate Paragraphs", + "stage_current": 8, + "stage_total": 20, + "overall_progress": 42, + "part_index": 1, + "total_parts": 1 + } +} +``` + +Terminal types are exactly one of `result`, `error`, or `cancelled`. A result +contains workroot-relative PDF paths plus metrics. Before success, every +reported PDF is checked against that execution's `paths.output_dir` for +containment, regular-file type, extension, PDF header, at least one page, and +successful first-page loading with MuPDF. The requested mono and/or dual +outputs must be present. Auto-extracted glossary output is likewise confined +to the execution output directory. + +A cancelled payload is stable across cooperative and forced cancellation: + +```json +{ + "reason": "client_request", + "code": "cancelled", + "message": "execution cancelled", + "message_for_user": null, + "details": {} +} +``` + +`reason` may also be `service_shutdown` or `parent_exit`. + +## Reconnection + +Gloss stores `instance_id`, `execution_id`, and the last consumed sequence. +Disconnecting an event stream does not cancel its execution. For the same +instance, reconnect with `after_sequence=`. + +The service retains the 16 most recent executions and up to 1,000 events per +execution. An already-trimmed cursor returns `410 replay_gap` with a snapshot; +resume from `first_available_sequence - 1`. A cursor ahead of +`last_sequence` returns `409 cursor_ahead`, which normally indicates a stale +instance/cursor pairing. + +If the event window changes after a `200` stream has started, the final NDJSON +line is a `stream_error` control record. Control records use `sequence: null` +and must be handled before normal sequence de-duplication. Its payload contains +the last consumed sequence and, when the execution is still retained, its +authoritative snapshot. The snapshot is `null` if retention evicted the +execution while the stream was attached. + +When the instance ID changes, do not send an old cancellation request to the +new service. Mark the old running task interrupted and let an explicit user +retry create a new task ID. + +## Cancellation and shutdown + +Task cancellation first asks BabelDOC to stop cooperatively, then escalates to +process-group `SIGTERM` and `SIGKILL` after bounded grace periods. A cancelling +task holds the single-worker slot until the worker and descendants have exited. + +`POST /v1/shutdown` accepts `{"cancel_active": true}` by default. Admission is +closed atomically before the response is sent. `false` drains the current task; +a later shutdown request with `true` escalates that drain to cancellation. +SIGINT, SIGTERM, and parent disappearance use the same controlled path. + +## Compatibility boundary + +The protocol is advertised by `gloss-babeldoc runtime-info` as +`executor.http.v1` and `executor.events.ndjson.v1`. The ordinary `babeldoc` CLI +remains available while Gloss migrates to the service. Font caching, layout-IR +caching, and PDF-generation performance changes remain separate downstream +patches. diff --git a/docs/release-notes/v0.6.4-gloss.2.md b/docs/release-notes/v0.6.4-gloss.2.md new file mode 100644 index 00000000..9540694f --- /dev/null +++ b/docs/release-notes/v0.6.4-gloss.2.md @@ -0,0 +1,31 @@ +# BabelDOC 0.6.4+gloss.2 + +This downstream release adds the first stable PDF-execution service boundary +for the Gloss macOS application. + +## Added + +- An authenticated loopback HTTP service that binds an ephemeral port and + reports a versioned ready handshake. +- Reconnectable execution snapshots and NDJSON event streams with bounded + history, stable timestamps, and explicit replay-gap handling. +- Idempotent task creation, one active PDF execution at a time, targeted + cancellation, controlled drain, and shutdown endpoints. +- Worker-process isolation with cooperative cancellation followed by bounded + process-group termination. +- Workroot, token-file, request, and generated-PDF validation at the service + boundary. + +## Packaging and compatibility + +- Clean-wheel CI now imports the real BabelDOC adapter and starts, probes, and + shuts down the packaged service on Linux and macOS. +- The existing `babeldoc` CLI remains available while Gloss migrates to the + service protocol. +- Batch scheduling remains in Gloss; this service deliberately runs one PDF + execution at a time. +- This release does not include font caching, layout-IR caching, or PDF-save + performance changes. Those remain separate downstream patches. + +The protocol contract and reconnection rules are documented in +[`docs/gloss-service.md`](../gloss-service.md). diff --git a/mkdocs.yml b/mkdocs.yml index f2ac4eed..581bc30d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -138,6 +138,7 @@ nav: - Supported Languages: supported_languages.md - API: - Async Translation API: ImplementationDetails/AsyncTranslate/AsyncTranslate.md + - Gloss Service Boundary: gloss-service.md - Implementation Details: - ImplementationDetails/README.md - PDF Parsing: ImplementationDetails/PDFParsing/PDFParsing.md diff --git a/pyproject.toml b/pyproject.toml index 6d1ea8af..499f8e31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ # Gloss downstream modification: release identity and runtime handshake, 2026-07-21. [project] name = "BabelDOC" -version = "0.6.4+gloss.1" +version = "0.6.4+gloss.2" description = "Gloss-maintained downstream of the BabelDOC document translator" license = "AGPL-3.0" readme = "README.md" @@ -139,7 +139,6 @@ max-complexity = 10 # 函数圈复杂度阈值 [tool.ruff.lint.per-file-ignores] "babeldoc/babeldoc_exception/BabelDOCException.py" = ["N999"] "babeldoc/format/pdf/pdfinterp.py" = ["N"] # 忽略命名规范 -"babeldoc/tools/executor/babeldoc_adapter.py" = ["F401"] "babeldoc/tools/executor/server.py" = ["N802"] "tests/*" = ["S101"] # 在测试文件中允许 assert "scripts/*/tests/*" = ["S101", "S106"] # scaffold tests: allow assert + token fixture @@ -169,7 +168,7 @@ pythonpath = [".", "src"] testpaths = ["tests"] [bumpver] -current_version = "0.6.4+gloss.1" +current_version = "0.6.4+gloss.2" version_pattern = "MAJOR.MINOR.PATCH+gloss.NUM" [bumpver.file_patterns] @@ -186,6 +185,9 @@ version_pattern = "MAJOR.MINOR.PATCH+gloss.NUM" "babeldoc/const.py" = [ '__version__ = "{version}"' ] +"UPSTREAM_BASE.toml" = [ + 'version = "{version}"' +] [tool.pyright] pythonVersion = "3.10" diff --git a/tests/test_gloss_cli.py b/tests/test_gloss_cli.py index 773c2a67..62849d92 100644 --- a/tests/test_gloss_cli.py +++ b/tests/test_gloss_cli.py @@ -37,7 +37,11 @@ def test_build_runtime_info_contract() -> None: "version": "0.6.4", "commit": "17480db9df92ddcb37349ce34b312335226e8ec9", } - assert payload["capabilities"] == ["runtime-info.v1"] + assert payload["capabilities"] == [ + "executor.events.ndjson.v1", + "executor.http.v1", + "runtime-info.v1", + ] def test_runtime_info_json_output(capsys: pytest.CaptureFixture[str]) -> None: @@ -101,3 +105,66 @@ def test_unknown_command_exits_with_argparse_error() -> None: cli(["unknown"]) assert error.value.code == 2 + + +def test_serve_forwards_service_options( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + received: dict[str, object] = {} + token_file = tmp_path / "token" + + def fake_serve(host: str, port: int, **kwargs: object) -> None: + received.update({"host": host, "port": port, **kwargs}) + + monkeypatch.setattr("babeldoc.tools.executor.server.serve", fake_serve) + + assert ( + cli( + [ + "serve", + "--host", + "127.0.0.1", + "--port", + "49152", + "--runner", + "fake", + "--token-file", + str(token_file), + "--work-dir", + str(tmp_path), + "--instance-id", + "instance-1", + "--parent-pid", + "42", + "--parent-start-time", + "123.5", + ] + ) + == 0 + ) + assert received == { + "host": "127.0.0.1", + "port": 49152, + "runner_name": "fake", + "token_file": str(token_file), + "work_dir": str(tmp_path), + "instance_id": "instance-1", + "parent_pid": 42, + "parent_start_time": 123.5, + } + + +def test_serve_defaults_to_ephemeral_loopback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + received: dict[str, object] = {} + + def fake_serve(host: str, port: int, **kwargs: object) -> None: + received.update({"host": host, "port": port, **kwargs}) + + monkeypatch.setattr("babeldoc.tools.executor.server.serve", fake_serve) + + assert cli(["serve", "--runner", "fake"]) == 0 + assert received["host"] == "127.0.0.1" + assert received["port"] == 0 diff --git a/tests/tools/executor/test_babeldoc_adapter.py b/tests/tools/executor/test_babeldoc_adapter.py new file mode 100644 index 00000000..6ae99270 --- /dev/null +++ b/tests/tools/executor/test_babeldoc_adapter.py @@ -0,0 +1,522 @@ +from __future__ import annotations + +import asyncio +import secrets +import signal +import threading +import time +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pymupdf +import pytest +from babeldoc.format.pdf.translation_config import TranslateResult +from babeldoc.tools.executor import babeldoc_adapter +from babeldoc.tools.executor.babeldoc_adapter import build_translation_config +from babeldoc.tools.executor.babeldoc_adapter import run_babeldoc_request +from babeldoc.tools.executor.babeldoc_adapter import translate_result_to_payload +from babeldoc.tools.executor.workroot import WORKROOT_ENV + + +class _ProgressSender: + def __init__(self) -> None: + self.items: list[Any] = [] + + def send(self, item: Any) -> None: + self.items.append(item) + + +class _NeverCancelReceiver: + def poll(self, timeout: float) -> bool: + time.sleep(min(timeout, 0.001)) + return False + + +class _LostParentReceiver: + def poll(self, _timeout: float) -> bool: + return True + + def recv(self) -> None: + raise EOFError + + +def _config() -> SimpleNamespace: + return SimpleNamespace( + input_file="input.pdf", + output_dir="output", + no_dual=True, + no_mono=False, + pages=None, + page_ranges=None, + lang_in="en", + lang_out="zh-CN", + model="test", + ) + + +def _result( + mono: Path | None = None, + dual: Path | None = None, + glossary: Path | None = None, +) -> TranslateResult: + return TranslateResult( + mono_pdf_path=mono, + dual_pdf_path=dual, + auto_extracted_glossary_path=glossary, + ) + + +def _write_pdf(path: Path) -> None: + document = pymupdf.open() + document.new_page() + document.save(path) + document.close() + + +def _complete_execution_request(api_key: str) -> dict[str, Any]: + gateway = { + "model": "gloss-provider", + "base_url": "http://127.0.0.1:1/v1", + "api_key": api_key, + } + return { + "task_id": "contract-fixture", + "paths": { + "input_file": "input.pdf", + "output_dir": "output", + "working_dir": "working", + }, + "translation_config": { + "debug": False, + "lang_in": "en", + "lang_out": "zh-CN", + "pages": None, + "no_dual": True, + "no_mono": False, + "skip_clean": False, + "dual_translate_first": False, + "disable_rich_text_translate": True, + "use_side_by_side_dual": False, + "use_alternating_pages_dual": False, + "skip_scanned_detection": True, + "ocr_workaround": False, + "custom_system_prompt": None, + "primary_font_family": None, + "auto_extract_glossary": False, + "auto_enable_ocr_workaround": False, + "only_include_translated_page": False, + "merge_alternating_line_numbers": True, + "remove_non_formula_lines": False, + }, + "runtime_limits": { + "qps": 4, + "report_interval_seconds": 0.5, + "max_pages_per_part": 50, + "pool_max_workers": 4, + "term_pool_max_workers": 4, + }, + "gateways": { + "main_llm": dict(gateway), + "ate_llm": dict(gateway), + "layout": { + "adapter": "rpc_doclayout8", + "base_url": "http://127.0.0.1:2", + "requires_line_extraction": False, + }, + }, + "assets": {"glossaries": []}, + "metadata": {"metadata_extra_data": None}, + } + + +def test_result_payload_requires_a_real_pdf_inside_workroot( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + workroot = tmp_path / "workroot" + workroot.mkdir() + pdf = workroot / "output" / "translated.pdf" + pdf.parent.mkdir() + _write_pdf(pdf) + monkeypatch.setenv(WORKROOT_ENV, str(workroot)) + + payload = translate_result_to_payload(_result(mono=pdf), _config()) + + assert payload["files"] == { + "mono_pdf": "output/translated.pdf", + "mono_no_watermark_pdf": "output/translated.pdf", + } + + +def test_complete_execution_request_builds_translation_config( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + workroot = tmp_path / "workroot" + workroot.mkdir() + (workroot / "input.pdf").write_bytes(b"fixture") + monkeypatch.setenv(WORKROOT_ENV, str(workroot)) + api_key = secrets.token_urlsafe(32) + + config = build_translation_config( + _complete_execution_request(api_key), + task_id="contract-fixture", + ) + try: + assert config.input_file == str(workroot / "input.pdf") + assert config.output_dir == str(workroot / "output") + assert config.lang_in == "en" + assert config.lang_out == "zh-CN" + assert config.no_dual is True + assert config.no_mono is False + assert config.qps == 4 + assert config.pool_max_workers == 4 + assert config.term_pool_max_workers == 4 + assert config.skip_scanned_detection is True + assert config.disable_rich_text_translate is True + assert config.translator.api_key == api_key + assert config.doc_layout_model.host == "http://127.0.0.1:2" + finally: + config.translator._client.close() + config.term_extraction_translator._client.close() + + +@pytest.mark.parametrize( + "invalid_kind", + ["missing", "not_pdf", "corrupt_pdf", "outside_output", "outside_workroot"], +) +def test_result_payload_rejects_invalid_pdf_outputs( + invalid_kind: str, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + workroot = tmp_path / "workroot" + workroot.mkdir() + output_dir = workroot / "output" + output_dir.mkdir() + monkeypatch.setenv(WORKROOT_ENV, str(workroot)) + + if invalid_kind == "missing": + invalid = output_dir / "missing.pdf" + elif invalid_kind == "not_pdf": + invalid = output_dir / "not-a-pdf.pdf" + invalid.write_text("plain text", encoding="utf-8") + elif invalid_kind == "corrupt_pdf": + invalid = output_dir / "corrupt.pdf" + invalid.write_bytes(b"%PDF-1.7\n%%EOF\n") + elif invalid_kind == "outside_output": + invalid = workroot / "stale.pdf" + _write_pdf(invalid) + else: + invalid = tmp_path / "outside.pdf" + _write_pdf(invalid) + + with pytest.raises(ValueError): + translate_result_to_payload(_result(mono=invalid), _config()) + + +def test_result_payload_rejects_empty_result( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setenv(WORKROOT_ENV, str(tmp_path)) + (tmp_path / "output").mkdir() + + with pytest.raises(ValueError, match="did not contain a PDF"): + translate_result_to_payload(_result(), _config()) + + +def test_result_payload_requires_each_requested_pdf_mode( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + workroot = tmp_path / "workroot" + output_dir = workroot / "output" + output_dir.mkdir(parents=True) + mono = output_dir / "mono.pdf" + dual = output_dir / "dual.pdf" + _write_pdf(mono) + _write_pdf(dual) + monkeypatch.setenv(WORKROOT_ENV, str(workroot)) + + both_modes = _config() + both_modes.no_dual = False + with pytest.raises(ValueError, match="requested dual PDF"): + translate_result_to_payload(_result(mono=mono), both_modes) + + dual_only = _config() + dual_only.no_dual = False + dual_only.no_mono = True + payload = translate_result_to_payload(_result(dual=dual), dual_only) + assert set(payload["files"]) == {"dual_pdf", "dual_no_watermark_pdf"} + + with pytest.raises(ValueError, match="requested mono PDF"): + translate_result_to_payload(_result(dual=dual), _config()) + + +def test_result_payload_rejects_disabled_mono_and_dual_modes( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + output_dir = tmp_path / "output" + output_dir.mkdir() + pdf = output_dir / "translated.pdf" + _write_pdf(pdf) + monkeypatch.setenv(WORKROOT_ENV, str(tmp_path)) + config = _config() + config.no_mono = True + config.no_dual = True + + with pytest.raises(ValueError, match="cannot both be true"): + translate_result_to_payload(_result(mono=pdf), config) + + +def test_result_payload_rejects_glossary_outside_execution_output( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + output_dir = tmp_path / "output" + output_dir.mkdir() + pdf = output_dir / "translated.pdf" + _write_pdf(pdf) + stale_glossary = tmp_path / "stale.csv" + stale_glossary.write_text("source,target\n", encoding="utf-8") + monkeypatch.setenv(WORKROOT_ENV, str(tmp_path)) + + with pytest.raises(ValueError, match="outside the execution output directory"): + translate_result_to_payload( + _result(mono=pdf, glossary=stale_glossary), + _config(), + ) + + +@pytest.mark.parametrize( + ("field_name", "invalid_value"), + [ + (field_name, invalid_value) + for field_name in ( + "qps", + "max_pages_per_part", + "pool_max_workers", + "term_pool_max_workers", + ) + for invalid_value in (True, 0, -1) + ], +) +def test_runtime_integer_limits_must_be_positive_non_boolean( + field_name: str, + invalid_value: object, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + workroot = tmp_path / "workroot" + workroot.mkdir() + (workroot / "input.pdf").write_bytes(b"fixture") + monkeypatch.setenv(WORKROOT_ENV, str(workroot)) + monkeypatch.setattr( + babeldoc_adapter, + "_create_translator", + lambda *_args, **_kwargs: object(), + ) + monkeypatch.setattr( + babeldoc_adapter, + "_create_doc_layout_model", + lambda *_args, **_kwargs: SimpleNamespace( + init_font_mapper=lambda _config: None + ), + ) + request = _complete_execution_request(secrets.token_urlsafe(32)) + request["runtime_limits"][field_name] = invalid_value + + with pytest.raises(ValueError, match=f"{field_name} must be a positive integer"): + build_translation_config(request) + + +@pytest.mark.parametrize("invalid_value", [True, 0, -0.1]) +def test_report_interval_must_be_positive_non_boolean( + invalid_value: object, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + workroot = tmp_path / "workroot" + workroot.mkdir() + (workroot / "input.pdf").write_bytes(b"fixture") + monkeypatch.setenv(WORKROOT_ENV, str(workroot)) + request = _complete_execution_request(secrets.token_urlsafe(32)) + request["runtime_limits"]["report_interval_seconds"] = invalid_value + + with pytest.raises( + ValueError, + match="report_interval_seconds must be a positive number", + ): + build_translation_config(request) + + +def test_build_config_rejects_disabling_both_output_modes( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + workroot = tmp_path / "workroot" + workroot.mkdir() + (workroot / "input.pdf").write_bytes(b"fixture") + monkeypatch.setenv(WORKROOT_ENV, str(workroot)) + request = _complete_execution_request(secrets.token_urlsafe(32)) + request["translation_config"]["no_dual"] = True + request["translation_config"]["no_mono"] = True + + with pytest.raises(ValueError, match="cannot both be true"): + build_translation_config(request) + + +class _HardExitCalled(BaseException): + pass + + +@pytest.mark.skipif( + not hasattr(signal, "SIGKILL") + or not hasattr(babeldoc_adapter.os, "getpgrp") + or not hasattr(babeldoc_adapter.os, "killpg"), + reason="process-group SIGKILL is unavailable", +) +def test_parent_loss_kills_only_a_private_worker_process_group( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[object, ...]] = [] + + monkeypatch.setattr(babeldoc_adapter.os, "getpid", lambda: 123) + monkeypatch.setattr(babeldoc_adapter.os, "getpgrp", lambda: 123) + monkeypatch.setattr( + babeldoc_adapter.os, + "killpg", + lambda process_group, signal_number: calls.append( + ("killpg", process_group, signal_number) + ), + ) + + def hard_exit(status: int) -> None: + calls.append(("exit", status)) + raise _HardExitCalled + + monkeypatch.setattr(babeldoc_adapter.os, "_exit", hard_exit) + + with pytest.raises(_HardExitCalled): + babeldoc_adapter._hard_exit_after_parent_loss() + + assert calls == [("killpg", 123, signal.SIGKILL), ("exit", 130)] + + +@pytest.mark.skipif( + not hasattr(signal, "SIGKILL") + or not hasattr(babeldoc_adapter.os, "getpgrp") + or not hasattr(babeldoc_adapter.os, "killpg"), + reason="process-group SIGKILL is unavailable", +) +def test_parent_loss_does_not_signal_a_shared_process_group( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[object, ...]] = [] + + monkeypatch.setattr(babeldoc_adapter.os, "getpid", lambda: 123) + monkeypatch.setattr(babeldoc_adapter.os, "getpgrp", lambda: 456) + monkeypatch.setattr( + babeldoc_adapter.os, + "killpg", + lambda *_args: calls.append(("killpg",)), + ) + + def hard_exit(status: int) -> None: + calls.append(("exit", status)) + raise _HardExitCalled + + monkeypatch.setattr(babeldoc_adapter.os, "_exit", hard_exit) + + with pytest.raises(_HardExitCalled): + babeldoc_adapter._hard_exit_after_parent_loss() + + assert calls == [("exit", 130)] + + +def test_adapter_emits_cancelled_terminal_for_async_cancellation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sender = _ProgressSender() + monkeypatch.setattr( + babeldoc_adapter, + "build_translation_config", + lambda _request, **_kwargs: _config(), + ) + + def cancel(_config, _emit): + raise asyncio.CancelledError + + monkeypatch.setattr(babeldoc_adapter, "_run_async_translate", cancel) + + run_babeldoc_request( + {"task_id": "cancelled"}, + sender, + _NeverCancelReceiver(), + ) + + assert sender.items == [ + {"type": "cancelled", "payload": {"reason": "client_request"}}, + None, + ] + + +def test_cancel_state_reapplies_cancel_after_monitor_replacement() -> None: + class Config: + generation = 0 + + def __init__(self) -> None: + self.cancelled_generations: list[int] = [] + + def cancel_translation(self) -> None: + self.cancelled_generations.append(self.generation) + + config = Config() + state = babeldoc_adapter._CancelState() + state.attach_config(config) # type: ignore[arg-type] + state.request_cancel() + config.generation = 1 + state.reapply_cancel() + + assert config.cancelled_generations == [0, 1] + + +def test_parent_pipe_eof_requests_cooperative_cancellation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cancelled = threading.Event() + hard_exit_requested = threading.Event() + config = _config() + config.cancel_translation = cancelled.set + sender = _ProgressSender() + monkeypatch.setattr( + babeldoc_adapter, + "build_translation_config", + lambda _request, **_kwargs: config, + ) + + def wait_for_cancel(_config, _emit): + assert cancelled.wait(timeout=1) + raise asyncio.CancelledError + + monkeypatch.setattr(babeldoc_adapter, "_run_async_translate", wait_for_cancel) + monkeypatch.setattr( + babeldoc_adapter, + "_hard_exit_after_parent_loss", + hard_exit_requested.set, + ) + + run_babeldoc_request( + {"task_id": "parent-lost"}, + sender, + _LostParentReceiver(), + ) + + assert sender.items[0] == { + "type": "cancelled", + "payload": {"reason": "client_request"}, + } + assert hard_exit_requested.wait(timeout=1) diff --git a/tests/tools/executor/test_runner.py b/tests/tools/executor/test_runner.py new file mode 100644 index 00000000..708b79e1 --- /dev/null +++ b/tests/tools/executor/test_runner.py @@ -0,0 +1,334 @@ +from __future__ import annotations + +import json +import os +import signal +import subprocess +import sys +import threading +import time +from pathlib import Path +from typing import Any + +import psutil +import pytest +from babeldoc.tools.executor.protocol import WorkerEvent +from babeldoc.tools.executor.runner import MultiprocessExecutionRunner + + +def _result_then_cleanup_target(request, progress_send, _cancel_recv) -> None: + progress_send.send({"type": "result", "payload": {"files": {}}}) + time.sleep(request["cleanup_delay"]) + Path(request["marker"]).write_text("clean", encoding="utf-8") + progress_send.send(None) + + +def _duplicate_terminal_target(_request, progress_send, _cancel_recv) -> None: + progress_send.send({"type": "result", "payload": {"files": {}}}) + progress_send.send( + { + "type": "error", + "payload": {"code": "late_error", "message": "too late"}, + } + ) + progress_send.send({"type": "cancelled", "payload": {"reason": "too_late"}}) + progress_send.send(None) + + +def _cooperative_cancel_target(_request, progress_send, cancel_recv) -> None: + progress_send.send({"type": "progress", "payload": {"stage": "ready"}}) + cancel_recv.recv() + progress_send.send({"type": "cancelled", "payload": {"reason": "cooperative"}}) + progress_send.send( + { + "type": "error", + "payload": {"code": "late_error", "message": "too late"}, + } + ) + progress_send.send(None) + + +def _ignore_cancel_target(_request, progress_send, _cancel_recv) -> None: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + progress_send.send({"type": "progress", "payload": {"stage": "ready"}}) + while True: + time.sleep(1) + + +def _descendant_ignores_cancel_target(request, progress_send, _cancel_recv) -> None: + descendant = subprocess.Popen( # noqa: S603 + [ + sys.executable, + "-c", + "import signal,time; " + "signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "time.sleep(30)", + ] + ) + Path(request["pid_file"]).write_text(str(descendant.pid), encoding="utf-8") + progress_send.send({"type": "progress", "payload": {"stage": "ready"}}) + while True: + time.sleep(1) + + +def _parent_loss_worker_target(request, _progress_send, cancel_recv) -> None: + from babeldoc.tools.executor.babeldoc_adapter import _CancelState + from babeldoc.tools.executor.babeldoc_adapter import _watch_cancel_pipe + + descendant = subprocess.Popen( # noqa: S603 + [ + sys.executable, + "-c", + "import signal,time; " + "signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "time.sleep(30)", + ] + ) + Path(request["pid_file"]).write_text( + json.dumps( + { + "leader": os.getpid(), + "descendant": descendant.pid, + } + ), + encoding="utf-8", + ) + _watch_cancel_pipe(cancel_recv, _CancelState(), threading.Event()) + + +def _run_parent_loss_service(pid_file: str) -> None: + MultiprocessExecutionRunner( + _parent_loss_worker_target, + start_method="spawn", + poll_seconds=0.01, + join_timeout_seconds=0.1, + ).run( + {"task_id": "parent-loss", "pid_file": pid_file}, + lambda _event: None, + threading.Event(), + ) + + +def _runner(target, *, join_timeout: float = 0.5) -> MultiprocessExecutionRunner: + return MultiprocessExecutionRunner( + target, + start_method="spawn", + poll_seconds=0.005, + join_timeout_seconds=join_timeout, + ) + + +def test_result_allows_child_normal_cleanup(tmp_path: Path) -> None: + marker = tmp_path / "cleaned" + events: list[WorkerEvent] = [] + + _runner(_result_then_cleanup_target).run( + { + "task_id": "success", + "marker": str(marker), + "cleanup_delay": 0.05, + }, + events.append, + threading.Event(), + ) + + assert [event.type for event in events] == ["result"] + assert marker.read_text(encoding="utf-8") == "clean" + + +def test_runner_emits_only_first_terminal_event() -> None: + events: list[WorkerEvent] = [] + + _runner(_duplicate_terminal_target).run( + {"task_id": "duplicate-terminal"}, + events.append, + threading.Event(), + ) + + assert [event.type for event in events] == ["result"] + + +def test_cooperative_cancel_emits_one_cancelled_terminal() -> None: + abort_event = threading.Event() + events: list[WorkerEvent] = [] + + def emit(event: WorkerEvent) -> None: + events.append(event) + if event.type == "progress": + abort_event.set() + + _runner(_cooperative_cancel_target).run( + {"task_id": "cooperative-cancel"}, + emit, + abort_event, + ) + + assert [event.type for event in events] == ["progress", "cancelled"] + assert events[-1].payload == {"reason": "cooperative"} + + +def test_cancel_escalates_when_child_ignores_cooperative_and_sigterm() -> None: + abort_event = threading.Event() + events: list[WorkerEvent] = [] + + def emit(event: WorkerEvent) -> None: + events.append(event) + if event.type == "progress": + abort_event.set() + + started_at = time.monotonic() + _runner(_ignore_cancel_target, join_timeout=0.05).run( + {"task_id": "forced-cancel"}, + emit, + abort_event, + ) + + assert time.monotonic() - started_at < 2 + assert [event.type for event in events] == ["progress", "cancelled"] + assert events[-1].payload == {"reason": "client_request"} + + +def test_cancel_kills_descendants_in_the_worker_process_group(tmp_path: Path) -> None: + abort_event = threading.Event() + events: list[WorkerEvent] = [] + pid_file = tmp_path / "descendant-pid" + + def emit(event: WorkerEvent) -> None: + events.append(event) + if event.type == "progress": + abort_event.set() + + _runner(_descendant_ignores_cancel_target, join_timeout=0.05).run( + {"task_id": "descendant-cancel", "pid_file": str(pid_file)}, + emit, + abort_event, + ) + + descendant_pid = int(pid_file.read_text(encoding="utf-8")) + deadline = time.monotonic() + 2 + while time.monotonic() < deadline: + if not psutil.pid_exists(descendant_pid): + break + try: + if psutil.Process(descendant_pid).status() == psutil.STATUS_ZOMBIE: + break + except psutil.NoSuchProcess: + break + time.sleep(0.01) + else: + raise AssertionError("worker descendant survived cancellation") + + assert events[-1].type == "cancelled" + + +@pytest.mark.skipif( + not hasattr(os, "setsid") + or not hasattr(os, "getpgid") + or not hasattr(os, "killpg") + or not hasattr(signal, "SIGKILL"), + reason="POSIX process groups are required", +) +def test_parent_loss_kills_worker_and_descendant_process_group(tmp_path: Path) -> None: + pid_file = tmp_path / "parent-loss-pids.json" + command = ( + "from tests.tools.executor.test_runner import _run_parent_loss_service; " + "import sys; _run_parent_loss_service(sys.argv[1])" + ) + outer = subprocess.Popen( # noqa: S603 + [sys.executable, "-c", command, str(pid_file)], + cwd=Path(__file__).resolve().parents[3], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + process_handles: list[psutil.Process] = [] + worker_pid: int | None = None + try: + deadline = time.monotonic() + 10 + process_ids: dict[str, int] | None = None + while time.monotonic() < deadline: + if outer.poll() is not None: + raise AssertionError( + f"service-like parent exited before worker startup: {outer.returncode}" + ) + try: + loaded = json.loads(pid_file.read_text(encoding="utf-8")) + process_ids = { + "leader": int(loaded["leader"]), + "descendant": int(loaded["descendant"]), + } + break + except (FileNotFoundError, json.JSONDecodeError, KeyError, ValueError): + time.sleep(0.01) + if process_ids is None: + raise AssertionError("worker process group did not become ready") + + worker_pid = process_ids["leader"] + process_handles = [ + psutil.Process(worker_pid), + psutil.Process(process_ids["descendant"]), + ] + assert os.getpgid(worker_pid) == worker_pid + + outer.send_signal(signal.SIGKILL) + outer.wait(timeout=5) + + deadline = time.monotonic() + 8 + while time.monotonic() < deadline: + if all(_process_is_gone_or_zombie(process) for process in process_handles): + break + time.sleep(0.02) + else: + surviving = [ + process.pid + for process in process_handles + if not _process_is_gone_or_zombie(process) + ] + raise AssertionError( + f"worker process group survived parent loss: {surviving}" + ) + finally: + cleanup_handles = list(process_handles) + if outer.poll() is None: + try: + cleanup_handles.extend( + psutil.Process(outer.pid).children(recursive=True) + ) + except psutil.NoSuchProcess: + pass + outer.kill() + try: + outer.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + if worker_pid is not None: + try: + if os.getpgid(worker_pid) == worker_pid: + os.killpg(worker_pid, signal.SIGKILL) + except ProcessLookupError: + pass + for process in {process.pid: process for process in cleanup_handles}.values(): + try: + if process.is_running() and process.status() != psutil.STATUS_ZOMBIE: + process.kill() + except (psutil.NoSuchProcess, psutil.ZombieProcess): + pass + + +def _process_is_gone_or_zombie(process: psutil.Process) -> bool: + try: + return not process.is_running() or process.status() == psutil.STATUS_ZOMBIE + except (psutil.NoSuchProcess, psutil.ZombieProcess): + return True + + +def test_worker_event_coercion_rejects_invalid_payload() -> None: + invalid: dict[str, Any] = {"type": "progress", "payload": "not-an-object"} + + try: + MultiprocessExecutionRunner._coerce_event(invalid) + except ValueError as error: + assert str(error) == "subprocess emitted an invalid executor event" + else: + raise AssertionError("invalid worker event was accepted") diff --git a/tests/tools/executor/test_server.py b/tests/tools/executor/test_server.py new file mode 100644 index 00000000..b0be845f --- /dev/null +++ b/tests/tools/executor/test_server.py @@ -0,0 +1,1190 @@ +from __future__ import annotations + +import http.client +import json +import os +import secrets +import signal +import subprocess +import sys +import threading +import time +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from urllib.parse import urlparse + +import pymupdf +import pytest +from babeldoc.tools.executor import server as executor_server +from babeldoc.tools.executor.protocol import EventEnvelope +from babeldoc.tools.executor.runner import FakeExecutionRunner +from babeldoc.tools.executor.server import ALLOW_FAKE_RUNNER_ENV +from babeldoc.tools.executor.server import MAX_JSON_BODY_BYTES +from babeldoc.tools.executor.server import READY_PREFIX +from babeldoc.tools.executor.server import ExecutorHandler +from babeldoc.tools.executor.server import ExecutorServer +from babeldoc.tools.executor.server import _configure_workroot +from babeldoc.tools.executor.server import _is_loopback_host +from babeldoc.tools.executor.server import _load_token +from babeldoc.tools.executor.server import _resolve_parent_identity +from babeldoc.tools.executor.state import CursorAheadError +from babeldoc.tools.executor.state import ExecutionNotFoundError +from babeldoc.tools.executor.state import ExecutionStore +from babeldoc.tools.executor.state import ReplayGapError +from babeldoc.tools.executor.workroot import WORKROOT_ENV +from babeldoc.tools.executor.workroot import WORKROOT_READY_FILE + +DEFAULT_SERVER_TOKEN = object() + + +def prepare_private_workroot(path: Path) -> None: + path.chmod(0o700) + marker = path / WORKROOT_READY_FILE + marker.write_text("ready\n", encoding="utf-8") + marker.chmod(0o600) + + +def write_blank_pdf(path: Path, *, width: float = 200, height: float = 200) -> None: + document = pymupdf.open() + document.new_page(width=width, height=height) + document.save(path) + document.close() + + +class StubbornWatermarkProcess: + pid = 987_654_321 + + def __init__(self) -> None: + self.exitcode: int | None = None + self.started = False + self.terminated = False + self.killed = False + self.closed = False + + def start(self) -> None: + self.started = True + + def is_alive(self) -> bool: + return self.exitcode is None + + def join(self, timeout: float | None = None) -> None: + _ = timeout + + def terminate(self) -> None: + self.terminated = True + + def kill(self) -> None: + self.killed = True + self.exitcode = -9 + + def close(self) -> None: + self.closed = True + + +class WatermarkProcessContext: + def __init__(self, process: StubbornWatermarkProcess) -> None: + self.process = process + self.target = None + self.args = None + + def Process(self, *, target, args): # noqa: N802 + self.target = target + self.args = args + return self.process + + +class GapDuringStreamStore: + def replay(self, _execution_id: str, _after_sequence: int) -> list: + return [] + + def stream(self, execution_id: str, _after_sequence: int): + yield EventEnvelope( + type="progress", + execution_id=execution_id, + sequence=11, + emitted_at=time.time(), + payload={"index": 1}, + ) + raise ReplayGapError + + def snapshot(self, execution_id: str) -> dict: + return { + "execution_id": execution_id, + "task_id": "gap-task", + "status": "running", + "initial_sequence": 10, + "first_available_sequence": 15, + "last_sequence": 16, + "worker_finished": False, + "created_at": time.time(), + "finished_at": None, + } + + +class EvictedDuringStreamStore: + def replay(self, _execution_id: str, _after_sequence: int) -> list: + return [] + + def stream(self, execution_id: str, _after_sequence: int): + yield EventEnvelope( + type="progress", + execution_id=execution_id, + sequence=11, + emitted_at=time.time(), + payload={"index": 1}, + ) + raise ExecutionNotFoundError(execution_id) + + def snapshot(self, execution_id: str) -> dict: + raise ExecutionNotFoundError(execution_id) + + +class CursorAheadThenEvictedStore: + def replay(self, execution_id: str, _after_sequence: int) -> list: + raise CursorAheadError(execution_id) + + def snapshot(self, execution_id: str) -> dict: + raise ExecutionNotFoundError(execution_id) + + +class PausingCreateStore(ExecutionStore): + def __init__(self) -> None: + super().__init__(FakeExecutionRunner()) + self.create_entered = threading.Event() + self.allow_create = threading.Event() + + def create(self, request: dict) -> dict: + self.create_entered.set() + if not self.allow_create.wait(timeout=2): + raise TimeoutError("test did not release create") + return super().create(request) + + +class NeverIdleStore(ExecutionStore): + def __init__(self) -> None: + super().__init__(FakeExecutionRunner()) + self.drain_wait_started = threading.Event() + self.release_drain_wait = threading.Event() + self.wait_timeouts: list[float | None] = [] + self.abort_reasons: list[str] = [] + + def abort_current(self, *, reason: str = "client_request") -> None: + self.abort_reasons.append(reason) + super().abort_current(reason=reason) + + def wait_until_idle(self, timeout_seconds: float | None = None) -> bool: + self.wait_timeouts.append(timeout_seconds) + if timeout_seconds == 5.0: + return False + self.drain_wait_started.set() + self.release_drain_wait.wait(timeout=2) + return False + + +class ClearsWorkrootEnvironmentOnBeginStore(ExecutionStore): + def begin_heavy_operation(self, operation_id: str) -> threading.Event: + abort_event = super().begin_heavy_operation(operation_id) + os.environ.pop(WORKROOT_ENV, None) + return abort_event + + +@contextmanager +def running_server( + tmp_path: Path, + *, + max_event_log_size: int = 1000, + execution_store=None, + workroot: Path | None = None, + parent_pid: int | None = None, + parent_start_time: float | None = None, +) -> Iterator[tuple[ExecutorServer, threading.Thread]]: + previous_workroot = os.environ.get(WORKROOT_ENV) + os.environ[WORKROOT_ENV] = str(tmp_path) + store = execution_store or ExecutionStore( + FakeExecutionRunner(), max_event_log_size=max_event_log_size + ) + server = ExecutorServer( + ("127.0.0.1", 0), + store, + token=secrets.token_urlsafe(24), + instance_id="test-instance", + workroot=workroot, + parent_pid=parent_pid, + parent_start_time=parent_start_time, + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server, thread + finally: + if thread.is_alive(): + server.shutdown() + server.server_close() + thread.join(timeout=2) + if previous_workroot is None: + os.environ.pop(WORKROOT_ENV, None) + else: + os.environ[WORKROOT_ENV] = previous_workroot + + +def request( + server: ExecutorServer, + method: str, + path: str, + *, + body: object | None = None, + token: str | None | object = DEFAULT_SERVER_TOKEN, + headers: dict[str, str] | None = None, +) -> tuple[int, dict[str, str], bytes]: + connection = http.client.HTTPConnection(*server.server_address[:2], timeout=3) + request_headers = dict(headers or {}) + if token is DEFAULT_SERVER_TOKEN: + token = server.token + if isinstance(token, str): + request_headers["Authorization"] = f"Bearer {token}" + encoded: bytes | None = None + if isinstance(body, bytes): + encoded = body + request_headers["Content-Type"] = "application/json" + elif body is not None: + encoded = json.dumps(body).encode("utf-8") + request_headers["Content-Type"] = "application/json" + connection.request(method, path, body=encoded, headers=request_headers) + response = connection.getresponse() + data = response.read() + result_headers = {name.lower(): value for name, value in response.getheaders()} + status = response.status + connection.close() + return status, result_headers, data + + +def json_request(*args, **kwargs) -> tuple[int, dict[str, str], dict]: + status, headers, body = request(*args, **kwargs) + return status, headers, json.loads(body) + + +def test_requires_bearer_token_and_reports_service_identity(tmp_path: Path) -> None: + with running_server(tmp_path) as (server, _thread): + status, headers, payload = json_request( + server, + "GET", + "/healthz", + token=None, + ) + assert status == 401 + assert headers["www-authenticate"] == "Bearer" + assert payload["code"] == "unauthorized" + + status, _headers, payload = json_request(server, "GET", "/healthz") + assert status == 200 + assert payload["ok"] is True + assert payload["service_id"] == "gloss-babeldoc" + assert payload["instance_id"] == "test-instance" + assert payload["endpoint"].endswith(f":{server.server_port}") + + status, _headers, payload = json_request( + server, + "GET", + "/healthz", + token="é" * 40, + ) + assert status == 401 + assert payload["code"] == "unauthorized" + + +def test_runtime_endpoint_exposes_protocol_capabilities(tmp_path: Path) -> None: + with running_server(tmp_path) as (server, _thread): + status, _headers, payload = json_request(server, "GET", "/v1/runtime") + + assert status == 200 + assert payload["service"]["instance_id"] == "test-instance" + assert "executor.http.v1" in payload["capabilities"] + assert "executor.events.ndjson.v1" in payload["capabilities"] + + +def test_create_snapshot_stream_and_idempotent_replay(tmp_path: Path) -> None: + with running_server(tmp_path) as (server, _thread): + execution_request = {"task_id": "task-1", "mode": "burst"} + status, _headers, created = json_request( + server, + "POST", + "/v1/executions", + body=execution_request, + ) + assert status == 201 + assert created["replayed"] is False + execution_id = created["execution_id"] + + status, _headers, replayed = json_request( + server, + "POST", + "/v1/executions", + body=execution_request, + ) + assert status == 200 + assert replayed["replayed"] is True + assert replayed["execution_id"] == execution_id + + status, _headers, conflict = json_request( + server, + "POST", + "/v1/executions", + body={"task_id": "task-1", "mode": "different"}, + ) + assert status == 409 + assert conflict["code"] == "idempotency_conflict" + assert conflict["snapshot"]["execution_id"] == execution_id + + status, _headers, latest = json_request( + server, + "GET", + "/v1/executions/latest", + ) + assert status == 200 + assert latest["execution"]["execution_id"] == execution_id + + status, _headers, snapshot = json_request( + server, + "GET", + f"/v1/executions/{execution_id}", + ) + assert status == 200 + assert snapshot["task_id"] == "task-1" + + status, headers, body = request( + server, + "GET", + f"/v1/executions/{execution_id}/events" + f"?after_sequence={created['initial_sequence']}", + ) + events = [json.loads(line) for line in body.splitlines()] + assert status == 200 + assert headers["content-type"] == "application/x-ndjson" + assert [event["type"] for event in events] == [ + "progress", + "progress", + "result", + ] + assert all(event["execution_id"] == execution_id for event in events) + assert all(event["schema_version"] == 1 for event in events) + assert all(event["service_id"] == "gloss-babeldoc" for event in events) + assert all(event["instance_id"] == "test-instance" for event in events) + assert all(isinstance(event["emitted_at"], float) for event in events) + + status, _headers, current = json_request( + server, + "GET", + "/v1/executions/current", + ) + assert status == 200 + assert current["execution"] is None + + status, _headers, payload = json_request( + server, + "GET", + f"/v1/executions/{execution_id}/events" + f"?after_sequence={events[-1]['sequence'] + 1}", + ) + assert status == 409 + assert payload["code"] == "cursor_ahead" + assert payload["snapshot"]["execution_id"] == execution_id + + +def test_replay_gap_includes_authoritative_snapshot(tmp_path: Path) -> None: + with running_server(tmp_path, max_event_log_size=1) as (server, _thread): + status, _headers, created = json_request( + server, + "POST", + "/v1/executions", + body={"task_id": "task-gap", "mode": "burst"}, + ) + assert status == 201 + execution_id = created["execution_id"] + + deadline = time.monotonic() + 2 + while time.monotonic() < deadline: + _, _, snapshot = json_request( + server, + "GET", + f"/v1/executions/{execution_id}", + ) + if snapshot["status"] == "succeeded": + break + time.sleep(0.01) + + status, _headers, payload = json_request( + server, + "GET", + f"/v1/executions/{execution_id}/events" + f"?after_sequence={created['initial_sequence']}", + ) + + assert status == 410 + assert payload["code"] == "replay_gap" + assert payload["snapshot"]["execution_id"] == execution_id + assert payload["snapshot"]["status"] == "succeeded" + + +def test_stream_gap_emits_an_unsequenced_control_record(tmp_path: Path) -> None: + with running_server( + tmp_path, + execution_store=GapDuringStreamStore(), + ) as (server, _thread): + status, _headers, body = request( + server, + "GET", + "/v1/executions/gap-execution/events?after_sequence=10", + ) + + records = [json.loads(line) for line in body.splitlines()] + assert status == 200 + assert [record["type"] for record in records] == ["progress", "stream_error"] + assert records[1]["sequence"] is None + assert records[1]["payload"]["code"] == "replay_gap" + assert records[1]["payload"]["after_sequence"] == 11 + + +def test_stream_eviction_emits_control_record_with_nullable_snapshot( + tmp_path: Path, +) -> None: + with running_server( + tmp_path, + execution_store=EvictedDuringStreamStore(), + ) as (server, _thread): + status, _headers, body = request( + server, + "GET", + "/v1/executions/evicted-execution/events?after_sequence=10", + ) + + records = [json.loads(line) for line in body.splitlines()] + assert status == 200 + assert [record["type"] for record in records] == ["progress", "stream_error"] + assert records[1]["sequence"] is None + assert records[1]["payload"]["code"] == "execution_not_found" + assert records[1]["payload"]["snapshot"] is None + + +def test_cursor_ahead_snapshot_may_be_evicted_concurrently(tmp_path: Path) -> None: + with running_server( + tmp_path, + execution_store=CursorAheadThenEvictedStore(), + ) as (server, _thread): + status, _headers, payload = json_request( + server, + "GET", + "/v1/executions/evicted-execution/events?after_sequence=10", + ) + + assert status == 409 + assert payload["code"] == "cursor_ahead" + assert payload["snapshot"] is None + + +def test_targeted_cancel_emits_terminal_event_and_shutdown_stops_server( + tmp_path: Path, +) -> None: + with running_server(tmp_path) as (server, thread): + status, _headers, created = json_request( + server, + "POST", + "/v1/executions", + body={"task_id": "task-block", "mode": "block"}, + ) + assert status == 201 + execution_id = created["execution_id"] + + status, _headers, current = json_request( + server, + "GET", + "/v1/executions/current", + ) + assert status == 200 + assert current["execution"]["execution_id"] == execution_id + + status, _headers, cancelling = json_request( + server, + "POST", + f"/v1/executions/{execution_id}/cancel", + body={}, + ) + assert status == 202 + assert cancelling["execution_id"] == execution_id + assert cancelling["status"] in {"cancelling", "cancelled"} + + status, _headers, body = request( + server, + "GET", + f"/v1/executions/{execution_id}/events" + f"?after_sequence={created['initial_sequence']}", + ) + events = [json.loads(line) for line in body.splitlines()] + assert status == 200 + assert events[-1]["type"] == "cancelled" + assert events[-1]["payload"]["reason"] == "client_request" + + status, _headers, payload = json_request( + server, + "POST", + "/v1/shutdown", + body={}, + ) + assert status == 202 + assert payload["status"] == "stopping" + thread.join(timeout=2) + assert not thread.is_alive() + + +def test_json_body_must_be_an_object_and_is_limited_to_one_mib(tmp_path: Path) -> None: + with running_server(tmp_path) as (server, _thread): + status, _headers, payload = json_request( + server, + "POST", + "/v1/executions", + body=["not", "an", "object"], + ) + assert status == 400 + assert payload["message"] == "json body must be an object" + + status, _headers, payload = json_request( + server, + "POST", + "/v1/executions", + headers={"Content-Length": str(MAX_JSON_BODY_BYTES + 1)}, + ) + assert status == 413 + assert payload["code"] == "invalid_request" + + +def test_json_parser_failures_return_a_structured_bad_request(tmp_path: Path) -> None: + oversized_integer = b'{"value":' + (b"9" * 5000) + b"}" + deeply_nested = b'{"value":' + (b"[" * 10_000) + b"0" + (b"]" * 10_000) + b"}" + cases = ( + ("/v1/shutdown", oversized_integer), + ("/v1/executions/missing/cancel", oversized_integer), + ("/v1/executions", deeply_nested), + ("/v1/pdf/watermark1", deeply_nested), + ) + + with running_server(tmp_path) as (server, thread): + for path, body in cases: + status, _headers, payload = json_request( + server, + "POST", + path, + body=body, + ) + assert status == 400 + assert payload == {"code": "invalid_request", "message": "invalid json"} + assert thread.is_alive() + + +def test_stopping_service_rejects_new_work(tmp_path: Path) -> None: + with running_server(tmp_path) as (server, _thread): + server._stopping.set() + + status, _headers, payload = json_request( + server, + "POST", + "/v1/executions", + body={"task_id": "too-late", "mode": "burst"}, + ) + + assert status == 503 + assert payload["code"] == "service_stopping" + assert server.store.current_snapshot() is None + + status, _headers, payload = json_request( + server, + "POST", + "/v1/abort", + body={}, + ) + assert status == 404 + assert payload["code"] == "not_found" + + +def test_shutdown_admission_gate_cancels_a_concurrent_create(tmp_path: Path) -> None: + store = PausingCreateStore() + with running_server(tmp_path, execution_store=store) as (server, thread): + created: list[dict] = [] + create_thread = threading.Thread( + target=lambda: created.append( + server.admit_execution({"task_id": "racing-task", "mode": "block"}) + ) + ) + create_thread.start() + assert store.create_entered.wait(timeout=1) + + shutdown_thread = threading.Thread(target=server.request_shutdown) + shutdown_thread.start() + time.sleep(0.02) + assert shutdown_thread.is_alive() + + store.allow_create.set() + create_thread.join(timeout=2) + shutdown_thread.join(timeout=2) + assert created + assert store.wait_until_idle(timeout_seconds=2) + snapshot = store.snapshot(created[0]["execution_id"]) + assert snapshot["status"] == "cancelled" + thread.join(timeout=2) + assert not thread.is_alive() + + +def test_shutdown_escalation_bounds_an_existing_graceful_drain( + tmp_path: Path, +) -> None: + store = NeverIdleStore() + with running_server(tmp_path, execution_store=store) as (server, thread): + assert server.request_shutdown(cancel_active=False) is True + assert store.drain_wait_started.wait(timeout=1) + + assert server.request_shutdown(cancel_active=True) is False + store.release_drain_wait.set() + + thread.join(timeout=2) + assert not thread.is_alive() + assert store.wait_timeouts[-1] == 5.0 + + +def test_parent_loss_escalates_an_existing_graceful_drain( + monkeypatch, + tmp_path: Path, +) -> None: + parent_alive = threading.Event() + parent_alive.set() + monkeypatch.setattr( + "babeldoc.tools.executor.server.PARENT_WATCHDOG_INTERVAL_SECONDS", + 0.01, + ) + monkeypatch.setattr( + "babeldoc.tools.executor.server._process_matches", + lambda _process_id, _start_time: parent_alive.is_set(), + ) + store = NeverIdleStore() + with running_server( + tmp_path, + execution_store=store, + parent_pid=123, + parent_start_time=456.0, + ) as (server, thread): + server.start_parent_watchdog() + assert server.request_shutdown(cancel_active=False) is True + assert store.drain_wait_started.wait(timeout=1) + + parent_alive.clear() + deadline = time.monotonic() + 1 + while "parent_exit" not in store.abort_reasons and time.monotonic() < deadline: + time.sleep(0.01) + assert "parent_exit" in store.abort_reasons + store.release_drain_wait.set() + + thread.join(timeout=2) + assert not thread.is_alive() + assert store.wait_timeouts[-1] == 5.0 + assert server._watchdog is not None + server._watchdog.join(timeout=1) + assert not server._watchdog.is_alive() + + +def test_service_rejects_non_loopback_hosts() -> None: + unspecified_address = ".".join(["0", "0", "0", "0"]) + assert _is_loopback_host("127.0.0.1") + assert not _is_loopback_host("localhost") + assert not _is_loopback_host("::1") + assert not _is_loopback_host(unspecified_address) + assert not _is_loopback_host("example.com") + + +def test_token_file_is_used_without_ready_disclosure(tmp_path: Path) -> None: + token_file = tmp_path / "token" + token_value = secrets.token_urlsafe(32) + token_file.write_text(f"{token_value}\n", encoding="utf-8") + token_file.chmod(0o600) + + assert _load_token(token_file) == (token_value, False) + generated, should_disclose = _load_token(None) + assert should_disclose is True + assert len(generated) >= 32 + + +def test_token_file_fails_closed_when_missing_or_insecure(tmp_path: Path) -> None: + token_file = tmp_path / "token" + token_file.write_text("x" * 40, encoding="utf-8") + token_file.chmod(0o644) + + try: + _load_token(token_file) + except ValueError as error: + assert "permissions" in str(error) + else: + raise AssertionError("insecure token file was accepted") + + try: + _load_token(tmp_path / "missing-token") + except ValueError as error: + assert "unable to read" in str(error) + else: + raise AssertionError("missing token file generated a replacement token") + + +def test_configure_workroot_requires_an_existing_directory( + monkeypatch, + tmp_path: Path, +) -> None: + monkeypatch.delenv(WORKROOT_ENV, raising=False) + prepare_private_workroot(tmp_path) + assert _configure_workroot(tmp_path, required=True) == tmp_path.resolve() + assert os.environ[WORKROOT_ENV] == str(tmp_path.resolve()) + + try: + _configure_workroot(tmp_path / "missing", required=True) + except ValueError as error: + assert "existing directory" in str(error) + else: + raise AssertionError("missing workroot was accepted") + + +def test_configure_workroot_requires_private_marker( + monkeypatch, + tmp_path: Path, +) -> None: + private_root = tmp_path / "private-root" + private_root.mkdir(mode=0o700) + monkeypatch.delenv(WORKROOT_ENV, raising=False) + + try: + _configure_workroot(private_root, required=True) + except ValueError as error: + assert "readiness proof" in str(error) + else: + raise AssertionError("workroot without readiness proof was accepted") + + +def test_health_degrades_when_workroot_disappears(tmp_path: Path) -> None: + workroot = tmp_path / "runtime" + workroot.mkdir(mode=0o700) + prepare_private_workroot(workroot) + store = ExecutionStore(FakeExecutionRunner()) + with running_server( + tmp_path, + execution_store=store, + workroot=workroot, + ) as (server, _thread): + moved = tmp_path / "runtime-moved" + workroot.rename(moved) + status, _headers, payload = json_request(server, "GET", "/healthz") + + assert status == 503 + assert payload["ok"] is False + assert payload["code"] == "workroot_unavailable" + + +def test_watermark_dispatch_calls_only_the_fixed_transform_operations( + monkeypatch, + tmp_path: Path, +) -> None: + calls: list[tuple] = [] + + def record_tiled(input_file: Path, output_file: Path, asset_file: Path) -> None: + calls.append(("watermark1", input_file, output_file, asset_file)) + + def record_corner( + input_file: Path, + output_file: Path, + black_asset_file: Path, + white_asset_file: Path, + ) -> None: + calls.append( + ( + "watermark2", + input_file, + output_file, + black_asset_file, + white_asset_file, + ) + ) + + monkeypatch.setattr( + "babeldoc.tools.executor.watermark_transform.add_tiled_watermark", + record_tiled, + ) + monkeypatch.setattr( + "babeldoc.tools.executor.watermark_transform.add_corner_watermark", + record_corner, + ) + input_file = tmp_path / "input.pdf" + output_file = tmp_path / "output.pdf" + black_asset = tmp_path / "black.pdf" + white_asset = tmp_path / "white.pdf" + + executor_server._dispatch_watermark_operation( + "watermark1", + input_file, + output_file, + (black_asset,), + ) + executor_server._dispatch_watermark_operation( + "watermark2", + input_file, + output_file, + (black_asset, white_asset), + ) + + assert calls == [ + ("watermark1", input_file, output_file, black_asset), + ("watermark2", input_file, output_file, black_asset, white_asset), + ] + + +def test_watermark_dispatch_rejects_unknown_operation(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="unsupported watermark operation"): + executor_server._dispatch_watermark_operation( + "user-provided-command", + tmp_path / "input.pdf", + tmp_path / "output.pdf", + (), + ) + + +def test_watermark_rejects_output_that_overwrites_an_asset(tmp_path: Path) -> None: + input_file = tmp_path / "input.pdf" + asset_file = tmp_path / "asset.pdf" + input_file.touch() + asset_file.touch() + + with pytest.raises(ValueError, match="must not overwrite an asset file"): + ExecutorHandler._validate_watermark_request( + tmp_path, + "watermark1", + { + "operation_id": "asset-overwrite", + "input_file": input_file.name, + "output_file": asset_file.name, + "asset_file": asset_file.name, + }, + ) + + +def test_watermark_process_closes_when_start_fails( + monkeypatch, + tmp_path: Path, +) -> None: + process = StubbornWatermarkProcess() + context = WatermarkProcessContext(process) + + def fail_start() -> None: + raise RuntimeError("spawn failed") + + monkeypatch.setattr(process, "start", fail_start) + monkeypatch.setattr( + executor_server.multiprocessing, + "get_context", + lambda _start_method: context, + ) + + with pytest.raises(RuntimeError, match="spawn failed"): + ExecutorHandler._run_watermark_process( + "watermark1", + tmp_path / "input.pdf", + tmp_path / "output.pdf", + [tmp_path / "asset.pdf"], + threading.Event(), + ) + + assert process.closed is True + + +@pytest.mark.parametrize( + ("exit_code", "expected_message"), + [(1, "transform failed"), (0, "did not create output")], +) +def test_watermark_process_validates_natural_exit_and_closes( + monkeypatch, + tmp_path: Path, + exit_code: int, + expected_message: str, +) -> None: + process = StubbornWatermarkProcess() + process.exitcode = exit_code + context = WatermarkProcessContext(process) + monkeypatch.setattr( + executor_server.multiprocessing, + "get_context", + lambda _start_method: context, + ) + + with pytest.raises(RuntimeError, match=expected_message): + ExecutorHandler._run_watermark_process( + "watermark1", + tmp_path / "input.pdf", + tmp_path / "missing-output.pdf", + [tmp_path / "asset.pdf"], + threading.Event(), + ) + + assert process.closed is True + + +@pytest.mark.parametrize( + ("abort_requested", "expected_message"), + [(True, "aborted"), (False, "timed out")], +) +def test_watermark_cancel_and_timeout_escalate_and_close_process( + monkeypatch, + tmp_path: Path, + abort_requested: bool, + expected_message: str, +) -> None: + process = StubbornWatermarkProcess() + context = WatermarkProcessContext(process) + requested_start_methods: list[str] = [] + + def get_context(start_method: str): + requested_start_methods.append(start_method) + return context + + monkeypatch.setattr(executor_server.multiprocessing, "get_context", get_context) + monkeypatch.setattr(executor_server, "WATERMARK_POLL_SECONDS", 0) + monkeypatch.setattr(executor_server, "WATERMARK_TIMEOUT_SECONDS", 0) + monkeypatch.setattr( + executor_server, + "WATERMARK_TERMINATE_TIMEOUT_SECONDS", + 0, + ) + abort_event = threading.Event() + if abort_requested: + abort_event.set() + + with pytest.raises(TimeoutError, match=expected_message): + ExecutorHandler._run_watermark_process( + "watermark1", + tmp_path / "input.pdf", + tmp_path / "output.pdf", + [tmp_path / "asset.pdf"], + abort_event, + ) + + assert requested_start_methods == ["spawn"] + assert context.target is executor_server._run_watermark_process_target + assert process.started is True + assert process.terminated is True + assert process.killed is True + assert process.closed is True + + +def test_watermark_transform_runs_in_a_spawned_process(tmp_path: Path) -> None: + input_file = tmp_path / "input.pdf" + black_asset = tmp_path / "black.pdf" + white_asset = tmp_path / "white.pdf" + output_file = tmp_path / "output.pdf" + write_blank_pdf(input_file) + write_blank_pdf(black_asset, width=20, height=10) + write_blank_pdf(white_asset, width=20, height=10) + + ExecutorHandler._run_watermark_process( + "watermark2", + input_file, + output_file, + [black_asset, white_asset], + threading.Event(), + ) + + output_document = pymupdf.open(output_file) + try: + assert len(output_document) == 1 + finally: + output_document.close() + + +def test_watermark_uses_server_workroot_and_finishes_after_begin( + monkeypatch, + tmp_path: Path, +) -> None: + input_file = tmp_path / "input.pdf" + asset_file = tmp_path / "watermark.pdf" + input_file.write_bytes(b"%PDF-1.4\ninput") + asset_file.write_bytes(b"%PDF-1.4\nasset") + store = ClearsWorkrootEnvironmentOnBeginStore() + + def write_output( + _operation: str, + _input_file: Path, + output_file: Path, + _asset_files: list[Path], + _abort_event, + ) -> None: + output_file.write_bytes(b"%PDF-1.4\noutput") + + monkeypatch.setattr( + ExecutorHandler, + "_run_watermark_process", + staticmethod(write_output), + ) + with running_server( + tmp_path, + execution_store=store, + workroot=tmp_path, + ) as (server, _thread): + status, _headers, payload = json_request( + server, + "POST", + "/v1/pdf/watermark1", + body={ + "operation_id": "watermark-stable-root", + "input_file": input_file.name, + "output_file": "output/result.pdf", + "asset_file": asset_file.name, + }, + ) + + snapshot = store.snapshot("watermark-stable-root") + assert status == 200 + assert payload["output_file"] == "output/result.pdf" + assert snapshot["status"] == "succeeded" + assert snapshot["worker_finished"] is True + assert (tmp_path / "output/result.pdf").is_file() + + +def test_parent_identity_is_captured_and_mismatch_is_rejected() -> None: + parent_pid, parent_start_time = _resolve_parent_identity(os.getpid(), None) + assert parent_pid == os.getpid() + assert isinstance(parent_start_time, float) + + try: + _resolve_parent_identity(os.getpid(), 0) + except ValueError as error: + assert "does not match" in str(error) + else: + raise AssertionError("stale parent identity was accepted") + + +def test_parent_identity_mismatch_stops_service() -> None: + server = ExecutorServer( + ("127.0.0.1", 0), + ExecutionStore(FakeExecutionRunner()), + token=secrets.token_urlsafe(24), + parent_pid=os.getpid(), + parent_start_time=0, + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + server.start_parent_watchdog() + thread.join(timeout=3) + assert not thread.is_alive() + finally: + if thread.is_alive(): + server.shutdown() + server.server_close() + thread.join(timeout=2) + + +def test_gloss_cli_serve_emits_generated_token_in_ready_handshake( + tmp_path: Path, +) -> None: + environment = os.environ.copy() + environment[WORKROOT_ENV] = str(tmp_path) + environment[ALLOW_FAKE_RUNNER_ENV] = "1" + prepare_private_workroot(tmp_path) + process = subprocess.Popen( # noqa: S603 + [ + sys.executable, + "-m", + "babeldoc.gloss_cli", + "serve", + "--runner", + "fake", + "--instance-id", + "subprocess-instance", + ], + cwd=Path(__file__).resolve().parents[3], + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + assert process.stdout is not None + ready_line = process.stdout.readline().strip() + assert ready_line.startswith(READY_PREFIX) + ready = json.loads(ready_line.removeprefix(READY_PREFIX)) + assert ready["instance_id"] == "subprocess-instance" + assert ready["auth_token"] + + parsed = urlparse(ready["endpoint"]) + connection = http.client.HTTPConnection( + parsed.hostname, + parsed.port, + timeout=3, + ) + connection.request( + "POST", + "/v1/shutdown", + body=b"{}", + headers={ + "Authorization": f"Bearer {ready['auth_token']}", + "Content-Type": "application/json", + }, + ) + response = connection.getresponse() + response.read() + connection.close() + assert response.status == 202 + assert process.wait(timeout=5) == 0 + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + + +def test_sigterm_performs_controlled_service_shutdown(tmp_path: Path) -> None: + environment = os.environ.copy() + environment[WORKROOT_ENV] = str(tmp_path) + environment[ALLOW_FAKE_RUNNER_ENV] = "1" + prepare_private_workroot(tmp_path) + process = subprocess.Popen( # noqa: S603 + [ + sys.executable, + "-m", + "babeldoc.gloss_cli", + "serve", + "--runner", + "fake", + ], + cwd=Path(__file__).resolve().parents[3], + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + assert process.stdout is not None + ready_line = process.stdout.readline().strip() + ready = json.loads(ready_line.removeprefix(READY_PREFIX)) + parsed = urlparse(ready["endpoint"]) + connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=3) + connection.request( + "POST", + "/v1/executions", + body=json.dumps({"task_id": "signal-task", "mode": "block"}), + headers={ + "Authorization": f"Bearer {ready['auth_token']}", + "Content-Type": "application/json", + }, + ) + response = connection.getresponse() + response.read() + connection.close() + assert response.status == 201 + + process.send_signal(signal.SIGTERM) + return_code = process.wait(timeout=5) + stderr = process.stderr.read() if process.stderr is not None else "" + assert return_code == 0, stderr + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) diff --git a/tests/tools/executor/test_state.py b/tests/tools/executor/test_state.py new file mode 100644 index 00000000..085ac921 --- /dev/null +++ b/tests/tools/executor/test_state.py @@ -0,0 +1,383 @@ +from __future__ import annotations + +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from typing import Any + +import pytest +from babeldoc.tools.executor.protocol import WorkerEvent +from babeldoc.tools.executor.runner import ExecutionRunner +from babeldoc.tools.executor.state import CursorAheadError +from babeldoc.tools.executor.state import ExecutionBusyError +from babeldoc.tools.executor.state import ExecutionConflictError +from babeldoc.tools.executor.state import ExecutionNotFoundError +from babeldoc.tools.executor.state import ExecutionStore +from babeldoc.tools.executor.state import ReplayGapError + + +def wait_until(predicate, timeout: float = 2.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.005) + assert predicate() + + +class BlockingRunner(ExecutionRunner): + def __init__(self) -> None: + self.started = threading.Event() + self.release = threading.Event() + self.cancel_seen = threading.Event() + self._lock = threading.Lock() + self.invocations = 0 + + def run(self, request, emit, abort_event) -> None: + with self._lock: + self.invocations += 1 + self.started.set() + if request.get("mode") == "result_then_block": + emit(WorkerEvent("result", {"files": {}})) + self.release.wait(timeout=2) + return + if request.get("mode") == "finish": + emit(WorkerEvent("result", {"files": {}})) + return + abort_event.wait(timeout=2) + if abort_event.is_set(): + self.cancel_seen.set() + self.release.wait(timeout=2) + + +class ImmediateEventRunner(ExecutionRunner): + def run(self, request, emit, abort_event) -> None: + for index in range(request.get("progress_events", 0)): + emit(WorkerEvent("progress", {"index": index})) + event_type = request.get("event_type", "result") + emit(WorkerEvent(event_type, {"code": event_type, "files": {}})) + + +def snapshot_is_finished(store: ExecutionStore, execution_id: str) -> bool: + return bool(store.snapshot(execution_id)["worker_finished"]) + + +def test_create_is_idempotent_for_same_task_and_canonical_request() -> None: + runner = BlockingRunner() + store = ExecutionStore(runner) + request = { + "task_id": "task-1", + "mode": "wait", + "nested": {"b": 2, "a": 1}, + } + + first = store.create(request) + assert runner.started.wait(timeout=1) + replay = store.create( + { + "nested": {"a": 1, "b": 2}, + "mode": "wait", + "task_id": "task-1", + } + ) + + assert first["execution_id"] == replay["execution_id"] + assert first["replayed"] is False + assert replay["replayed"] is True + assert replay["status"] == "running" + assert runner.invocations == 1 + + with pytest.raises(ExecutionConflictError) as conflict: + store.create({"task_id": "task-1", "mode": "different"}) + assert conflict.value.snapshot["execution_id"] == first["execution_id"] + + store.cancel(first["execution_id"]) + runner.release.set() + wait_until(lambda: snapshot_is_finished(store, first["execution_id"])) + + +def test_worker_start_failure_returns_replayable_failed_execution( + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = ExecutionStore(ImmediateEventRunner()) + + def fail_to_start(_record) -> None: + raise RuntimeError("thread limit reached") + + monkeypatch.setattr(store, "_start_worker", fail_to_start) + request = {"task_id": "worker-start-failure", "mode": "finish"} + + first = store.create(request) + snapshot = store.snapshot(first["execution_id"]) + events = store.replay(first["execution_id"], first["initial_sequence"]) + replay = store.create(request) + + assert first["status"] == "failed" + assert first["replayed"] is False + assert snapshot["status"] == "failed" + assert snapshot["worker_finished"] is True + assert store.active_snapshot() is None + assert store.wait_until_idle(timeout_seconds=0) is True + assert [event.type for event in events] == ["error"] + assert events[0].payload == { + "code": "worker_start_failed", + "message": "thread limit reached", + "message_for_user": None, + "details": {"exception_type": "RuntimeError"}, + } + assert replay == {**first, "replayed": True} + + +def test_terminal_event_does_not_release_single_active_worker_early() -> None: + runner = BlockingRunner() + store = ExecutionStore(runner) + first = store.create({"task_id": "first", "mode": "result_then_block"}) + assert runner.started.wait(timeout=1) + wait_until(lambda: store.snapshot(first["execution_id"])["status"] == "succeeded") + + snapshot = store.snapshot(first["execution_id"]) + assert snapshot["worker_finished"] is False + with pytest.raises(ExecutionBusyError) as busy: + store.create({"task_id": "second", "mode": "finish"}) + assert busy.value.snapshot["execution_id"] == first["execution_id"] + + runner.release.set() + wait_until(lambda: snapshot_is_finished(store, first["execution_id"])) + second = store.create({"task_id": "second", "mode": "finish"}) + wait_until(lambda: snapshot_is_finished(store, second["execution_id"])) + assert store.snapshot(second["execution_id"])["status"] == "succeeded" + + +def test_targeted_cancel_stays_busy_until_worker_exits_and_emits_terminal() -> None: + runner = BlockingRunner() + store = ExecutionStore(runner) + started = store.create({"task_id": "cancel-me", "mode": "wait"}) + execution_id = started["execution_id"] + assert runner.started.wait(timeout=1) + + cancelling = store.cancel(execution_id) + assert cancelling["status"] == "cancelling" + assert cancelling["worker_finished"] is False + assert runner.cancel_seen.wait(timeout=1) + assert store.replay(execution_id, started["initial_sequence"]) == [] + + with pytest.raises(ExecutionBusyError): + store.create({"task_id": "too-early", "mode": "finish"}) + with pytest.raises(ExecutionNotFoundError): + store.cancel("missing-execution") + + runner.release.set() + wait_until(lambda: snapshot_is_finished(store, execution_id)) + snapshot = store.snapshot(execution_id) + events = store.replay(execution_id, started["initial_sequence"]) + assert snapshot["status"] == "cancelled" + assert snapshot["worker_finished"] is True + assert [event.type for event in events] == ["cancelled"] + assert events[0].payload == { + "reason": "client_request", + "code": "cancelled", + "message": "execution cancelled", + "message_for_user": None, + "details": {}, + } + + repeated = store.cancel(execution_id) + assert repeated == snapshot + + +def test_wait_until_idle_tracks_worker_cleanup() -> None: + runner = BlockingRunner() + store = ExecutionStore(runner) + started = store.create({"task_id": "shutdown", "mode": "wait"}) + assert runner.started.wait(timeout=1) + + store.cancel(started["execution_id"]) + assert store.wait_until_idle(timeout_seconds=0.01) is False + with pytest.raises(ValueError): + store.wait_until_idle(timeout_seconds=-1) + + runner.release.set() + assert store.wait_until_idle(timeout_seconds=1) is True + assert store.snapshot(started["execution_id"])["worker_finished"] is True + + +@pytest.mark.parametrize( + ("event_type", "expected_status"), + [ + ("result", "succeeded"), + ("error", "failed"), + ("cancelled", "cancelled"), + ], +) +def test_terminal_events_map_to_stable_statuses( + event_type: str, + expected_status: str, +) -> None: + store = ExecutionStore(ImmediateEventRunner()) + started = store.create({"task_id": event_type, "event_type": event_type}) + wait_until(lambda: snapshot_is_finished(store, started["execution_id"])) + + snapshot = store.snapshot(started["execution_id"]) + events = store.replay(started["execution_id"], started["initial_sequence"]) + assert snapshot["status"] == expected_status + assert [event.type for event in events] == [event_type] + + +def test_old_execution_remains_available_after_new_execution() -> None: + store = ExecutionStore(ImmediateEventRunner()) + first = store.create({"task_id": "first"}) + wait_until(lambda: snapshot_is_finished(store, first["execution_id"])) + second = store.create({"task_id": "second"}) + wait_until(lambda: snapshot_is_finished(store, second["execution_id"])) + + assert store.snapshot(first["execution_id"])["status"] == "succeeded" + assert [ + event.type + for event in store.replay(first["execution_id"], first["initial_sequence"]) + ] == ["result"] + assert store.current_snapshot() == store.snapshot(second["execution_id"]) + + +def test_store_retains_sixteen_most_recent_executions() -> None: + store = ExecutionStore(ImmediateEventRunner()) + executions: list[dict[str, Any]] = [] + for index in range(17): + execution = store.create({"task_id": f"task-{index}"}) + execution_id = execution["execution_id"] + wait_until( + lambda execution_id=execution_id: snapshot_is_finished(store, execution_id) + ) + executions.append(execution) + + with pytest.raises(ExecutionNotFoundError): + store.snapshot(executions[0]["execution_id"]) + for execution in executions[1:]: + assert store.snapshot(execution["execution_id"])["status"] == "succeeded" + + +def test_replay_gap_reports_trimmed_event_history() -> None: + store = ExecutionStore(ImmediateEventRunner(), max_event_log_size=2) + started = store.create( + {"task_id": "bursty", "progress_events": 3, "event_type": "result"} + ) + execution_id = started["execution_id"] + wait_until(lambda: snapshot_is_finished(store, execution_id)) + + with pytest.raises(ReplayGapError): + store.replay(execution_id, started["initial_sequence"]) + + snapshot = store.snapshot(execution_id) + first_available = snapshot["first_available_sequence"] + assert isinstance(first_available, int) + events = store.replay(execution_id, first_available - 1) + assert [event.type for event in events] == ["progress", "result"] + + +def test_future_cursor_is_rejected_and_snapshots_have_timestamps() -> None: + store = ExecutionStore(ImmediateEventRunner()) + started = store.create({"task_id": "future-cursor"}) + execution_id = started["execution_id"] + wait_until(lambda: snapshot_is_finished(store, execution_id)) + + snapshot = store.snapshot(execution_id) + assert isinstance(snapshot["created_at"], float) + assert isinstance(snapshot["finished_at"], float) + assert snapshot["finished_at"] >= snapshot["created_at"] + with pytest.raises(CursorAheadError): + store.replay(execution_id, snapshot["last_sequence"] + 1) + + +def test_finished_record_keeps_only_request_fingerprint() -> None: + store = ExecutionStore(ImmediateEventRunner()) + started = store.create( + { + "task_id": "redact-request", + "gateways": {"main_llm": {"api_key": "should-not-remain"}}, + } + ) + execution_id = started["execution_id"] + wait_until(lambda: snapshot_is_finished(store, execution_id)) + + record = store._records[execution_id] + assert record.request == {} + assert len(record.request_fingerprint) == 64 + + +def test_concurrent_create_allows_exactly_one_active_execution() -> None: + runner = BlockingRunner() + store = ExecutionStore(runner) + barrier = threading.Barrier(8) + + def create(index: int) -> tuple[str, str]: + barrier.wait(timeout=1) + try: + response = store.create({"task_id": f"task-{index}", "mode": "wait"}) + return "created", response["execution_id"] + except ExecutionBusyError as exc: + return "busy", exc.snapshot["execution_id"] + + with ThreadPoolExecutor(max_workers=8) as executor: + outcomes = list(executor.map(create, range(8))) + + created = [ + execution_id for outcome, execution_id in outcomes if outcome == "created" + ] + busy = [execution_id for outcome, execution_id in outcomes if outcome == "busy"] + assert len(created) == 1 + assert busy == [created[0]] * 7 + + store.cancel(created[0]) + runner.release.set() + wait_until(lambda: snapshot_is_finished(store, created[0])) + + +def test_watermark_heavy_operation_keeps_compatible_lifecycle() -> None: + store = ExecutionStore() + abort_event = store.begin_heavy_operation("watermark-1") + assert abort_event.is_set() is False + assert store.current_snapshot()["status"] == "running" + + with pytest.raises(ExecutionBusyError): + store.begin_heavy_operation("watermark-2") + + store.finish_heavy_operation("watermark-1") + snapshot = store.snapshot("watermark-1") + assert snapshot["status"] == "succeeded" + assert snapshot["worker_finished"] is True + assert [ + event.type + for event in store.replay("watermark-1", snapshot["initial_sequence"]) + ][-1] == "result" + + +def test_abort_current_cancels_active_heavy_operation() -> None: + store = ExecutionStore() + abort_event = store.begin_heavy_operation("watermark-cancel") + store.abort_current() + assert abort_event.wait(timeout=1) + assert store.snapshot("watermark-cancel")["status"] == "cancelling" + + store.finish_heavy_operation("watermark-cancel") + snapshot = store.snapshot("watermark-cancel") + assert snapshot["status"] == "cancelled" + assert [ + event.type + for event in store.replay("watermark-cancel", snapshot["initial_sequence"]) + ][-1] == "cancelled" + + +def test_failed_heavy_operation_records_error_terminal() -> None: + store = ExecutionStore() + store.begin_heavy_operation("watermark-failed") + store.finish_heavy_operation( + "watermark-failed", + error_code="transform_failed", + error_message="watermark transform failed", + ) + + snapshot = store.snapshot("watermark-failed") + events = store.replay("watermark-failed", snapshot["initial_sequence"]) + assert snapshot["status"] == "failed" + assert snapshot["worker_finished"] is True + assert events[-1].type == "error" + assert events[-1].payload["code"] == "transform_failed" diff --git a/tests/tools/executor/test_workroot.py b/tests/tools/executor/test_workroot.py new file mode 100644 index 00000000..f0170ebc --- /dev/null +++ b/tests/tools/executor/test_workroot.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +from babeldoc.tools.executor import workroot as workroot_module +from babeldoc.tools.executor.workroot import WORKROOT_ENV +from babeldoc.tools.executor.workroot import get_workroot +from babeldoc.tools.executor.workroot import relative_to_workroot +from babeldoc.tools.executor.workroot import resolve_dir +from babeldoc.tools.executor.workroot import resolve_file +from babeldoc.tools.executor.workroot import resolve_inside_workroot + + +def test_get_workroot_returns_the_real_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + workroot = tmp_path / "workroot" + workroot.mkdir() + alias = tmp_path / "workroot-alias" + alias.symlink_to(workroot, target_is_directory=True) + monkeypatch.setenv(WORKROOT_ENV, str(alias)) + + assert get_workroot() == workroot + + +def test_get_workroot_rejects_empty_environment_value( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(WORKROOT_ENV, "") + + with pytest.raises(ValueError): + get_workroot() + + +def test_get_workroot_rejects_nul_environment_value( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + workroot_module.os, + "environ", + {WORKROOT_ENV: "bad\x00path"}, + ) + + with pytest.raises(ValueError, match="NUL byte"): + get_workroot() + + +def test_resolve_inside_workroot_accepts_contained_relative_and_absolute_paths( + tmp_path: Path, +) -> None: + workroot = tmp_path / "workroot" + nested = workroot / "nested" + nested.mkdir(parents=True) + target = nested / "document.pdf" + + assert resolve_inside_workroot(workroot, ".") == workroot + assert resolve_inside_workroot(workroot, "nested/document.pdf") == target + assert resolve_inside_workroot(workroot, str(target)) == target + assert ( + resolve_inside_workroot( + workroot, + "nested/../nested/document.pdf", + ) + == target + ) + + +@pytest.mark.parametrize("value", ["", "\x00", "nested/file\x00.pdf"]) +def test_resolve_inside_workroot_rejects_empty_and_nul_paths( + tmp_path: Path, + value: str, +) -> None: + with pytest.raises(ValueError): + resolve_inside_workroot(tmp_path, value) + + +def test_resolve_inside_workroot_rejects_traversal_and_prefix_confusion( + tmp_path: Path, +) -> None: + workroot = tmp_path / "jobs" + workroot.mkdir() + prefix_sibling = tmp_path / "jobs-attacker" + prefix_sibling.mkdir() + + rejected = [ + "../secret.pdf", + "nested/../../../secret.pdf", + str(tmp_path / "secret.pdf"), + str(prefix_sibling / "secret.pdf"), + ] + for value in rejected: + with pytest.raises(ValueError, match="escapes executor workroot"): + resolve_inside_workroot(workroot, value) + + +def test_resolve_inside_workroot_rejects_symlink_escape( + tmp_path: Path, +) -> None: + workroot = tmp_path / "workroot" + outside = tmp_path / "outside" + workroot.mkdir() + outside.mkdir() + (workroot / "escape").symlink_to(outside, target_is_directory=True) + + with pytest.raises(ValueError, match="escapes executor workroot"): + resolve_inside_workroot(workroot, "escape/document.pdf") + + +def test_resolve_inside_workroot_allows_symlink_to_contained_directory( + tmp_path: Path, +) -> None: + workroot = tmp_path / "workroot" + nested = workroot / "nested" + nested.mkdir(parents=True) + (workroot / "alias").symlink_to(nested, target_is_directory=True) + + assert resolve_inside_workroot(workroot, "alias/document.pdf") == ( + nested / "document.pdf" + ) + + +def test_resolve_file_and_dir_use_the_safe_root_boundary(tmp_path: Path) -> None: + workroot = tmp_path / "workroot" + workroot.mkdir() + input_file = workroot / "input.pdf" + input_file.write_bytes(b"pdf") + + assert resolve_file(workroot, "input.pdf") == input_file + assert resolve_dir(workroot, "output", create=True) == workroot / "output" + + escaped_dir = tmp_path / "escaped" + with pytest.raises(ValueError, match="escapes executor workroot"): + resolve_dir(workroot, "../escaped", create=True) + assert not escaped_dir.exists() + + +def test_relative_to_workroot_preserves_relative_output_and_rejects_escapes( + tmp_path: Path, +) -> None: + workroot = tmp_path / "workroot" + nested = workroot / "nested" + nested.mkdir(parents=True) + outside = tmp_path / "outside.pdf" + + assert relative_to_workroot(workroot, nested / "result.pdf") == str( + Path("nested") / "result.pdf", + ) + assert relative_to_workroot(workroot, workroot) == "." + assert relative_to_workroot(workroot, None) is None + + for value in [outside, "", "bad\x00path"]: + with pytest.raises(ValueError): + relative_to_workroot(workroot, value) + + +def test_relative_to_workroot_rejects_symlink_escape(tmp_path: Path) -> None: + workroot = tmp_path / "workroot" + outside = tmp_path / "outside" + workroot.mkdir() + outside.mkdir() + escaped_file = outside / "result.pdf" + escaped_file.touch() + link = workroot / "linked-result.pdf" + link.symlink_to(escaped_file) + + with pytest.raises(ValueError, match="escapes executor workroot"): + relative_to_workroot(workroot, link) diff --git a/uv.lock b/uv.lock index 277678cf..0bc01167 100644 --- a/uv.lock +++ b/uv.lock @@ -42,7 +42,7 @@ wheels = [ [[package]] name = "babeldoc" -version = "0.6.4+gloss.1" +version = "0.6.4+gloss.2" source = { editable = "." } dependencies = [ { name = "bitstring" },