Fix tokenizer packaging and sequential executor handoff

UPSTREAM_BASE.toml

@@ -1,7 +1,7 @@
 # Gloss downstream provenance. Update downstream.version for every downstream
 # release; update the upstream block only in a reviewed upstream sync PR.
 [downstream]
-version = "0.6.4+gloss.4"
+version = "0.6.4+gloss.5"
 runtime_api_version = 1
 
 [upstream]

babeldoc/__init__.py

@@ -1,2 +1,2 @@
 # Gloss downstream release identity; see DOWNSTREAM.md.
-__version__ = "0.6.4+gloss.4"
+__version__ = "0.6.4+gloss.5"

babeldoc/const.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 
 # Gloss downstream release identity; see DOWNSTREAM.md.
-__version__ = "0.6.4+gloss.4"
+__version__ = "0.6.4+gloss.5"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

babeldoc/gloss_cli.py

@@ -167,8 +167,10 @@ def _verify_packaged_dependencies() -> None:
     import onnx
     import onnxruntime
     import pymupdf
+    import tiktoken
 
-    del bitstring, onnx, onnxruntime, pymupdf
+    tiktoken.get_encoding("o200k_base")
+    del bitstring, onnx, onnxruntime, pymupdf, tiktoken
 
 
 if __name__ == "__main__":

babeldoc/main.py

@@ -27,7 +27,7 @@
 
 logger = logging.getLogger(__name__)
 # Gloss downstream release identity; see DOWNSTREAM.md.
-__version__ = "0.6.4+gloss.4"
+__version__ = "0.6.4+gloss.5"
 
 
 def create_parser():

babeldoc/tools/executor/state.py

@@ -79,14 +79,18 @@ def __init__(
         runner: ExecutionRunner | None = None,
         max_event_log_size: int = MAX_EVENT_LOG_SIZE,
         max_execution_history_size: int = MAX_EXECUTION_HISTORY_SIZE,
+        terminal_handoff_wait_seconds: float = 2.0,
     ):
         if max_event_log_size <= 0:
             raise ValueError("max_event_log_size must be positive")
         if max_execution_history_size <= 0:
             raise ValueError("max_execution_history_size must be positive")
+        if terminal_handoff_wait_seconds < 0:
+            raise ValueError("terminal_handoff_wait_seconds must not be negative")
         self._runner = runner or UnavailableRunner()
         self._max_event_log_size = max_event_log_size
         self._max_execution_history_size = max_execution_history_size
+        self._terminal_handoff_wait_seconds = terminal_handoff_wait_seconds
         self._lock = threading.RLock()
         self._condition = threading.Condition(self._lock)
         self._records: OrderedDict[str, ExecutionRecord] = OrderedDict()
@@ -107,6 +111,29 @@ def create(self, request: dict[str, Any]) -> dict[str, Any]:
                 return self._create_response_locked(existing, replayed=True)
 
             active = self._active_record_locked()
+            if (
+                active is not None
+                and active.status in TERMINAL_EXECUTION_STATUSES
+                and self._terminal_handoff_wait_seconds > 0
+            ):
+                previous_execution_id = active.execution_id
+                logger.info(
+                    "executor waiting for terminal worker handoff: requested_task_id=%s active_task_id=%s active_execution_id=%s timeout_seconds=%s",
+                    task_id,
+                    active.task_id,
+                    active.execution_id,
+                    self._terminal_handoff_wait_seconds,
+                )
+                self._condition.wait_for(
+                    lambda: self._active_execution_id != previous_execution_id,
+                    timeout=self._terminal_handoff_wait_seconds,
+                )
+                existing = self._record_for_task_locked(task_id)
+                if existing is not None:
+                    if existing.request_fingerprint != request_fingerprint:
+                        raise ExecutionConflictError(self._snapshot_locked(existing))
+                    return self._create_response_locked(existing, replayed=True)
+                active = self._active_record_locked()
             if active is not None:
                 logger.warning(
                     "executor create rejected because active execution exists: requested_task_id=%s active_task_id=%s active_execution_id=%s",

docs/release-notes/v0.6.4-gloss.5.md

@@ -0,0 +1,31 @@
+# BabelDOC 0.6.4+gloss.5
+
+This downstream patch release fixes the self-contained macOS runtime used by
+Gloss PDF translation.
+
+## Runtime packaging
+
+- The PyInstaller build now includes the `tiktoken_ext` encoding registry
+  required by BabelDOC's GPT-4o tokenizer.
+- The packaged dependency smoke test loads `o200k_base`, so an incomplete
+  runtime fails during pull-request and release validation instead of during
+  the first real PDF translation.
+
+## Batch handoff
+
+- The executor now gives a terminal worker up to two seconds to finish cleanup
+  before rejecting the next task as busy. This closes the short race between
+  emitting a result and accepting the next PDF in a serial batch.
+- Running, cancelling, and stalled workers remain protected by the
+  single-active-execution boundary.
+
+## Validation
+
+- The arm64 candidate runtime completed the full packaged layout and executor
+  smoke test.
+- The 15-page *Attention Is All You Need* PDF completed an English-to-Chinese
+  translation twice in sequence through the authenticated Gloss bridge with
+  selectable text and intact tables, formulas, and figures.
+
+All upstream provenance, runtime protocol capabilities, and the minimum Gloss
+version remain unchanged from `0.6.4+gloss.4`.

pyproject.toml

@@ -1,7 +1,7 @@
 # Gloss downstream modification: release identity and runtime handshake, 2026-07-21.
 [project]
 name = "BabelDOC"
-version = "0.6.4+gloss.4"
+version = "0.6.4+gloss.5"
 description = "Gloss-maintained downstream of the BabelDOC document translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -170,7 +170,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.6.4+gloss.4"
+current_version = "0.6.4+gloss.5"
 version_pattern = "MAJOR.MINOR.PATCH+gloss.NUM"
 
 [bumpver.file_patterns]

scripts/release/build_macos_runtime.sh

@@ -47,6 +47,7 @@ uv run --no-sync pyinstaller \
   --collect-all babeldoc \
   --collect-all bitstring \
   --collect-all pymupdf \
+  --collect-submodules tiktoken_ext \
   --hidden-import babeldoc.tools.executor.babeldoc_adapter \
   --hidden-import babeldoc.tools.executor.layout_server \
   babeldoc/gloss_cli.py

tests/test_gloss_cli.py

@@ -9,8 +9,10 @@
 import sys
 from importlib import metadata
 from pathlib import Path
+from types import ModuleType
 
 import pytest
+from babeldoc.gloss_cli import _verify_packaged_dependencies
 from babeldoc.gloss_cli import build_runtime_info
 from babeldoc.gloss_cli import cli
 
@@ -129,6 +131,21 @@ def test_package_smoke_runs_dependency_check(
     }
 
 
+def test_packaged_dependency_check_loads_o200k_encoding(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    for name in ("bitstring", "onnx", "onnxruntime", "pymupdf"):
+        monkeypatch.setitem(sys.modules, name, ModuleType(name))
+    tiktoken = ModuleType("tiktoken")
+    requested_encodings: list[str] = []
+    tiktoken.get_encoding = requested_encodings.append  # type: ignore[attr-defined]
+    monkeypatch.setitem(sys.modules, "tiktoken", tiktoken)
+
+    _verify_packaged_dependencies()
+
+    assert requested_encodings == ["o200k_base"]
+
+
 def test_serve_forwards_service_options(
     monkeypatch: pytest.MonkeyPatch,
     tmp_path: Path,

tests/test_release_tools.py

@@ -18,8 +18,8 @@
 from scripts.release.sign_manifest import sign_manifest
 from scripts.release.sign_manifest import verify_manifest
 
-VERSION = "0.6.4+gloss.4"
-TAG_VERSION = "0.6.4-gloss.4"
+VERSION = "0.6.4+gloss.5"
+TAG_VERSION = "0.6.4-gloss.5"
 SOURCE_DATE_EPOCH = 1_750_000_000
 
 

tests/tools/executor/test_state.py

@@ -130,7 +130,7 @@ def fail_to_start(_record) -> None:
 
 def test_terminal_event_does_not_release_single_active_worker_early() -> None:
     runner = BlockingRunner()
-    store = ExecutionStore(runner)
+    store = ExecutionStore(runner, terminal_handoff_wait_seconds=0)
     first = store.create({"task_id": "first", "mode": "result_then_block"})
     assert runner.started.wait(timeout=1)
     wait_until(lambda: store.snapshot(first["execution_id"])["status"] == "succeeded")
@@ -148,6 +148,25 @@ def test_terminal_event_does_not_release_single_active_worker_early() -> None:
     assert store.snapshot(second["execution_id"])["status"] == "succeeded"
 
 
+def test_create_waits_for_terminal_worker_handoff() -> None:
+    runner = BlockingRunner()
+    store = ExecutionStore(runner, terminal_handoff_wait_seconds=1)
+    first = store.create({"task_id": "first", "mode": "result_then_block"})
+    assert runner.started.wait(timeout=1)
+    wait_until(lambda: store.snapshot(first["execution_id"])["status"] == "succeeded")
+
+    release = threading.Timer(0.05, runner.release.set)
+    release.start()
+    try:
+        second = store.create({"task_id": "second", "mode": "finish"})
+    finally:
+        release.join(timeout=1)
+
+    wait_until(lambda: snapshot_is_finished(store, first["execution_id"]))
+    wait_until(lambda: snapshot_is_finished(store, second["execution_id"]))
+    assert store.snapshot(second["execution_id"])["status"] == "succeeded"
+
+
 def test_targeted_cancel_stays_busy_until_worker_exits_and_emits_terminal() -> None:
     runner = BlockingRunner()
     store = ExecutionStore(runner)