diff --git a/Dockerfile b/Dockerfile index 0de944e..50eaba1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -84,11 +84,11 @@ RUN apt-get -y --no-install-recommends -o Dpkg::Options::="--force-confold" -y - ##### utils for python and TESSERACT RUN echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true" | debconf-set-selections -RUN apt-get install -y --no-install-recommends fontconfig ttf-mscorefonts-installer libimage-exiftool-perl libtcnative-1 \ +RUN apt-get update && apt-get install -y --no-install-recommends fontconfig ttf-mscorefonts-installer libimage-exiftool-perl libtcnative-1 \ libsm6 libxext6 gstreamer1.0-libav fonts-deva fonts-dejavu fonts-gfs-didot fonts-gfs-didot-classic fonts-junicode fonts-ebgaramond fonts-noto-cjk fonts-takao-gothic fonts-vlgothic \ ghostscript ghostscript-x gsfonts gsfonts-other gsfonts-x11 fonts-croscore fonts-crosextra-caladea fonts-crosextra-carlito fonts-liberation fonts-open-sans fonts-noto-core fonts-ibm-plex fonts-urw-base35 \ fonts-noto fonts-noto-cjk fonts-noto-extra xfonts-terminus fonts-font-awesome fonts-hack fonts-inconsolata fonts-liberation2 fonts-mononoki \ - libpcre3 libpcre3-dev \ + libpcre3 libpcre3-dev libxml2 libxml2-dev libxslt1.1 libxslt-dev \ mesa-opencl-icd pocl-opencl-icd libvips-tools libvips libvips-dev \ imagemagick libcairo2-dev tesseract-ocr tesseract-ocr-all libtesseract5 libtesseract-dev libleptonica-dev liblept5 @@ -168,5 +168,4 @@ RUN groupadd --system --gid "$OCR_SERVICE_GID" ocrsvc && \ ENV HOME=/home/ocrsvc USER ocrsvc -# Now run the simple api -CMD ["/bin/bash", "start_service_production.sh"] +CMD ["./start_service_production.sh"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1439990 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +SHELL := /usr/bin/env bash +.SHELLFLAGS := -o pipefail -c + +HELM_RELEASE ?= ocr-service +HELM_CHART ?= ./charts/ocr-service +HELM_TEXT_ONLY_VALUES ?= ./charts/ocr-service/values-text-only.yaml +HELM_ARGS ?= + +.PHONY: helm-install helm-install-text-only helm-template helm-lint helm-uninstall + +helm-install: + helm upgrade --install $(HELM_RELEASE) $(HELM_CHART) $(HELM_ARGS) + +helm-install-text-only: + helm upgrade --install $(HELM_RELEASE) $(HELM_CHART) -f $(HELM_TEXT_ONLY_VALUES) $(HELM_ARGS) + +helm-template: + helm template $(HELM_RELEASE) $(HELM_CHART) $(HELM_ARGS) + +helm-lint: + helm lint $(HELM_CHART) + +helm-uninstall: + helm uninstall $(HELM_RELEASE) $(HELM_ARGS) diff --git a/export_env_vars.sh b/export_env_vars.sh old mode 100644 new mode 100755 diff --git a/ocr_service/app/app.py b/ocr_service/app/app.py index dc032e3..f913311 100755 --- a/ocr_service/app/app.py +++ b/ocr_service/app/app.py @@ -141,7 +141,7 @@ def create_app() -> FastAPI: global _started try: - app = FastAPI(title="OCR Service", + app = FastAPI(title="OCR_Service", description="OCR Service API", version=settings.OCR_SERVICE_VERSION, default_response_class=ORJSONResponse, diff --git a/ocr_service/processor/converter.py b/ocr_service/processor/converter.py index 35b596d..f962aa4 100755 --- a/ocr_service/processor/converter.py +++ b/ocr_service/processor/converter.py @@ -3,9 +3,12 @@ import atexit import multiprocessing import os +import re import time import traceback import uuid +import zipfile +from html import unescape from io import BytesIO from subprocess import PIPE, Popen from threading import Timer @@ -19,12 +22,21 @@ from ocr_service.dto.process_context import ProcessContext from ocr_service.settings import settings -from ocr_service.utils.utils import INPUT_FILTERS, delete_tmp_files, terminate_hanging_process +from ocr_service.utils.utils import ( + INPUT_FILTERS, + delete_tmp_files, + is_encrypted_office_document, + terminate_hanging_process, +) CURRENT_PDF_FILE: pdfium.PdfDocument | None = None class DocumentConverter: + + MULTI_WHITESPACE = re.compile(r"[ \t]+") + MULTI_NEWLINES = re.compile(r"\n{3,}") + def __init__(self, log, loffice_process_list: dict[str, Any]) -> None: self.log = log self.loffice_process_list = loffice_process_list @@ -45,8 +57,15 @@ def resolve_content_type(file_type: object | None) -> str: @staticmethod def finalize_output_text(output_text: str) -> str: - output_text = output_text.translate({'\\n': '', '\\t': '', '\n\n': '\n'}) # type: ignore - return str(output_text).encode("utf-8", errors="replace").decode("utf-8") + + # normalize line endings + output_text = output_text.replace("\r\n", "\n").replace("\r", "\n") + # remove multiple whitespaces + output_text = DocumentConverter.MULTI_WHITESPACE.sub(" ", output_text) + # remove multiple new-lines + output_text = DocumentConverter.MULTI_NEWLINES.sub("\n\n", output_text) + + return output_text.encode("utf-8", errors="replace").decode("utf-8").strip() def _extract_text_fallback(self, stream: bytes, *, @@ -57,7 +76,7 @@ def _extract_text_fallback(self, text = "" if is_html or is_xml: - parser = "html.parser" if is_html else "xml" + parser = "html.parser" if is_html else "lxml-xml" try: soup = BeautifulSoup(stream, parser) except Exception: @@ -70,6 +89,11 @@ def _extract_text_fallback(self, else: text = soup.get_text(separator="\n") + # remove XML-ish self-closing tags + text = re.sub(r"<[^>]+/>", "", text) + # remove empty XML tags + text = re.sub(r"", "", text) + if not text and is_rtf: try: text = rtf_to_text(stream.decode("utf-8", "ignore")) @@ -77,9 +101,22 @@ def _extract_text_fallback(self, self.log.warning("Failed to parse RTF during fallback; using raw decode") if not text: - text = stream.decode("utf-8", "ignore") + text = stream.decode("utf-8", "ignore") + + return unescape(text) - return text.strip() + def _extract_office_zip_text_fallback(self, stream: bytes, file_name: str) -> str: + ext = os.path.splitext(file_name)[1].lower() + xml_path = {".docx": "word/document.xml", ".odt": "content.xml"}.get(ext) + if not xml_path: + return "" + + try: + with zipfile.ZipFile(BytesIO(stream)) as archive: + return self._extract_text_fallback(archive.read(xml_path), is_xml=True) + except Exception: + self.log.warning("Failed to extract %s from %s during fallback", xml_path, file_name) + return "" @staticmethod def initialize_pdf_worker(stream: bytes) -> None: @@ -111,11 +148,14 @@ def render_page(page_num: int) -> Image.Image: crop=(0, 0, 0, 0), grayscale=settings.OCR_CONVERT_GRAYSCALE_IMAGES, ).to_pil() + + page.close() return img def _pdf_to_img(self, stream: bytes) -> tuple[list[Image.Image], dict]: + pdf_image_pages = [] doc_metadata: dict[str, Any] = {} @@ -351,6 +391,35 @@ def _xml_to_text(self, ctx: ProcessContext) -> str: return " ".join(parts) + def _apply_text_fallback( + self, + ctx: ProcessContext, + *, + is_html: bool = False, + is_xml: bool = False, + is_rtf: bool = False, + reason: str, + ) -> None: + self.log.warning( + "Falling back to text extraction for %s after %s", + ctx.file_name, + reason, + ) + ctx.pdf_stream = b"" + ctx.images = [] + ctx.output_text = self._extract_office_zip_text_fallback(ctx.stream, ctx.file_name) + if not ctx.output_text: + ctx.output_text = self._extract_text_fallback( + ctx.stream, + is_html=is_html, + is_xml=is_xml, + is_rtf=is_rtf, + ) + ctx.metadata["pages"] = 1 + ctx.metadata["content-type"] = "text/plain" + ctx.metadata["fallback_reason"] = reason + + def _handle_pdf_stream(self, ctx: ProcessContext) -> None: if settings.OPERATION_MODE == "OCR": ctx.images, pdf_metadata = self._preprocess_pdf_to_img(ctx.pdf_stream) @@ -364,11 +433,24 @@ def prepare(self, ctx: ProcessContext) -> None: self.log.info("Checking file type for doc id: %s", ctx.file_name) + if is_encrypted_office_document(ctx.stream): + self.log.warning( + "Encrypted Office document detected for %s; skipping LibreOffice conversion", + ctx.file_name, + ) + ctx.metadata["content-type"] = "application/vnd.openxmlformats-officedocument" + ctx.metadata["encrypted"] = True + ctx.metadata["unsupported_reason"] = "encrypted_office_document" + ctx.metadata["pages"] = 0 + return + _is_pdf = type(ctx.file_type) is archive.Pdf _is_rtf = type(ctx.file_type) is archive.Rtf or ctx.checks.is_rtf() _is_html = ctx.checks.is_html() _is_xml = ctx.checks.is_xml() and not _is_html _is_plain = ctx.checks.is_plain_text() + _has_office_zip_fallback = os.path.splitext(ctx.file_name)[1].lower() in {".docx", ".odt"} + text_fallback_allowed = _is_xml or _is_rtf or _has_office_zip_fallback if _is_pdf: ctx.pdf_stream = ctx.stream @@ -427,19 +509,38 @@ def prepare(self, ctx: ProcessContext) -> None: self.log.info("Unknown file type; attempting to convert to PDF via unoserver/LO") ctx.pdf_stream = self._preprocess_doc(ctx.stream, file_name=ctx.file_name) - if not ctx.pdf_stream and not ctx.output_text and ctx.checks.is_text_like(): - self.log.warning( - "No PDF produced for %s; falling back to plain-text extraction", - ctx.file_name, - ) - ctx.output_text = self._extract_text_fallback( - ctx.stream, + if not ctx.pdf_stream and not ctx.output_text and (ctx.checks.is_text_like() or _has_office_zip_fallback): + self._apply_text_fallback( + ctx, is_html=_is_html, is_xml=_is_xml, is_rtf=_is_rtf, + reason="no_pdf_produced", ) - ctx.metadata["pages"] = 1 - ctx.metadata["content-type"] = "text/plain" if ctx.pdf_stream: - self._handle_pdf_stream(ctx) + try: + self._handle_pdf_stream(ctx) + except Exception: + if not text_fallback_allowed: + raise + self.log.exception( + "Converted PDF handling failed for %s; trying text fallback", + ctx.file_name, + ) + self._apply_text_fallback( + ctx, + is_html=_is_html, + is_xml=_is_xml, + is_rtf=_is_rtf, + reason="converted_pdf_handling_failed", + ) + else: + if text_fallback_allowed and not ctx.output_text and not ctx.images: + self._apply_text_fallback( + ctx, + is_html=_is_html, + is_xml=_is_xml, + is_rtf=_is_rtf, + reason="converted_pdf_handling_failed", + ) diff --git a/ocr_service/processor/ocr_engine.py b/ocr_service/processor/ocr_engine.py index 498fae6..97f9e7c 100755 --- a/ocr_service/processor/ocr_engine.py +++ b/ocr_service/processor/ocr_engine.py @@ -1,5 +1,6 @@ from __future__ import annotations +from enum import Enum import logging import time import traceback @@ -11,11 +12,16 @@ from ocr_service.settings import settings +class OcrPipeline(str, Enum): + TESSERACT = "tesseract" + VLM = "vlm" + MIXED = "mixed" + + class OcrEngine: def __init__(self, log: logging.Logger) -> None: self.log = log - def _init_tesseract_api_worker(self) -> PyTessBaseAPI: tesseract_api = PyTessBaseAPI(path=settings.TESSDATA_PREFIX, lang=settings.TESSERACT_LANGUAGE) # type: ignore self.log.debug("Initialised pytesseract api worker for language:" + str(settings.TESSERACT_LANGUAGE)) diff --git a/ocr_service/settings.py b/ocr_service/settings.py index c095c23..09b861c 100755 --- a/ocr_service/settings.py +++ b/ocr_service/settings.py @@ -1,6 +1,7 @@ # mypy: disable-error-code=prop-decorator import ast +from enum import Enum import logging import multiprocessing import os @@ -89,6 +90,10 @@ def validate_lo_port_range(cls, value: str | None) -> str | None: return value def model_post_init(self, __context: Any) -> None: + """ + Performs additional actions after the model is instantiated and all field validators are applied. + """ + default_lo_python = "/Applications/LibreOffice.app/Contents/Resources/python" default_lo_exec = "/Applications/LibreOffice.app/Contents/MacOS/soffice" tessdata_prefix = self.OCR_TESSDATA_PREFIX diff --git a/ocr_service/tests/resources/docs/generic/pat_id_1_openofficexml.odt b/ocr_service/tests/resources/docs/generic/pat_id_1_openofficexml.odt new file mode 100644 index 0000000..b5be945 --- /dev/null +++ b/ocr_service/tests/resources/docs/generic/pat_id_1_openofficexml.odt @@ -0,0 +1,335 @@ + + + + + + Clinical Test Document + OpenAI Synthetic Generator + Example NHS-like clinical XML document for OCR/NLP testing + + + + + + + Example NHS Foundation Trust + + + + Community Mental Health Team + + + + Document Type: + Outpatient Review Letter + + + + Date Created: + 28 May 2026 + + + + + + Patient Demographics + + + + NHS Number: + 485 777 3456 + + + + Hospital Number: + HN0092847 + + + + The patient’s name is + + Bart Davidson + . + + + + Date of Birth: + 14 February 1978 + + + + Gender: + Male + + + + His telephone number is + + 07754828992 + + + + + Address + + + + His Address is: + + + + + 61 Basildon Way + + + East Croyhurst + + + Angelton + + + AL64 9HT + + + + + Family and Carer Information + + + + His mother’s name is + + Pauline Smith + . + + + + His carer’s Name is + + Paul Wayne + . + + + + Relationship to patient: + Primary Carer + + + + Clinical Summary + + + + This is an example of a clinical document. + + + + Bart Davidson attended clinic today for a routine follow-up review. + + + + Mood appeared stable and no acute distress was observed. + + + + The patient denied suicidal ideation or recent self-harm. + + + + Sleep pattern described as variable with intermittent insomnia. + + + + Appetite reported as normal. + + + + No visual or auditory hallucinations reported during assessment. + + + + Medication + + + + He is on + 100mg + Paracetamol, + and + 20 milligrams + clozapine. + + + + + + + Medication + + + + Dose + + + + Frequency + + + + + + Paracetamol + + + + 100mg + + + + PRN + + + + + + Clozapine + + + + 20mg + + + + OD + + + + + + + Risk Assessment + + + + + Risk to self: Low + + + + Risk to others: Low + + + + Safeguarding concerns: None identified + + + + + Plan + + + + Continue current medication regime. + + + + Arrange routine blood monitoring for clozapine. + + + + Follow-up appointment in 4 weeks. + + + + + + This is a synthetic note for XML parsing tests. + + + + + + NLP / OCR Structured Entities + + + + <patient> + <name>Bart Davidson</name> + <dob>1978-02-14</dob> + <nhs_number>4857773456</nhs_number> + </patient> + + + + <carer> + <name>Paul Wayne</name> + <relationship>Primary Carer</relationship> + </carer> + + + + <medication name="Paracetamol" dose="100mg" frequency="PRN" /> + + + + <medication name="Clozapine" dose="20mg" frequency="OD" /> + + + + <address postcode="AL64 9HT"> + 61 Basildon Way, East Croyhurst, Angelton + </address> + + + + Additional XML Features + + + + + Extra spacing test. + + + + Line break test + + second line + + third line. + + + + Hyperlink test: + + NHS Website + + + + + Timestamp: + 14:32 + + + + + + Confidentiality Notice + + + + This document contains confidential clinical information intended + for authorised healthcare professionals only. + + + + + + diff --git a/ocr_service/tests/test_filename_handling.py b/ocr_service/tests/test_filename_handling.py index 6cf0fd0..0a16a70 100644 --- a/ocr_service/tests/test_filename_handling.py +++ b/ocr_service/tests/test_filename_handling.py @@ -1,11 +1,15 @@ import os import unittest +from pathlib import Path from unittest.mock import Mock +from ocr_service.dto.process_context import ProcessContext from ocr_service.processor.converter import DocumentConverter from ocr_service.processor.processor import Processor from ocr_service.settings import settings -from ocr_service.utils.utils import normalise_file_name_with_ext +from ocr_service.utils.utils import is_encrypted_office_document, normalise_file_name_with_ext + +TEST_RESOURCES = Path(__file__).resolve().parent / "resources" class TestFilenameHandling(unittest.TestCase): @@ -22,6 +26,14 @@ def test_detected_file_type_extension_is_used_when_name_has_no_suffix(self): self.assertEqual(file_name, "request-id.docx") + def test_encrypted_ooxml_extension_is_inferred_when_name_has_no_suffix(self): + stream = (TEST_RESOURCES / "docs/invalid/word_enc_noerror.docx").read_bytes() + + file_name = normalise_file_name_with_ext("request-id", stream) + + self.assertEqual(file_name, "request-id.docx") + self.assertTrue(is_encrypted_office_document(stream)) + def test_processor_passes_extensionless_unknown_name_to_converter(self): processor = Processor() processor.converter = Mock() @@ -47,3 +59,66 @@ def test_converter_builds_pdf_path_for_extensionless_input(self): self.assertEqual(doc_path, os.path.join(settings.TMP_FILE_DIR, "abc123_request-id")) self.assertEqual(pdf_path, os.path.join(settings.TMP_FILE_DIR, "abc123_request-id.pdf")) + + def test_converter_skips_encrypted_office_without_unoserver(self): + stream = (TEST_RESOURCES / "docs/invalid/word_enc_noerror.docx").read_bytes() + converter = DocumentConverter(Mock(), {}) + converter._preprocess_doc = Mock(return_value=b"") # type: ignore[method-assign] + ctx = ProcessContext(stream=stream, file_name="request-id.docx", file_type=None) + + converter.prepare(ctx) + + converter._preprocess_doc.assert_not_called() + self.assertTrue(ctx.metadata["encrypted"]) + self.assertEqual(ctx.metadata["unsupported_reason"], "encrypted_office_document") + + def test_rtf_falls_back_to_text_when_converted_pdf_handling_fails(self): + converter = DocumentConverter(Mock(), {}) + converter._preprocess_doc = Mock(return_value=b"%PDF-1.7\nnot usable") # type: ignore[method-assign] + converter._handle_pdf_stream = Mock(side_effect=RuntimeError("bad converted pdf")) # type: ignore[method-assign] + ctx = ProcessContext(stream=b"{\\rtf1\\ansi fallback text}", file_name="request-id.rtf", file_type=None) + + converter.prepare(ctx) + + self.assertIn("fallback text", ctx.output_text) + self.assertEqual(ctx.pdf_stream, b"") + self.assertEqual(ctx.metadata["fallback_reason"], "converted_pdf_handling_failed") + + def test_xml_falls_back_to_text_when_converted_pdf_handling_fails(self): + converter = DocumentConverter(Mock(), {}) + converter._preprocess_xml_to_pdf = Mock(return_value=b"") # type: ignore[method-assign] + converter._preprocess_doc = Mock(return_value=b"%PDF-1.7\nnot usable") # type: ignore[method-assign] + converter._handle_pdf_stream = Mock(side_effect=RuntimeError("bad converted pdf")) # type: ignore[method-assign] + ctx = ProcessContext( + stream=b'fallback text', + file_name="request-id.xml", + file_type=None, + ) + + converter.prepare(ctx) + + self.assertIn("fallback text", ctx.output_text) + self.assertEqual(ctx.pdf_stream, b"") + self.assertEqual(ctx.metadata["fallback_reason"], "converted_pdf_handling_failed") + + def test_docx_falls_back_to_document_xml_when_libreoffice_produces_no_pdf(self): + stream = (TEST_RESOURCES / "docs/generic/pat_id_1.docx").read_bytes() + converter = DocumentConverter(Mock(), {}) + converter._preprocess_doc = Mock(return_value=b"") # type: ignore[method-assign] + ctx = ProcessContext(stream=stream, file_name="request-id.docx", file_type=None) + + converter.prepare(ctx) + + self.assertIn("Bart Davidson", ctx.output_text) + self.assertEqual(ctx.metadata["fallback_reason"], "no_pdf_produced") + + def test_odt_falls_back_to_content_xml_when_libreoffice_produces_no_pdf(self): + stream = (TEST_RESOURCES / "docs/generic/pat_id_1.odt").read_bytes() + converter = DocumentConverter(Mock(), {}) + converter._preprocess_doc = Mock(return_value=b"") # type: ignore[method-assign] + ctx = ProcessContext(stream=stream, file_name="request-id.odt", file_type=None) + + converter.prepare(ctx) + + self.assertIn("Bart Davidson", ctx.output_text) + self.assertEqual(ctx.metadata["fallback_reason"], "no_pdf_produced") diff --git a/ocr_service/utils/utils.py b/ocr_service/utils/utils.py index 7efd2ed..f5f789c 100755 --- a/ocr_service/utils/utils.py +++ b/ocr_service/utils/utils.py @@ -15,12 +15,15 @@ import string import sys import xml.sax +import zipfile from datetime import datetime +from io import BytesIO from pathlib import Path from sys import platform from typing import Any import filetype +import olefile import psutil from html2image import Html2Image from PIL import Image @@ -30,6 +33,32 @@ logger = logging.getLogger(__name__) PRINTABLE = set(bytes(string.printable, "ascii")) | {9, 10, 13} +OLE_SIGNATURE = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + +ODF_MIME_EXTENSIONS: dict[str, str] = { + "application/vnd.oasis.opendocument.text": "odt", + "application/vnd.oasis.opendocument.text-template": "ott", + "application/vnd.oasis.opendocument.spreadsheet": "ods", + "application/vnd.oasis.opendocument.spreadsheet-template": "ots", + "application/vnd.oasis.opendocument.presentation": "odp", + "application/vnd.oasis.opendocument.presentation-template": "otp", + "application/vnd.oasis.opendocument.graphics": "odg", + "application/vnd.oasis.opendocument.formula": "odf", +} + +OOXML_PATH_EXTENSIONS: tuple[tuple[str, str], ...] = ( + ("word/document.xml", "docx"), + ("xl/workbook.xml", "xlsx"), + ("ppt/presentation.xml", "pptx"), +) + +OLE_STREAM_EXTENSIONS: tuple[tuple[str, str], ...] = ( + ("worddocument", "doc"), + ("workbook", "xls"), + ("book", "xls"), + ("powerpoint document", "ppt"), +) +ENCRYPTED_OOXML_STREAMS = {"encryptedpackage", "encryptioninfo"} INPUT_FILTERS: dict[str, str] = { # ── Writer / text ── @@ -227,7 +256,7 @@ def is_file_type_xml(stream: bytes) -> bool: xml.sax.parseString(stream, xml.sax.ContentHandler()) return True except Exception: - logger.warning("Could not determine if file is XML.") + logger.debug("Could not determine if file is XML.") return False def is_file_type_rtf(stream: bytes) -> bool: @@ -243,6 +272,77 @@ def is_file_type_rtf(stream: bytes) -> bool: return head.startswith(b"{\\rtf") +def _infer_zip_office_extension(stream: bytes) -> str | None: + try: + with zipfile.ZipFile(BytesIO(stream)) as archive: + names = set(archive.namelist()) + + if "mimetype" in names: + mimetype = archive.read("mimetype").decode("ascii", "ignore").strip() + extension = ODF_MIME_EXTENSIONS.get(mimetype) + if extension: + return extension + + for marker_path, extension in OOXML_PATH_EXTENSIONS: + if marker_path in names: + return extension + + lowered_names = {name.lower() for name in names} + if any(name.startswith("word/") for name in lowered_names): + return "docx" + if any(name.startswith("xl/") for name in lowered_names): + return "xlsx" + if any(name.startswith("ppt/") for name in lowered_names): + return "pptx" + except Exception: + logger.debug("Could not infer Office extension from ZIP container.") + + return None + + +def _ole_stream_names(stream: bytes) -> set[str]: + try: + with olefile.OleFileIO(BytesIO(stream)) as ole: + return {"/".join(path).lower() for path in ole.listdir()} + except Exception: + logger.debug("Could not inspect OLE streams.") + + return set() + + +def is_encrypted_office_document(stream: bytes) -> bool: + """Return True for encrypted OOXML packages stored in an OLE container.""" + if not stream.startswith(OLE_SIGNATURE): + return False + + return ENCRYPTED_OOXML_STREAMS.issubset(_ole_stream_names(stream)) + + +def _infer_ole_office_extension(stream: bytes) -> str | None: + stream_names = _ole_stream_names(stream) + leaf_names = {name.rsplit("/", 1)[-1] for name in stream_names} + + if ENCRYPTED_OOXML_STREAMS.issubset(stream_names): + return "docx" + + for stream_name, extension in OLE_STREAM_EXTENSIONS: + if stream_name in leaf_names: + return extension + + return None + + +def infer_office_extension_from_content(stream: bytes) -> str | None: + """Infer Office extensions for containers that generic file sniffing misses.""" + if stream.startswith(b"PK"): + return _infer_zip_office_extension(stream) + + if stream.startswith(OLE_SIGNATURE): + return _infer_ole_office_extension(stream) + + return None + + class TextChecks: """Lazy, cached text-type detection helpers for a single stream.""" __slots__ = ("stream", "_is_html", "_is_xml", "_is_rtf", "_is_plain_text") @@ -364,15 +464,26 @@ def normalise_file_name_with_ext(file_name: str, stream: bytes, file_type: objec # 2) prefer an already detected extension when available detected_ext = getattr(file_type, "extension", None) + if detected_ext and detected_ext != "zip": + return f"{base}.{str(detected_ext)}" + + # 3) inspect Office containers that generic file sniffing may miss + office_ext = infer_office_extension_from_content(stream) + if office_ext: + return f"{base}.{office_ext}" + if detected_ext: return f"{base}.{str(detected_ext)}" - # 3) let filetype guess it from content + # 4) let filetype guess it from content guessed_ext = filetype.guess_extension(stream) + if guessed_ext and guessed_ext != "zip": + return f"{base}.{guessed_ext}" + if guessed_ext: return f"{base}.{guessed_ext}" - # 4) fallbacks for texty formats our filetype may not catch + # 5) fallbacks for texty formats our filetype may not catch if is_file_type_html(stream): return base + ".html" if is_file_type_xml(stream): @@ -380,7 +491,7 @@ def normalise_file_name_with_ext(file_name: str, stream: bytes, file_type: objec if is_file_type_rtf(stream): return base + ".rtf" - # 5) only tag as plain text when the content actually looks like text + # 6) only tag as plain text when the content actually looks like text if is_file_content_plain_text(stream): return base + ".txt" @@ -547,15 +658,13 @@ def is_file_locked(path: str) -> bool: if not file_path.exists(): return False - f = open(file_path, "a+b") - try: - fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - fcntl.flock(f.fileno(), fcntl.LOCK_UN) - return False - except OSError: - return True - finally: - f.close() + with open(file_path, "a+b") as f: + try: + fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + fcntl.flock(f.fileno(), fcntl.LOCK_UN) + return False + except OSError: + return True def get_assigned_port(current_worker_pid: int) -> int: """Return the LibreOffice port previously assigned to a worker PID. diff --git a/requirements.txt b/requirements.txt index eb2178e..57420ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ Pillow==12.2.0 html2image==2.0.7 tesserocr==2.9.2 gunicorn==23.0.0 -pypdfium2==5.2.0 +pypdfium2==5.9.0 opencv-python-headless==4.12.0.88 pyxml2pdf==0.3.4 fastapi==0.116.1 @@ -17,6 +17,7 @@ pydantic-settings==2.14.2 httpx==0.28.1 beautifulsoup4==4.12.3 striprtf==0.0.29 +lxml==6.1.1 python-multipart==0.0.31 # Pillow package dependencies @@ -24,4 +25,4 @@ defusedxml==0.7.1 olefile==0.47 # unoserver>=3 switched to the unoconvert client CLI; tested with 3.6 -unoserver==3.6 +unoserver==3.7 diff --git a/start_service_debug.sh b/start_service_debug.sh old mode 100644 new mode 100755 index 6c6a674..cdf2eea --- a/start_service_debug.sh +++ b/start_service_debug.sh @@ -1,4 +1,6 @@ -#!/bin/bash +#!/usr/bin/env bash +set -euo pipefail + # set default config values if [[ -f ./env/ocr_service.env ]]; then diff --git a/start_service_production.sh b/start_service_production.sh old mode 100644 new mode 100755 index 56dc0c1..9fbfc93 --- a/start_service_production.sh +++ b/start_service_production.sh @@ -1,7 +1,7 @@ -#!/bin/bash +#!/usr/bin/env bash +set -euo pipefail # if ran manually execute "export_env_vars.sh" - export OCR_SERVICE_HOST="${OCR_SERVICE_HOST:-0.0.0.0}" export OCR_SERVICE_PORT="${OCR_SERVICE_PORT:-8090}" export OCR_SERVICE_WORKER_CLASS="${OCR_SERVICE_WORKER_CLASS:-sync}"