From 6e0bb6155cd2aeb3619a514f344715e48313bf9d Mon Sep 17 00:00:00 2001 From: tokebe <43009413+tokebe@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:09:28 -0400 Subject: [PATCH 1/8] unify benchmarks --- {perf => bench}/test_sd.py | 14 +- {perf => bench}/test_sd_tom.py | 30 +- bench/test_semantic_validation.py | 103 +++++ bench/utils.py | 36 ++ pyproject.toml | 6 +- src/translator_tom/utils/dict_util_base.py | 381 ++++++++++++++++++ tests/conftest.py | 34 -- tests/test_models/test_model_read.py | 26 -- .../test_semantic_validation.py | 29 -- tests/util/general.py | 33 -- 10 files changed, 530 insertions(+), 162 deletions(-) rename {perf => bench}/test_sd.py (94%) rename {perf => bench}/test_sd_tom.py (77%) create mode 100644 bench/test_semantic_validation.py create mode 100644 bench/utils.py create mode 100644 src/translator_tom/utils/dict_util_base.py delete mode 100644 tests/conftest.py delete mode 100644 tests/test_models/test_model_read.py delete mode 100644 tests/test_validation/test_semantic_validation.py delete mode 100644 tests/util/general.py diff --git a/perf/test_sd.py b/bench/test_sd.py similarity index 94% rename from perf/test_sd.py rename to bench/test_sd.py index cfc7fb1..b1b6e75 100644 --- a/perf/test_sd.py +++ b/bench/test_sd.py @@ -4,16 +4,15 @@ summary table across files at the end. """ -import gzip import time -from pathlib import Path import orjson from pydantic import TypeAdapter +from utils import CORPUS_ROOT, read_corpus_file + LABEL_WIDTH = 23 VALUE_FMT = "{:>8.4f}s" -CORPUS_ROOT = Path("data/example_trapi") def pair_row( @@ -61,7 +60,7 @@ def section(title: str) -> None: # One representative file per size bucket. To benchmark every file, see -# `perf/test_sd_tom.py`. +# `bench/test_sd_tom.py`. TEST_FILES = [ CORPUS_ROOT / "10mb/pathfinder.json", CORPUS_ROOT / "50mb/lookup.json", @@ -79,12 +78,7 @@ def section(title: str) -> None: # --- Read --- t0 = time.perf_counter() - if response_path.suffix == ".gz": - with gzip.open(response_path, "rt", encoding="utf-8") as f: - response_json = f.read() - else: - with response_path.open() as f: - response_json = f.read() + response_json = read_corpus_file(response_path) t_read = time.perf_counter() - t0 size_mb = len(response_json.encode("utf-8")) / 1024 / 1024 diff --git a/perf/test_sd_tom.py b/bench/test_sd_tom.py similarity index 77% rename from perf/test_sd_tom.py rename to bench/test_sd_tom.py index b0bcf49..7f1eb79 100644 --- a/perf/test_sd_tom.py +++ b/bench/test_sd_tom.py @@ -5,16 +5,15 @@ then prints a summary table across files at the end. For a quicker comparison run that also benches reasoner-pydantic on one file -per size bucket, see `perf/test_sd.py`. +per size bucket, see `bench/test_sd.py`. """ -import gzip import time -from pathlib import Path + +from utils import CORPUS_ROOT, discover_files, read_corpus_file LABEL_WIDTH = 10 VALUE_FMT = "{:>8.4f}s" -CORPUS_ROOT = Path("data/example_trapi") def pair_row( @@ -38,22 +37,6 @@ def section(title: str) -> None: print(f"\n{bar}\n {title}\n{bar}") -def discover_files(root: Path) -> list[Path]: - """Return every `.json` and `.json.gz` under `root`, sorted by bucket size. - - Buckets are the immediate-parent directory name (`mb`); we sort by N - rather than by on-disk size since gzipped files compress smaller than - their uncompressed JSON. - """ - - def bucket_size(p: Path) -> int: - name = p.parent.name.removesuffix("mb") - return int(name) if name.isdigit() else 0 - - paths = [p for p in root.rglob("*") if p.is_file() and p.suffix in (".json", ".gz")] - return sorted(paths, key=lambda p: (bucket_size(p), p.name)) - - # --- Import --- t0 = time.perf_counter() @@ -77,12 +60,7 @@ def bucket_size(p: Path) -> int: results[label] = file_results t0 = time.perf_counter() - if response_path.suffix == ".gz": - with gzip.open(response_path, "rt", encoding="utf-8") as f: - response_json = f.read() - else: - with response_path.open() as f: - response_json = f.read() + response_json = read_corpus_file(response_path) t_read = time.perf_counter() - t0 size_mb = len(response_json.encode("utf-8")) / 1024 / 1024 diff --git a/bench/test_semantic_validation.py b/bench/test_semantic_validation.py new file mode 100644 index 0000000..a442750 --- /dev/null +++ b/bench/test_semantic_validation.py @@ -0,0 +1,103 @@ +"""Semantic-validation benchmark across every example response. + +Walks `data/example_trapi/**`, deserializes each file, and runs +`semantic_validate` on the resulting `Response`. Streams per-file timings and +the error/warning counts as they complete, then prints a summary table across +files at the end. + +For the serdes benchmarks see `bench/test_sd_tom.py` (TOM-only, every file) and +`bench/test_sd.py` (one file per size bucket, TOM vs reasoner-pydantic). +""" + +import time + +from utils import CORPUS_ROOT, discover_files, read_corpus_file + +VALUE_FMT = "{:>8.4f}s" + + +def section(title: str) -> None: + bar = "=" * (len(title) + 2) + print(f"\n{bar}\n {title}\n{bar}") + + +# --- Import --- + +t0 = time.perf_counter() +from translator_tom import Response # noqa: E402 +from translator_tom.validation import semantic_validate # noqa: E402 + +t_tom = time.perf_counter() - t0 + +section("Imports") +print(f" translator_tom + validation {VALUE_FMT.format(t_tom)}") + + +TEST_FILES = discover_files(CORPUS_ROOT) +print(f"\nDiscovered {len(TEST_FILES)} corpus file(s) under {CORPUS_ROOT}/") + +results: dict[str, dict[str, str]] = {} + + +for response_path in TEST_FILES: + label = str(response_path.relative_to(CORPUS_ROOT)) + file_results: dict[str, str] = {} + results[label] = file_results + + t0 = time.perf_counter() + response_json = read_corpus_file(response_path) + t_read = time.perf_counter() - t0 + size_mb = len(response_json.encode("utf-8")) / 1024 / 1024 + + section(f"{label} ({size_mb:.2f} MB JSON, {t_read:.4f}s)") + + t0 = time.perf_counter() + response = Response.from_json(response_json) + t_from_json = time.perf_counter() - t0 + + t0 = time.perf_counter() + warnings, errors = semantic_validate(response) + t_validate = time.perf_counter() - t0 + + print( + f" from_json {VALUE_FMT.format(t_from_json)}" + f" | validate {VALUE_FMT.format(t_validate)}" + f" | {len(errors)} errors, {len(warnings)} warnings" + ) + file_results["from_json"] = f"{t_from_json:.4f}s" + file_results["validate"] = f"{t_validate:.4f}s" + file_results["errors"] = str(len(errors)) + file_results["warnings"] = str(len(warnings)) + + +# --- Summary --- + +section("Summary") + +short_labels = { + lbl: lbl.split("/")[-1].removesuffix(".gz").removesuffix(".json") + for lbl in results +} +ops = list(next(iter(results.values())).keys()) + + +def fmt_cell(v: str | None) -> str: + return "—" if v is None else v + + +# Transposed: files as rows, metrics as columns (keeps the table narrow as +# more files are added to the corpus). +file_col = max(len(short_labels[lbl]) for lbl in results) +data_col = max( + max(len(op) for op in ops), + max(len(fmt_cell(results[lbl].get(op))) for lbl in results for op in ops), +) + +header = " " * file_col + " | " + " | ".join(f"{op:>{data_col}}" for op in ops) +print(header) +for lbl in results: + cells = [fmt_cell(results[lbl].get(op)) for op in ops] + print( + f"{short_labels[lbl]:<{file_col}} | " + + " | ".join(f"{c:>{data_col}}" for c in cells) + ) diff --git a/bench/utils.py b/bench/utils.py new file mode 100644 index 0000000..4f1ba23 --- /dev/null +++ b/bench/utils.py @@ -0,0 +1,36 @@ +"""Shared corpus helpers for the standalone benchmark scripts in this folder. + +Kept stdlib-only on purpose: the scripts time the `translator_tom` import, so +importing this module must not pull in `translator_tom` (or any heavy dep) and +perturb that measurement. +""" + +import gzip +from pathlib import Path + +CORPUS_ROOT = Path("data/example_trapi") + + +def discover_files(root: Path = CORPUS_ROOT) -> list[Path]: + """Return every `.json` and `.json.gz` under `root`, sorted by bucket size. + + Buckets are the immediate-parent directory name (`mb`); we sort by N + rather than by on-disk size since gzipped files compress smaller than + their uncompressed JSON. + """ + + def bucket_size(p: Path) -> int: + name = p.parent.name.removesuffix("mb") + return int(name) if name.isdigit() else 0 + + paths = [p for p in root.rglob("*") if p.is_file() and p.suffix in (".json", ".gz")] + return sorted(paths, key=lambda p: (bucket_size(p), p.name)) + + +def read_corpus_file(path: Path) -> str: + """Read a corpus file as text, transparently decompressing `.gz`.""" + if path.suffix == ".gz": + with gzip.open(path, "rt", encoding="utf-8") as f: + return f.read() + with path.open() as f: + return f.read() diff --git a/pyproject.toml b/pyproject.toml index 6755e00..cc00ded 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -163,12 +163,10 @@ DEP002 = [ ] [tool.pytest.ini_options] -norecursedirs = ["utils"] +# bench uses standalone benchmark scripts +norecursedirs = ["bench"] log_cli = true log_cli_level = "INFO" -markers = [ - "bench: heavy benchmarks; skipped unless the test file is named explicitly on the command line", -] [tool.coverage.run] branch = true diff --git a/src/translator_tom/utils/dict_util_base.py b/src/translator_tom/utils/dict_util_base.py new file mode 100644 index 0000000..2e05304 --- /dev/null +++ b/src/translator_tom/utils/dict_util_base.py @@ -0,0 +1,381 @@ +"""Supporting base for `*DictUtil` siblings of the `TypedDict` models. + +Provides shared I/O and model-parity hashing over the raw `TypedDict` form. +""" + +from __future__ import annotations + +__all__ = ["DictUtil", "register_union_discriminator"] + +import types +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass +from typing import ( + Any, + ClassVar, + Generic, + Literal, + TypeVar, + Union, + cast, + get_args, + get_origin, + overload, +) + +import orjson +import ormsgpack +from pydantic.fields import FieldInfo +from pydantic_core import PydanticUndefined + +from translator_tom.utils.hash import tomhash +from translator_tom.utils.object_base import TOMBase, _stable_repr + +_TD = TypeVar("_TD", bound=Mapping[str, Any]) + + +def _nested_models(annotation: Any) -> set[type[TOMBase]]: + """Find the nested `TOMBase` subclasses referenced by a field annotation. + + Unwraps `Annotated`, `Optional`/unions, and `list`/`dict` containers. + """ + found: set[type[TOMBase]] = set() + + def _visit(node: Any) -> None: + metadata = getattr(node, "__metadata__", None) + if metadata is not None: # Annotated[X, ...] -> X + _visit(node.__origin__) + return + if get_origin(node) is None: + if isinstance(node, type) and issubclass(node, TOMBase): + found.add(node) + return + for arg in get_args(node): + if arg is not type(None): + _visit(arg) + + _visit(annotation) + return found + + +def _discriminator_for(annotation: Any) -> str | None: + """Return the discriminator field name declared in an annotation's metadata, if any. + + Handles a discriminated union declared as `Annotated[A | B, Field(discriminator=...)]`, + including when nested inside `list`/`dict`/`Optional` (as TRAPI's `workflow` is). + """ + result: str | None = None + + def _visit(node: Any) -> None: + nonlocal result + metadata = getattr(node, "__metadata__", None) + if metadata is not None: + for meta in metadata: + discriminator = getattr(meta, "discriminator", None) + if isinstance(discriminator, str): + result = discriminator + _visit(node.__origin__) + return + if get_origin(node) is None: + return + for arg in get_args(node): + _visit(arg) + + _visit(annotation) + return result + + +def _tag_literals(annotation: Any) -> tuple[Any, ...]: + """Return the `Literal` value(s) of a discriminator field's annotation.""" + values = get_args(annotation) + if not values: + raise ValueError( + f"Discriminator field annotation {annotation!r} is not a Literal." + ) + return values + + +def _container_kind(annotation: Any) -> Literal["scalar", "list", "dict"]: + """Classify a field's outermost container (after stripping `Annotated`/`Optional`). + + Distinguishes a single nested model (`scalar`, e.g. `Message.query_graph`) from a + `list`/`dict` of them, since all three serialize to `dict`/`list` and can't be told + apart from the runtime value alone. + """ + node = annotation + while getattr(node, "__metadata__", None) is not None: + node = node.__origin__ + origin = get_origin(node) + if origin is Union or origin is types.UnionType: + non_none = [arg for arg in get_args(node) if arg is not type(None)] + return _container_kind(non_none[0]) if len(non_none) == 1 else "scalar" + if origin in {list, set, frozenset, tuple}: + return "list" + if origin is dict: + return "dict" + return "scalar" + + +class DictUtil(Generic[_TD]): + """Base for the `*DictUtil` sibling classes of the `TypedDict` models. + + A `*DictUtil` reimplements the utility methods of its Pydantic counterpart for + the `TypedDict` form, so users can operate on plain dicts without time overhead + for model construction/validation. + + Subclasses set `_model` (the mirrored model). It provides + the field names/order used by `hash` and, via its field types, which fields + hold nested models that recurse into a sibling `DictUtil` (see `_nested_fields`). + """ + + # The Pydantic model this dict mirrors; the source of truth for hashing. + _model: ClassVar[type[TOMBase]] + # Registry of model -> its DictUtil, populated as subclasses are defined. + _registry: ClassVar[dict[type[TOMBase], type[DictUtil[Any]]]] = {} + # Per-subclass cache for `_nested_fields()`. + _nested_fields_cache: ClassVar[dict[str, _NestedField] | None] = None + # Per-subclass cache for `_field_defaults()`. + _field_defaults_cache: ClassVar[dict[str, Any] | None] = None + + def __init_subclass__(cls, **kwargs: Any) -> None: + """Register each concrete `*DictUtil` under the model it mirrors.""" + super().__init_subclass__(**kwargs) + model = cls.__dict__.get("_model") + if model is not None: + DictUtil._registry[model] = cls + + ##### I/O methods ##### + + @classmethod + def from_json(cls, json: str | bytes) -> _TD: + """Deserialize a dict from JSON.""" + return cast("_TD", orjson.loads(json)) + + @overload + @classmethod + def to_json(cls, obj: _TD) -> str: ... + + @overload + @classmethod + def to_json(cls, obj: _TD, as_str: Literal[True]) -> str: ... + + @overload + @classmethod + def to_json(cls, obj: _TD, as_str: Literal[False]) -> bytes: ... + + @classmethod + def to_json(cls, obj: _TD, as_str: bool = False) -> str | bytes: + """Serialize a dict to JSON. + + Dicts are expected to already be in canonical (None-omitted) form, so keys + are serialized as-is rather than filtered. + """ + json = orjson.dumps(obj) + if as_str: + return json.decode() + return json + + @classmethod + def from_msgpack(cls, msgpack: bytes) -> _TD: + """Deserialize a dict from MessagePack.""" + return cast("_TD", ormsgpack.unpackb(msgpack)) + + @classmethod + def to_msgpack(cls, obj: _TD) -> bytes: + """Serialize a dict to MessagePack.""" + return ormsgpack.packb(obj) + + ##### Hashing ##### + + @classmethod + def _nested_fields(cls) -> dict[str, _NestedField]: + """Map each field holding nested model(s) to how its value should be hashed. + + Derived and cached from the mirrored model's field types: each entry pairs the + field's container shape with a resolver that picks the member util per element + (see `_container_kind` and `_nested_resolver`). + """ + cached = cls.__dict__.get("_nested_fields_cache") + if cached is not None: + return cached + mapping: dict[str, _NestedField] = {} + for name, field in cls._model.model_fields.items(): + models = _nested_models(field.annotation) + if models: + mapping[name] = _NestedField( + _container_kind(field.annotation), + cls._nested_resolver(name, field, models), + ) + cls._nested_fields_cache = mapping + return mapping + + @classmethod + def _util_for(cls, field_name: str, model: type[TOMBase]) -> type[DictUtil[Any]]: + """Return the registered `DictUtil` for `model`, or raise if none exists.""" + util = cls._registry.get(model) + if util is None: + raise LookupError( + f"{cls.__name__}: field {field_name!r} holds {model.__name__}, but no " + f"DictUtil is registered for it (define {model.__name__}DictUtil)." + ) + return util + + @classmethod + def _nested_resolver( + cls, field_name: str, field: FieldInfo, models: set[type[TOMBase]] + ) -> _Resolver: + """Build the resolver mapping a nested field's value to the util that hashes it. + + Single-model fields resolve to a constant util. Union fields resolve per + element: by discriminator tag for a pydantic tagged union, otherwise via a + discriminator registered with `register_union_discriminator`. + """ + if len(models) == 1: + (model,) = models + return _ConstResolver(cls._util_for(field_name, model)) + discriminator = ( + field.discriminator + if isinstance(field.discriminator, str) + else _discriminator_for(field.annotation) + ) + if discriminator is not None: + by_tag: dict[Any, type[DictUtil[Any]]] = {} + for model in models: + util = cls._util_for(field_name, model) + for tag in _tag_literals(model.model_fields[discriminator].annotation): + by_tag[tag] = util + return _TagResolver(discriminator, by_tag) + discriminate = _UNION_DISCRIMINATORS.get(frozenset(models)) + if discriminate is None: + raise LookupError( + f"{cls.__name__}: field {field_name!r} is a union of " + f"{sorted(m.__name__ for m in models)} with no discriminator; register " + "one with register_union_discriminator()." + ) + return _StructuralResolver( + discriminate, {model: cls._util_for(field_name, model) for model in models} + ) + + @classmethod + def _hash_field(cls, key: str, value: Any) -> Any: + """Produce the stable representation of one declared field for hashing.""" + nested = cls._nested_fields().get(key) + if nested is None or value is None: + return _stable_repr(value) + return nested.hashed(value) + + @classmethod + def _field_defaults(cls) -> dict[str, Any]: + """Map each model field to its default (used for keys omitted from a dict). + + `to_dict` uses `exclude_defaults`, so a default-valued field is absent from + the serialized dict; hashing restores the default to match the model, whose + `hash()` reads live field values. Required fields (no default) map to None, + but they are always present in a valid serialization so the fallback is unused. + """ + cached = cls.__dict__.get("_field_defaults_cache") + if cached is not None: + return cached + defaults: dict[str, Any] = {} + for name, field in cls._model.model_fields.items(): + default = field.get_default(call_default_factory=True) + defaults[name] = None if default is PydanticUndefined else default + cls._field_defaults_cache = defaults + return defaults + + @classmethod + def hash(cls, obj: _TD) -> str: + """Hash the dict into a hex string, matching the corresponding model's `hash()`. + + Hashes only declared fields (not extra keys), keyed by the model's field + names in definition order, so a dict and its equivalent model hash equally. + Fields omitted from the dict fall back to their model default (see + `_field_defaults`), mirroring the model whose defaults are always live. + """ + defaults = cls._field_defaults() + return tomhash( + ( + cls._model.__name__, + *( + (key, cls._hash_field(key, obj.get(key, defaults[key]))) + for key in cls._model.model_fields + ), + ) + ) + + +# A resolver maps a nested field's dict value to the `DictUtil` that hashes it. +_Resolver = Callable[[Any], "type[DictUtil[Any]]"] +# A structural-union discriminator maps a raw dict to its concrete member model. +_Discriminator = Callable[[Mapping[str, Any]], type[TOMBase]] + + +@dataclass +class _NestedField: + """How to hash one field that holds nested model(s): container shape + util resolver. + + `kind` (from the field annotation, not the value) says whether the field is a single + nested model, a `list` of them, or a `dict` of them; `resolve` picks the member util + for each element. + """ + + kind: Literal["scalar", "list", "dict"] + resolve: _Resolver + + def hashed(self, value: Any) -> Any: + """Return the stable representation of the field's value for hashing.""" + if self.kind == "list": + return [self.resolve(v).hash(v) for v in value] + if self.kind == "dict": + return {k: self.resolve(v).hash(v) for k, v in value.items()} + return self.resolve(value).hash(value) + + +@dataclass +class _ConstResolver: + """Resolver for a single-model field: always the one util.""" + + util: type[DictUtil[Any]] + + def __call__(self, value: Any) -> type[DictUtil[Any]]: + """Return the field's util, ignoring `value`.""" + return self.util + + +@dataclass +class _TagResolver: + """Resolver for a pydantic tagged union: pick the util by the discriminator value.""" + + tag_field: str + by_tag: dict[Any, type[DictUtil[Any]]] + + def __call__(self, value: Any) -> type[DictUtil[Any]]: + """Return the util for `value`'s discriminator tag.""" + return self.by_tag[value[self.tag_field]] + + +@dataclass +class _StructuralResolver: + """Resolver for a structural union: discriminate the dict, then map model -> util.""" + + discriminate: _Discriminator + by_model: dict[type[TOMBase], type[DictUtil[Any]]] + + def __call__(self, value: Any) -> type[DictUtil[Any]]: + """Return the util for the model `discriminate` selects for `value`.""" + return self.by_model[self.discriminate(value)] + + +_UNION_DISCRIMINATORS: dict[frozenset[type[TOMBase]], _Discriminator] = {} + + +def register_union_discriminator( + members: Iterable[type[TOMBase]], discriminate: _Discriminator +) -> None: + """Register how to resolve a structural (non-tagged) union of models from a raw dict. + + `discriminate` receives the raw dict and returns the concrete member model class. + Needed only for unions without a pydantic discriminator (e.g. `QueryGraph` vs + `PathfinderQueryGraph`); tagged unions are resolved automatically. + """ + _UNION_DISCRIMINATORS[frozenset(members)] = discriminate diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 3a1c4fa..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Pytest configuration for the TOM test suite. - -Tests marked `bench` are skipped unless their containing file is named -explicitly on the pytest command line. This lets `pytest` (or any directory -sweep like `pytest tests/`) stay fast, while -`pytest tests/test_models/test_model_read.py` still runs them. -""" - -from __future__ import annotations - -import pytest - - -def pytest_collection_modifyitems( - config: pytest.Config, items: list[pytest.Item] -) -> None: - invocation_args = config.invocation_params.args - - def explicitly_invoked(item: pytest.Item) -> bool: - # item.path is the absolute path to the test file - for arg in invocation_args: - target = arg.split("::", 1)[0] - if not target: - continue - if str(item.path).endswith(target) or target.endswith(item.path.name): - return True - return False - - skip_bench = pytest.mark.skip( - reason="bench test — run the file directly (`pytest `) to invoke" - ) - for item in items: - if "bench" in item.keywords and not explicitly_invoked(item): - item.add_marker(skip_bench) diff --git a/tests/test_models/test_model_read.py b/tests/test_models/test_model_read.py deleted file mode 100644 index 3c31788..0000000 --- a/tests/test_models/test_model_read.py +++ /dev/null @@ -1,26 +0,0 @@ -import logging -import time - -import pytest -from util.general import TEST_FILES, get_test_json - -from translator_tom import Response - -pytestmark = pytest.mark.bench - -LOG = logging.getLogger(__name__) - - -@pytest.fixture(scope="module") -def examples() -> dict[str, str]: - return get_test_json() - - -@pytest.mark.parametrize("name", [p.stem for p in TEST_FILES]) -def test_convert(examples: dict[str, str], name: str) -> None: - json_str = examples[name] - t0 = time.perf_counter() - LOG.info("Deserializing %s example...", name) - Response.from_json(json_str) - t1 = time.perf_counter() - LOG.info("Deserialization took %s seconds.", round(t1 - t0, 6)) diff --git a/tests/test_validation/test_semantic_validation.py b/tests/test_validation/test_semantic_validation.py deleted file mode 100644 index 3a06476..0000000 --- a/tests/test_validation/test_semantic_validation.py +++ /dev/null @@ -1,29 +0,0 @@ -import logging -import time - -import pytest -from util.general import TEST_FILES, get_test_json - -from translator_tom import Response -from translator_tom.validation import semantic_validate - -pytestmark = pytest.mark.bench - -LOG = logging.getLogger(__name__) - - -@pytest.fixture(scope="module") -def examples() -> dict[str, str]: - return get_test_json() - - -@pytest.mark.parametrize("name", [p.stem for p in TEST_FILES]) -def test_semantic_validate(examples: dict[str, str], name: str) -> None: - json_str = examples[name] - t0 = time.perf_counter() - LOG.info("Running semantic validation on %s example...", name) - response = Response.from_json(json_str) - warnings, errors = semantic_validate(response) - t1 = time.perf_counter() - LOG.info("Semantic validation took %s seconds.", round(t1 - t0, 6)) - LOG.info("Got %s errors and %s warnings", len(errors), len(warnings)) diff --git a/tests/util/general.py b/tests/util/general.py deleted file mode 100644 index 4888bf9..0000000 --- a/tests/util/general.py +++ /dev/null @@ -1,33 +0,0 @@ -import gzip -import logging -from pathlib import Path - -LOG = logging.getLogger(__name__) - -# Enumerated as a module constant so callers (e.g. pytest parametrize) can -# discover the example set without paying the load cost. -TEST_FILES: list[Path] = [ - Path("data/example_trapi/10mb/pathfinder.json"), - # Path("data/example_trapi/10mb/log-heavy.json"), - # Path("data/example_trapi/50mb/lookup.json"), - Path("data/example_trapi/50mb/result-heavy.json"), - Path("data/example_trapi/250mb/attribute-heavy.json.gz"), -] - - -def _read(path: Path) -> str: - LOG.info( - "Read local JSON file %s of size %s MB", - path, - path.stat().st_size / 1024 / 1024, - ) - if path.suffix.endswith(".gz"): - with gzip.open(path, "rt", encoding="utf-8") as infile: - return infile.read() - with path.open() as infile: - return infile.read() - - -def get_test_json() -> dict[str, str]: - """Return a dictionary of label:response json.""" - return {path.stem: _read(path) for path in TEST_FILES} From aa27b5f0f01558d3247b40ab1a3bfab51e77009e Mon Sep 17 00:00:00 2001 From: tokebe <43009413+tokebe@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:09:50 -0400 Subject: [PATCH 2/8] add parse/validate scripts --- pyproject.toml | 4 + src/translator_tom/scripts/__init__.py | 7 ++ src/translator_tom/scripts/parse.py | 106 +++++++++++++++++++++++++ src/translator_tom/scripts/validate.py | 71 +++++++++++++++++ 4 files changed, 188 insertions(+) create mode 100644 src/translator_tom/scripts/__init__.py create mode 100644 src/translator_tom/scripts/parse.py create mode 100644 src/translator_tom/scripts/validate.py diff --git a/pyproject.toml b/pyproject.toml index cc00ded..82c7561 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,10 @@ Homepage = "https://github.com/NCATSTranslator/TRAPIObjectModeling" Repository = "https://github.com/NCATSTranslator/TRAPIObjectModeling" Issues = "https://github.com/NCATSTranslator/TRAPIObjectModeling/issues" +[project.scripts] +tom-parse = "translator_tom.scripts.parse:main" +tom-validate = "translator_tom.scripts.validate:main" + [dependency-groups] dev = [ "bpython>=0.26", diff --git a/src/translator_tom/scripts/__init__.py b/src/translator_tom/scripts/__init__.py new file mode 100644 index 0000000..77a6979 --- /dev/null +++ b/src/translator_tom/scripts/__init__.py @@ -0,0 +1,7 @@ +"""Small command-line utilities shipped with ``translator_tom``. + +Each module is runnable directly, for example:: + + python -m translator_tom.scripts.parse Response response.json + python -m translator_tom.scripts.validate Response response.json +""" diff --git a/src/translator_tom/scripts/parse.py b/src/translator_tom/scripts/parse.py new file mode 100644 index 0000000..ec472b4 --- /dev/null +++ b/src/translator_tom/scripts/parse.py @@ -0,0 +1,106 @@ +"""Parse a JSON file into a named TRAPI object model. + +Installed as ``tom-parse``:: + + tom-parse Response response.json + tom-parse Message message.json --out normalized.json +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +import orjson +from pydantic import ValidationError + +import translator_tom +from translator_tom import TOMBase + +if TYPE_CHECKING: + from collections.abc import Sequence + + +def available_models() -> dict[str, type[TOMBase]]: + """Return the exported model names mapped to their ``TOMBase`` subclass.""" + models: dict[str, type[TOMBase]] = {} + for name in translator_tom.__all__: + obj = getattr(translator_tom, name) + if isinstance(obj, type) and issubclass(obj, TOMBase) and obj is not TOMBase: + models[name] = obj + return models + + +def parse_file(model: type[TOMBase], path: Path) -> TOMBase: + """Read ``path`` and parse it into an instance of ``model``.""" + return model.from_json(path.read_bytes()) + + +def load_parse_or_report(model_name: str, path: Path) -> tuple[TOMBase | None, int]: + """Parse ``path`` into the named model, printing a diagnostic on failure. + + Returns ``(instance, exit_code)`` where ``instance`` is ``None`` on failure + and ``exit_code`` is a process return code (``0`` on success). Shared by the + ``parse`` and ``validate`` entry points so both report problems identically. + """ + models = available_models() + model = models.get(model_name) + if model is None: + print( + f"error: unknown model {model_name!r}. Available models:", file=sys.stderr + ) + print(" " + ", ".join(sorted(models)), file=sys.stderr) + return None, 2 + if not path.is_file(): + print(f"error: file missing: {path}", file=sys.stderr) + return None, 2 + try: + instance = parse_file(model, path) + except OSError as exc: + print(f"error: could not read {path}: {exc}", file=sys.stderr) + return None, 2 + except orjson.JSONDecodeError as exc: + print(f"✗ {path} is not valid JSON: {exc}", file=sys.stderr) + return None, 1 + except ValidationError as exc: + print( + f"✗ {path} failed to parse to model: {model_name}:\n{exc}", file=sys.stderr + ) + return None, 1 + return instance, 0 + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="tom-parse", + description="Parse a JSON file into a TRAPI Object Model.", + ) + parser.add_argument("model", help="model class name to parse into, e.g. Response") + parser.add_argument("file", help="path to a JSON file") + parser.add_argument( + "-o", + "--out", + metavar="PATH", + help="write the parsed model back out as normalized JSON to PATH", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Parse the file named on the command line, reporting success or failure.""" + args = _build_parser().parse_args(argv) + instance, code = load_parse_or_report(args.model, Path(args.file)) + if instance is None: + return code + print(f"✓ Parsed {args.file} as {args.model}") + if args.out is not None: + out = Path(args.out) + out.write_bytes(instance.to_json()) + print(f" wrote normalized JSON to {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/translator_tom/scripts/validate.py b/src/translator_tom/scripts/validate.py new file mode 100644 index 0000000..2fe4082 --- /dev/null +++ b/src/translator_tom/scripts/validate.py @@ -0,0 +1,71 @@ +"""Parse a file into a named model, then run semantic validation on the result. + +Installed as the ``tom-validate`` command (also runnable via +``python -m translator_tom.scripts.validate``):: + + tom-validate Response response.json + +Exits ``0`` when validation finds no errors and ``1`` when it does, so it can be +used as a check in scripts or CI. Warnings are reported but do not fail the run. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import TYPE_CHECKING + +from translator_tom.scripts.parse import load_parse_or_report +from translator_tom.validation import semantic_validate + +if TYPE_CHECKING: + from collections.abc import Sequence + + from translator_tom.validation import ( + SemanticValidationError, + SemanticValidationWarning, + ) + + +def _format_entry(entry: SemanticValidationError | SemanticValidationWarning) -> str: + """Render a validation error/warning as an indented, location-prefixed line.""" + location = ".".join(str(part) for part in entry.location) or "" + return f" [{location}] {entry.message}" + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="tom-validate", + description=( + "Parse a JSON file into a TRAPI object model, then run semantic " + "validation on it. Exits non-zero if validation finds errors." + ), + ) + parser.add_argument("model", help="model class name to parse into, e.g. Response") + parser.add_argument("file", help="path to a JSON file") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Parse and semantically validate the file named on the command line.""" + args = _build_parser().parse_args(argv) + instance, code = load_parse_or_report(args.model, Path(args.file)) + if instance is None: + return code + print(f"✓ Parsed {args.file} as {args.model}") + + warnings, errors = semantic_validate(instance) + + print(f"\nWarnings ({len(warnings)}):") + for warning in warnings: + print(_format_entry(warning)) + print(f"\nErrors ({len(errors)}):") + for error in errors: + print(_format_entry(error)) + + print(f"\nSemantic validation: {len(errors)} error(s), {len(warnings)} warning(s)") + return 1 if errors else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 5600d30aa415ff896ecbab9cdf9be84a46b41534 Mon Sep 17 00:00:00 2001 From: tokebe <43009413+tokebe@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:11:31 -0400 Subject: [PATCH 3/8] fix minor bugs (model builds, defaults, type) --- src/translator_tom/models/knowledge_graph.py | 3 +++ src/translator_tom/models/meta_knowledge_graph.py | 3 +++ src/translator_tom/models/query_graph.py | 6 ++++++ src/translator_tom/models/workflow_operations.py | 4 ++-- src/translator_tom/utils/object_base.py | 2 +- 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/translator_tom/models/knowledge_graph.py b/src/translator_tom/models/knowledge_graph.py index f42f780..e59e7f1 100644 --- a/src/translator_tom/models/knowledge_graph.py +++ b/src/translator_tom/models/knowledge_graph.py @@ -369,3 +369,6 @@ def append_aggregator(self, source: Infores) -> None: upstream_resource_ids=[last_downstream.resource_id], ) ) + + +KnowledgeGraph.model_rebuild() # Don't defer model build diff --git a/src/translator_tom/models/meta_knowledge_graph.py b/src/translator_tom/models/meta_knowledge_graph.py index 8c8c837..08fbece 100644 --- a/src/translator_tom/models/meta_knowledge_graph.py +++ b/src/translator_tom/models/meta_knowledge_graph.py @@ -202,3 +202,6 @@ def meets_qualifier_constraints( return any( constraint.met_by(self.qualifiers_list) for constraint in constraints ) + + +MetaKnowledgeGraph.model_rebuild() # Don't defer model build diff --git a/src/translator_tom/models/query_graph.py b/src/translator_tom/models/query_graph.py index 408bd82..de21f9c 100644 --- a/src/translator_tom/models/query_graph.py +++ b/src/translator_tom/models/query_graph.py @@ -299,3 +299,9 @@ def predicates_list(self) -> list[Biolink.Predicate]: def constraints_list(self) -> list[PathConstraint]: """Get the constraints as a guaranteed list, even if they are represented as None.""" return self.constraints if self.constraints is not None else [] + + +# Don't defer model builds +BaseQueryGraph.model_rebuild() +QueryGraph.model_rebuild() +PathfinderQueryGraph.model_rebuild() diff --git a/src/translator_tom/models/workflow_operations.py b/src/translator_tom/models/workflow_operations.py index 80eb15b..2625969 100644 --- a/src/translator_tom/models/workflow_operations.py +++ b/src/translator_tom/models/workflow_operations.py @@ -418,7 +418,7 @@ class FilterKgraphPercentileParameters(FilterKgraphParametersBase): edge_attribute: Annotated[str, Field(examples=["normalized_google_distance"])] """The name of the edge attribute to filter on.""" - threshold: Annotated[float, Field(gt=0, le=100, examples=[96.8])] = 95 + threshold: Annotated[float, Field(gt=0, le=100, examples=[96.8])] = 95.0 """The percentile to threshold on.""" remove_above_or_below: AboveOrBelow = "below" @@ -448,7 +448,7 @@ class FilterKgraphStdDevParameters(FilterKgraphParametersBase): edge_attribute: Annotated[str, Field(examples=["normalized_google_distance"])] """The name of the edge attribute to filter on.""" - num_sigma: Annotated[float, Field(gt=0, examples=[1.2])] = 1 + num_sigma: Annotated[float, Field(gt=0, examples=[1.2])] = 1.0 """The number of standard deviations to threshold on.""" remove_above_or_below: AboveOrBelow = "below" diff --git a/src/translator_tom/utils/object_base.py b/src/translator_tom/utils/object_base.py index f2c7375..398b50b 100644 --- a/src/translator_tom/utils/object_base.py +++ b/src/translator_tom/utils/object_base.py @@ -66,7 +66,7 @@ def from_json(cls, json: str | bytes) -> Self: return cls.model_validate(orjson.loads(json)) @overload - def to_json(self) -> str: ... + def to_json(self) -> bytes: ... @overload def to_json(self, as_str: Literal[True]) -> str: ... From ccbb3e0539b91825027e8f197516c62f235c4582 Mon Sep 17 00:00:00 2001 From: tokebe <43009413+tokebe@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:23:38 -0400 Subject: [PATCH 4/8] add model_dict util methods Mirrors model util methods --- bench/test_sd_tom_dicts.py | 114 ++++ pyproject.toml | 2 +- src/translator_tom/model_dicts/__init__.py | 99 +++- src/translator_tom/model_dicts/analysis.py | 151 ++++- src/translator_tom/model_dicts/attribute.py | 175 +++++- .../model_dicts/auxiliary_graph.py | 64 ++- .../model_dicts/edge_binding.py | 20 +- .../model_dicts/knowledge_graph.py | 359 +++++++++++- src/translator_tom/model_dicts/log_entry.py | 37 +- src/translator_tom/model_dicts/message.py | 130 ++++- .../model_dicts/meta_attribute.py | 42 +- .../model_dicts/meta_knowledge_graph.py | 160 +++++- .../model_dicts/meta_qualifier.py | 16 +- .../model_dicts/node_binding.py | 24 +- .../model_dicts/path_binding.py | 10 +- .../model_dicts/path_constraint.py | 18 +- src/translator_tom/model_dicts/qualifier.py | 120 ++++ src/translator_tom/model_dicts/query.py | 21 +- src/translator_tom/model_dicts/query_graph.py | 163 +++++- src/translator_tom/model_dicts/response.py | 27 +- src/translator_tom/model_dicts/result.py | 139 ++++- .../model_dicts/retrieval_source.py | 38 +- .../model_dicts/workflow_operations.py | 538 +++++++++++++++++- src/translator_tom/utils/dict_util_base.py | 27 +- tests/test_model_dicts/test_analysis_dicts.py | 137 +++++ .../test_model_dicts/test_attribute_dicts.py | 278 +++++++++ .../test_auxiliary_graph_dicts.py | 91 +++ tests/test_model_dicts/test_binding_dicts.py | 44 ++ .../test_knowledge_graph_dicts.py | 350 ++++++++++++ .../test_model_dicts/test_log_entry_dicts.py | 42 ++ tests/test_model_dicts/test_message_dicts.py | 158 +++++ tests/test_model_dicts/test_meta_dicts.py | 133 +++++ .../test_meta_knowledge_graph_dicts.py | 195 +++++++ .../test_path_constraint_dicts.py | 54 ++ .../test_model_dicts/test_qualifier_dicts.py | 188 ++++++ .../test_query_graph_dicts.py | 445 +++++++++++++++ .../test_query_response_dicts.py | 104 ++++ tests/test_model_dicts/test_result_dicts.py | 181 ++++++ .../test_retrieval_source_dicts.py | 93 +++ .../test_workflow_operations_dicts.py | 157 +++++ uv.lock | 2 +- 41 files changed, 5064 insertions(+), 82 deletions(-) create mode 100644 bench/test_sd_tom_dicts.py create mode 100644 tests/test_model_dicts/test_analysis_dicts.py create mode 100644 tests/test_model_dicts/test_attribute_dicts.py create mode 100644 tests/test_model_dicts/test_auxiliary_graph_dicts.py create mode 100644 tests/test_model_dicts/test_binding_dicts.py create mode 100644 tests/test_model_dicts/test_knowledge_graph_dicts.py create mode 100644 tests/test_model_dicts/test_log_entry_dicts.py create mode 100644 tests/test_model_dicts/test_message_dicts.py create mode 100644 tests/test_model_dicts/test_meta_dicts.py create mode 100644 tests/test_model_dicts/test_meta_knowledge_graph_dicts.py create mode 100644 tests/test_model_dicts/test_path_constraint_dicts.py create mode 100644 tests/test_model_dicts/test_qualifier_dicts.py create mode 100644 tests/test_model_dicts/test_query_graph_dicts.py create mode 100644 tests/test_model_dicts/test_query_response_dicts.py create mode 100644 tests/test_model_dicts/test_result_dicts.py create mode 100644 tests/test_model_dicts/test_retrieval_source_dicts.py create mode 100644 tests/test_model_dicts/test_workflow_operations_dicts.py diff --git a/bench/test_sd_tom_dicts.py b/bench/test_sd_tom_dicts.py new file mode 100644 index 0000000..407c9fa --- /dev/null +++ b/bench/test_sd_tom_dicts.py @@ -0,0 +1,114 @@ +"""Dict-util-only serdes benchmark across every example response. + +The `model_dicts` twin of `bench/test_sd_tom.py`: same corpus walk and output, +but driving the `*DictUtil` serdes (raw orjson/ormsgpack over the TypedDict form, +no model construction) instead of the `Response` model. Run both to see the cost +the model layer adds over operating on plain dicts. +""" + +import time + +from utils import CORPUS_ROOT, discover_files, read_corpus_file + +LABEL_WIDTH = 10 +VALUE_FMT = "{:>8.4f}s" + + +def pair_row( + label: str, + from_s: float, + to_s: float, + bucket: dict[str, tuple[float, float]] | None = None, +) -> None: + """Print one line with from/to timings; record (from, to) into the bucket.""" + print( + f" {label:<{LABEL_WIDTH}}" + f" from {VALUE_FMT.format(from_s)}" + f" | to {VALUE_FMT.format(to_s)}" + ) + if bucket is not None: + bucket[label] = (from_s, to_s) + + +def section(title: str) -> None: + bar = "=" * (len(title) + 2) + print(f"\n{bar}\n {title}\n{bar}") + + +# --- Import --- + +t0 = time.perf_counter() +from translator_tom.model_dicts import ResponseDictUtil # noqa: E402 + +t_tom = time.perf_counter() - t0 + +section("Imports") +print(f" model_dicts ResponseDictUtil {VALUE_FMT.format(t_tom)}") + + +TEST_FILES = discover_files(CORPUS_ROOT) +print(f"\nDiscovered {len(TEST_FILES)} corpus file(s) under {CORPUS_ROOT}/") + +results: dict[str, dict[str, tuple[float, float]]] = {} + + +for response_path in TEST_FILES: + label = str(response_path.relative_to(CORPUS_ROOT)) + file_results: dict[str, tuple[float, float]] = {} + results[label] = file_results + + t0 = time.perf_counter() + response_json = read_corpus_file(response_path) + t_read = time.perf_counter() - t0 + size_mb = len(response_json.encode("utf-8")) / 1024 / 1024 + + section(f"{label} ({size_mb:.2f} MB JSON, {t_read:.4f}s)") + + t0 = time.perf_counter() + response = ResponseDictUtil.from_json(response_json) + t_from_json = time.perf_counter() - t0 + t0 = time.perf_counter() + _ = ResponseDictUtil.to_json(response) + t_to_json = time.perf_counter() - t0 + pair_row("json", t_from_json, t_to_json, file_results) + + t0 = time.perf_counter() + response_msgpack = ResponseDictUtil.to_msgpack(response) + t_to_mp = time.perf_counter() - t0 + t0 = time.perf_counter() + _ = ResponseDictUtil.from_msgpack(response_msgpack) + t_from_mp = time.perf_counter() - t0 + pair_row("msgpack", t_from_mp, t_to_mp, file_results) + + +# --- Summary --- + +section("Summary (seconds): from / to") + +short_labels = { + lbl: lbl.split("/")[-1].removesuffix(".gz").removesuffix(".json") + for lbl in results +} +ops = list(next(iter(results.values())).keys()) + + +def fmt_cell(v: tuple[float, float] | None) -> str: + return "—" if v is None else f"{v[0]:.4f} / {v[1]:.4f}" + + +# Transposed: files as rows, operations as columns (keeps the table narrow as +# more files are added to the corpus). +file_col = max(len(short_labels[lbl]) for lbl in results) +data_col = max( + max(len(op) for op in ops), + max(len(fmt_cell(results[lbl].get(op))) for lbl in results for op in ops), +) + +header = " " * file_col + " | " + " | ".join(f"{op:>{data_col}}" for op in ops) +print(header) +for lbl in results: + cells = [fmt_cell(results[lbl].get(op)) for op in ops] + print( + f"{short_labels[lbl]:<{file_col}} | " + + " | ".join(f"{c:>{data_col}}" for c in cells) + ) diff --git a/pyproject.toml b/pyproject.toml index 82c7561..1678770 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "translator_tom" -version = "1.4.1" +version = "1.5.0" description = "TRAPI Object Models: A performant python data model and centralized utilities for the Translator Reasoner API." readme = "README.md" requires-python = ">=3.10" diff --git a/src/translator_tom/model_dicts/__init__.py b/src/translator_tom/model_dicts/__init__.py index f5906f6..6801c09 100644 --- a/src/translator_tom/model_dicts/__init__.py +++ b/src/translator_tom/model_dicts/__init__.py @@ -2,49 +2,83 @@ __all__ = [ "AnalysisDict", + "AnalysisDictUtil", "AsyncQueryDict", "AsyncQueryResponseDict", "AsyncQueryStatusResponseDict", "AttributeConstraintDict", + "AttributeConstraintDictUtil", "AttributeDict", + "AttributeDictUtil", "AuxiliaryGraphDict", + "AuxiliaryGraphDictUtil", "BaseAnalysisDict", + "BaseAnalysisDictUtil", "BaseQueryGraphDict", "EdgeBindingDict", + "EdgeBindingDictUtil", "EdgeDict", + "EdgeDictUtil", "KnowledgeGraphDict", + "KnowledgeGraphDictUtil", "LogEntryDict", + "LogEntryDictUtil", "MessageDict", + "MessageDictUtil", "MetaAttributeDict", + "MetaAttributeDictUtil", "MetaEdgeDict", + "MetaEdgeDictUtil", "MetaKnowledgeGraphDict", + "MetaKnowledgeGraphDictUtil", "MetaNodeDict", + "MetaNodeDictUtil", "MetaQualifierDict", + "MetaQualifierDictUtil", "NodeBindingDict", + "NodeBindingDictUtil", "NodeDict", + "NodeDictUtil", "OperationDict", "PathBindingDict", + "PathBindingDictUtil", "PathConstraintDict", + "PathConstraintDictUtil", "PathfinderAnalysisDict", + "PathfinderAnalysisDictUtil", "PathfinderQueryGraphDict", + "PathfinderQueryGraphDictUtil", "QEdgeDict", + "QEdgeDictUtil", "QNodeDict", + "QNodeDictUtil", "QPathDict", + "QPathDictUtil", "QualifierConstraintDict", + "QualifierConstraintDictUtil", "QualifierDict", + "QualifierDictUtil", "QueryDict", + "QueryDictUtil", "QueryGraphDict", + "QueryGraphDictUtil", "ResponseDict", + "ResponseDictUtil", "ResultDict", + "ResultDictUtil", "RetrievalSourceDict", + "RetrievalSourceDictUtil", "workflow", ] from translator_tom.model_dicts import workflow_operations as workflow from translator_tom.model_dicts.analysis import ( AnalysisDict, + AnalysisDictUtil, BaseAnalysisDict, + BaseAnalysisDictUtil, PathfinderAnalysisDict, + PathfinderAnalysisDictUtil, ) from translator_tom.model_dicts.asyncquery import ( AsyncQueryDict, @@ -53,41 +87,80 @@ ) from translator_tom.model_dicts.attribute import ( AttributeConstraintDict, + AttributeConstraintDictUtil, AttributeDict, + AttributeDictUtil, +) +from translator_tom.model_dicts.auxiliary_graph import ( + AuxiliaryGraphDict, + AuxiliaryGraphDictUtil, +) +from translator_tom.model_dicts.edge_binding import ( + EdgeBindingDict, + EdgeBindingDictUtil, ) -from translator_tom.model_dicts.auxiliary_graph import AuxiliaryGraphDict -from translator_tom.model_dicts.edge_binding import EdgeBindingDict from translator_tom.model_dicts.knowledge_graph import ( EdgeDict, + EdgeDictUtil, KnowledgeGraphDict, + KnowledgeGraphDictUtil, NodeDict, + NodeDictUtil, +) +from translator_tom.model_dicts.log_entry import LogEntryDict, LogEntryDictUtil +from translator_tom.model_dicts.message import MessageDict, MessageDictUtil +from translator_tom.model_dicts.meta_attribute import ( + MetaAttributeDict, + MetaAttributeDictUtil, ) -from translator_tom.model_dicts.log_entry import LogEntryDict -from translator_tom.model_dicts.message import MessageDict -from translator_tom.model_dicts.meta_attribute import MetaAttributeDict from translator_tom.model_dicts.meta_knowledge_graph import ( MetaEdgeDict, + MetaEdgeDictUtil, MetaKnowledgeGraphDict, + MetaKnowledgeGraphDictUtil, MetaNodeDict, + MetaNodeDictUtil, +) +from translator_tom.model_dicts.meta_qualifier import ( + MetaQualifierDict, + MetaQualifierDictUtil, +) +from translator_tom.model_dicts.node_binding import ( + NodeBindingDict, + NodeBindingDictUtil, +) +from translator_tom.model_dicts.path_binding import ( + PathBindingDict, + PathBindingDictUtil, +) +from translator_tom.model_dicts.path_constraint import ( + PathConstraintDict, + PathConstraintDictUtil, ) -from translator_tom.model_dicts.meta_qualifier import MetaQualifierDict -from translator_tom.model_dicts.node_binding import NodeBindingDict -from translator_tom.model_dicts.path_binding import PathBindingDict -from translator_tom.model_dicts.path_constraint import PathConstraintDict from translator_tom.model_dicts.qualifier import ( QualifierConstraintDict, + QualifierConstraintDictUtil, QualifierDict, + QualifierDictUtil, ) -from translator_tom.model_dicts.query import QueryDict +from translator_tom.model_dicts.query import QueryDict, QueryDictUtil from translator_tom.model_dicts.query_graph import ( BaseQueryGraphDict, PathfinderQueryGraphDict, + PathfinderQueryGraphDictUtil, QEdgeDict, + QEdgeDictUtil, QNodeDict, + QNodeDictUtil, QPathDict, + QPathDictUtil, QueryGraphDict, + QueryGraphDictUtil, +) +from translator_tom.model_dicts.response import ResponseDict, ResponseDictUtil +from translator_tom.model_dicts.result import ResultDict, ResultDictUtil +from translator_tom.model_dicts.retrieval_source import ( + RetrievalSourceDict, + RetrievalSourceDictUtil, ) -from translator_tom.model_dicts.response import ResponseDict -from translator_tom.model_dicts.result import ResultDict -from translator_tom.model_dicts.retrieval_source import RetrievalSourceDict from translator_tom.model_dicts.workflow_operations import OperationDict diff --git a/src/translator_tom/model_dicts/analysis.py b/src/translator_tom/model_dicts/analysis.py index f21d99c..46feb69 100644 --- a/src/translator_tom/model_dicts/analysis.py +++ b/src/translator_tom/model_dicts/analysis.py @@ -1,16 +1,31 @@ from __future__ import annotations +import copy +from collections.abc import Mapping + from typing_extensions import NotRequired, TypedDict -from translator_tom.model_dicts.attribute import AttributeDict -from translator_tom.model_dicts.edge_binding import EdgeBindingDict -from translator_tom.model_dicts.path_binding import PathBindingDict +from translator_tom.model_dicts.attribute import AttributeDict, AttributeDictUtil +from translator_tom.model_dicts.edge_binding import ( + EdgeBindingDict, + EdgeBindingDictUtil, +) +from translator_tom.model_dicts.path_binding import ( + PathBindingDict, + PathBindingDictUtil, +) +from translator_tom.models.analysis import Analysis, BaseAnalysis, PathfinderAnalysis from translator_tom.models.shared import CURIE, AuxGraphID, QEdgeID, QPathID +from translator_tom.utils.dict_util_base import DictUtil, register_union_discriminator +from translator_tom.utils.hash import tomhash __all__ = [ "AnalysisDict", + "AnalysisDictUtil", "BaseAnalysisDict", + "BaseAnalysisDictUtil", "PathfinderAnalysisDict", + "PathfinderAnalysisDictUtil", ] @@ -22,9 +37,139 @@ class BaseAnalysisDict(TypedDict): attributes: NotRequired[list[AttributeDict] | None] +def _update_base(analysis: BaseAnalysisDict, other: BaseAnalysisDict) -> None: + """Merge the shared BaseAnalysis fields (attributes, support graphs) in-place.""" + analysis_attrs = analysis.get("attributes") + other_attrs = other.get("attributes") + if (not analysis_attrs) and other_attrs: + analysis["attributes"] = other_attrs + elif analysis_attrs and other_attrs: + AttributeDictUtil.merge_attribute_lists(analysis_attrs, other_attrs) + + analysis_sg = analysis.get("support_graphs") + other_sg = other.get("support_graphs") + if (not analysis_sg) and other_sg: + analysis["support_graphs"] = other_sg + elif analysis_sg and other_sg: + analysis["support_graphs"] = list(set(analysis_sg) | set(other_sg)) + + +class BaseAnalysisDictUtil(DictUtil[BaseAnalysisDict]): + """Utility methods for `BaseAnalysisDict`, mirroring those on the `BaseAnalysis` model.""" + + _model = BaseAnalysis + + @staticmethod + def support_graphs_list(analysis: BaseAnalysisDict) -> list[AuxGraphID]: + """Get the support graphs as a guaranteed list, even if they are represented as None.""" + support_graphs = analysis.get("support_graphs") + return support_graphs if support_graphs is not None else [] + + @staticmethod + def attributes_list(analysis: BaseAnalysisDict) -> list[AttributeDict]: + """Get the attributes as a guaranteed list, even if they are represented as None.""" + attributes = analysis.get("attributes") + return attributes if attributes is not None else [] + + @classmethod + def hash(cls, obj: BaseAnalysisDict) -> str: + """Hash matching `BaseAnalysis.hash` (resource, score, support graphs, method).""" + return tomhash( + ( + obj["resource_id"], + obj.get("score"), + frozenset(cls.support_graphs_list(obj)), + obj.get("scoring_method"), + ) + ) + + class AnalysisDict(BaseAnalysisDict): edge_bindings: dict[QEdgeID, list[EdgeBindingDict]] +class AnalysisDictUtil(DictUtil[AnalysisDict]): + """Utility methods for `AnalysisDict`, mirroring those on the `Analysis` model.""" + + _model = Analysis + + @classmethod + def hash(cls, obj: AnalysisDict) -> str: + """Hash matching `Analysis.hash` (base analysis plus edge bindings).""" + return tomhash( + ( + BaseAnalysisDictUtil.hash(obj), + { + qedge_id: frozenset(EdgeBindingDictUtil.hash(b) for b in bindings) + for qedge_id, bindings in obj["edge_bindings"].items() + }, + ) + ) + + @staticmethod + def update(analysis: AnalysisDict, other: AnalysisDict) -> None: + """Update the analysis in-place with another analysis.""" + _update_base(analysis, other) + for k in other["edge_bindings"]: + if k in analysis["edge_bindings"]: + # Dedupe by hash, existing bindings win + merged = { + EdgeBindingDictUtil.hash(b): b for b in analysis["edge_bindings"][k] + } + for b in other["edge_bindings"][k]: + merged.setdefault(EdgeBindingDictUtil.hash(b), copy.deepcopy(b)) + analysis["edge_bindings"][k] = list(merged.values()) + else: + analysis["edge_bindings"][k] = copy.deepcopy(other["edge_bindings"][k]) + + class PathfinderAnalysisDict(BaseAnalysisDict): path_bindings: dict[QPathID, list[PathBindingDict]] + + +class PathfinderAnalysisDictUtil(DictUtil[PathfinderAnalysisDict]): + """Utility methods for `PathfinderAnalysisDict`, mirroring the `PathfinderAnalysis` model.""" + + _model = PathfinderAnalysis + + @classmethod + def hash(cls, obj: PathfinderAnalysisDict) -> str: + """Hash matching `PathfinderAnalysis.hash` (base analysis plus path bindings).""" + return tomhash( + ( + BaseAnalysisDictUtil.hash(obj), + { + qpath_id: frozenset(PathBindingDictUtil.hash(b) for b in bindings) + for qpath_id, bindings in obj["path_bindings"].items() + }, + ) + ) + + @staticmethod + def update(analysis: PathfinderAnalysisDict, other: PathfinderAnalysisDict) -> None: + """Update the analysis in-place with another analysis.""" + _update_base(analysis, other) + for k in other["path_bindings"]: + if k in analysis["path_bindings"]: + # Dedupe by hash, existing bindings win + merged = { + PathBindingDictUtil.hash(b): b for b in analysis["path_bindings"][k] + } + for b in other["path_bindings"][k]: + merged.setdefault(PathBindingDictUtil.hash(b), copy.deepcopy(b)) + analysis["path_bindings"][k] = list(merged.values()) + else: + analysis["path_bindings"][k] = copy.deepcopy(other["path_bindings"][k]) + + +def _discriminate_analysis( + value: Mapping[str, object], +) -> type[Analysis | PathfinderAnalysis]: + """Pick the concrete analysis model for a raw dict (`path_bindings` -> Pathfinder).""" + return PathfinderAnalysis if "path_bindings" in value else Analysis + + +# `Result.analyses` is an `Analysis | PathfinderAnalysis` union with no pydantic +# discriminator. `Result.hash` ignores `analyses`, so this isn't hit by base hashing +# today, but register it so any future base-hashed use resolves correctly. +register_union_discriminator((Analysis, PathfinderAnalysis), _discriminate_analysis) diff --git a/src/translator_tom/model_dicts/attribute.py b/src/translator_tom/model_dicts/attribute.py index 66dfe9b..cfd8b37 100644 --- a/src/translator_tom/model_dicts/attribute.py +++ b/src/translator_tom/model_dicts/attribute.py @@ -1,14 +1,31 @@ from __future__ import annotations +import re +from typing import cast + from pydantic import JsonValue from typing_extensions import NotRequired, TypedDict -from translator_tom.models.attribute import Operator +from translator_tom.model_dicts.meta_attribute import ( + MetaAttributeDict, + MetaAttributeDictUtil, +) +from translator_tom.models.attribute import ( + _OBJECT_RE, + _SUBJECT_RE, + Attribute, + AttributeConstraint, + Operator, +) from translator_tom.models.shared import CURIE +from translator_tom.utils.dict_util_base import DictUtil +from translator_tom.utils.hash import tomhash __all__ = [ "AttributeConstraintDict", + "AttributeConstraintDictUtil", "AttributeDict", + "AttributeDictUtil", ] @@ -23,16 +40,170 @@ class AttributeDict(TypedDict): attributes: NotRequired[list[AttributeDict] | None] +class AttributeDictUtil(DictUtil[AttributeDict]): + """Utility methods for `AttributeDict`, mirroring those on the `Attribute` model.""" + + _model = Attribute + + @staticmethod + def attributes_list(attribute: AttributeDict) -> list[AttributeDict]: + """Get the sub-attributes as a guaranteed list, even if they are represented as None.""" + attributes = attribute.get("attributes") + return attributes if attributes is not None else [] + + @classmethod + def hash(cls, obj: AttributeDict) -> str: + """Hash matching `Attribute.hash` (scalar fields plus nested sub-attributes).""" + return tomhash( + ( + obj["attribute_type_id"], + obj.get("original_attribute_name"), + obj["value"], + obj.get("value_type_id"), + obj.get("attribute_source"), + obj.get("value_url"), + obj.get("description"), + frozenset(cls.hash(a) for a in cls.attributes_list(obj)), + ) + ) + + @staticmethod + def merge_attribute_lists( + old: list[AttributeDict], new: list[AttributeDict] + ) -> None: + """Merge the new attributes into the existing attributes.""" + attrs = {AttributeDictUtil.hash(attr): attr for attr in old} + for attr in new: + attrs[AttributeDictUtil.hash(attr)] = attr + + old.clear() + old.extend(attrs.values()) + + # Functional syntax so the `not` alias (a Python keyword) can be used as a key. AttributeConstraintDict = TypedDict( "AttributeConstraintDict", { "id": CURIE, "name": str, - "not": NotRequired[bool | None], + "not": NotRequired[bool], "operator": Operator, "value": JsonValue, "unit_id": NotRequired[CURIE | None], "unit_name": NotRequired[str | None], }, ) + + +class AttributeConstraintDictUtil(DictUtil[AttributeConstraintDict]): + """Utility methods for `AttributeConstraintDict`, mirroring those on the `AttributeConstraint` model.""" + + _model = AttributeConstraint + + @classmethod + def hash(cls, obj: AttributeConstraintDict) -> str: + """Hash matching `AttributeConstraint.hash` (declared scalar fields only).""" + return tomhash( + ( + obj["id"], + obj["name"], + obj.get("not", cls._default("negated")), + obj["operator"], + obj["value"], + obj.get("unit_id"), + obj.get("unit_name"), + ) + ) + + @staticmethod + def get_inverse(constraint: AttributeConstraintDict) -> AttributeConstraintDict: + """Return a (SPO) inverse of the constraint, for reversing edges. + + Flips subject/object for the few directional attribute types. + """ + inverted = cast("AttributeConstraintDict", {**constraint}) + cid = constraint["id"] + name = constraint["name"] + if _OBJECT_RE.search(cid): + inverted["id"] = _OBJECT_RE.sub("subject", cid) + inverted["name"] = _OBJECT_RE.sub("subject", name) + elif _SUBJECT_RE.search(cid): + inverted["id"] = _SUBJECT_RE.sub("object", cid) + inverted["name"] = _SUBJECT_RE.sub("object", name) + return inverted + + @staticmethod + def met_by( + constraint: AttributeConstraintDict, + attribute: AttributeDict | MetaAttributeDict, + ) -> bool: + """Check if the given attribute satisfies the constraint.""" + # A MetaAttributeDict has no `value` key (required on an AttributeDict). + if "value" not in attribute: + return ( + constraint["id"] == attribute["attribute_type_id"] + and attribute.get( + "constraint_use", MetaAttributeDictUtil._default("constraint_use") + ) + is not False + ) + + attr = cast("AttributeDict", attribute) + if constraint["id"] != attr["attribute_type_id"]: + return False + + operator = constraint["operator"] + con_value = constraint["value"] + if operator == "===": + result = attr["value"] == con_value + else: + attr_vals = ( + attr["value"] if isinstance(attr["value"], list) else [attr["value"]] + ) + con_vals = con_value if isinstance(con_value, list) else [con_value] + + match operator: + case "==": + result = any(av == cv for av in attr_vals for cv in con_vals) + case ">" | "<": + result = any( + (av > cv if operator == ">" else av < cv) + for av in attr_vals + for cv in con_vals + if isinstance(av, int | float) and isinstance(cv, int | float) + ) + case "matches": + result = any( + bool(re.search(cv, av)) + for cv in con_vals + for av in attr_vals + if isinstance(cv, str) and isinstance(av, str) + ) + + return ( + not result + if constraint.get("not", AttributeConstraintDictUtil._default("negated")) + else result + ) + + @staticmethod + def set_met_by( + constraints: list[AttributeConstraintDict], + attributes: list[AttributeDict] | list[MetaAttributeDict], + ) -> bool: + """Check if the given set of constraints are met by the given attributes.""" + if len(constraints) == 0: + return True + elif len(attributes) == 0: + return False + + attrs_by_type: dict[CURIE, list[AttributeDict | MetaAttributeDict]] = {} + for attr in attributes: + attrs_by_type.setdefault(attr["attribute_type_id"], []).append(attr) + return all( + any( + AttributeConstraintDictUtil.met_by(c, attr) + for attr in attrs_by_type.get(c["id"], []) + ) + for c in constraints + ) diff --git a/src/translator_tom/model_dicts/auxiliary_graph.py b/src/translator_tom/model_dicts/auxiliary_graph.py index 371299f..186429b 100644 --- a/src/translator_tom/model_dicts/auxiliary_graph.py +++ b/src/translator_tom/model_dicts/auxiliary_graph.py @@ -2,12 +2,70 @@ from typing_extensions import TypedDict -from translator_tom.model_dicts.attribute import AttributeDict -from translator_tom.models.shared import EdgeID +from translator_tom.model_dicts.attribute import AttributeDict, AttributeDictUtil +from translator_tom.models.auxiliary_graph import AuxiliaryGraph +from translator_tom.models.shared import AuxGraphID, EdgeID +from translator_tom.utils.dict_util_base import DictUtil +from translator_tom.utils.hash import tomhash -__all__ = ["AuxiliaryGraphDict"] +__all__ = ["AuxiliaryGraphDict", "AuxiliaryGraphDictUtil"] class AuxiliaryGraphDict(TypedDict): edges: list[EdgeID] attributes: list[AttributeDict] + + +AuxiliaryGraphsDict = dict[AuxGraphID, AuxiliaryGraphDict] + + +class AuxiliaryGraphDictUtil(DictUtil[AuxiliaryGraphDict]): + """Utility methods for `AuxiliaryGraphDict`, mirroring those on the `AuxiliaryGraph` model.""" + + _model = AuxiliaryGraph + + @classmethod + def hash(cls, obj: AuxiliaryGraphDict) -> str: + """Hash matching `AuxiliaryGraph.hash` (unordered edges plus attributes).""" + return tomhash( + ( + frozenset(obj["edges"]), + frozenset(AttributeDictUtil.hash(a) for a in obj["attributes"]), + ) + ) + + @staticmethod + def normalize( + auxiliary_graph: AuxiliaryGraphDict, mapping: dict[EdgeID, EdgeID] + ) -> None: + """Normalize the auxiliary graph given a mapping of old:new EdgeIDs.""" + auxiliary_graph["edges"] = [ + mapping.get(edge_id, edge_id) for edge_id in auxiliary_graph["edges"] + ] + + @staticmethod + def normalize_aux_dict( + auxiliary_graphs_dict: AuxiliaryGraphsDict, mapping: dict[EdgeID, EdgeID] + ) -> None: + """Normalize an AuxiliaryGraphsDict given a mapping of old:new EdgeIDs.""" + for auxg in auxiliary_graphs_dict.values(): + AuxiliaryGraphDictUtil.normalize(auxg, mapping) + + @staticmethod + def update(auxiliary_graph: AuxiliaryGraphDict, other: AuxiliaryGraphDict) -> None: + """Update the auxiliary graph in-place using the other.""" + if (not auxiliary_graph["attributes"]) and other["attributes"]: + auxiliary_graph["attributes"] = other["attributes"] + elif auxiliary_graph["attributes"] and other["attributes"]: + AttributeDictUtil.merge_attribute_lists( + auxiliary_graph["attributes"], other["attributes"] + ) + + @staticmethod + def merge_dictionaries(old: AuxiliaryGraphsDict, new: AuxiliaryGraphsDict) -> None: + """Merge the new auxiliary graphs into the existing auxiliary graphs.""" + for aux_id, graph in new.items(): + if aux_id in old: + AuxiliaryGraphDictUtil.update(old[aux_id], graph) + else: + old[aux_id] = graph diff --git a/src/translator_tom/model_dicts/edge_binding.py b/src/translator_tom/model_dicts/edge_binding.py index 3818d02..8dc8dc2 100644 --- a/src/translator_tom/model_dicts/edge_binding.py +++ b/src/translator_tom/model_dicts/edge_binding.py @@ -2,12 +2,28 @@ from typing_extensions import TypedDict -from translator_tom.model_dicts.attribute import AttributeDict +from translator_tom.model_dicts.attribute import AttributeDict, AttributeDictUtil +from translator_tom.models.edge_binding import EdgeBinding from translator_tom.models.shared import EdgeID +from translator_tom.utils.dict_util_base import DictUtil +from translator_tom.utils.hash import tomhash -__all__ = ["EdgeBindingDict"] +__all__ = ["EdgeBindingDict", "EdgeBindingDictUtil"] class EdgeBindingDict(TypedDict): id: EdgeID attributes: list[AttributeDict] + + +class EdgeBindingDictUtil(DictUtil[EdgeBindingDict]): + """Utility methods for `EdgeBindingDict`, mirroring those on the `EdgeBinding` model.""" + + _model = EdgeBinding + + @classmethod + def hash(cls, obj: EdgeBindingDict) -> str: + """Hash matching `EdgeBinding.hash` (bound edge id plus its attributes).""" + return tomhash( + (obj["id"], frozenset(AttributeDictUtil.hash(a) for a in obj["attributes"])) + ) diff --git a/src/translator_tom/model_dicts/knowledge_graph.py b/src/translator_tom/model_dicts/knowledge_graph.py index 032c8c7..5732a52 100644 --- a/src/translator_tom/model_dicts/knowledge_graph.py +++ b/src/translator_tom/model_dicts/knowledge_graph.py @@ -1,17 +1,43 @@ from __future__ import annotations +import itertools +from typing import Literal, cast + from typing_extensions import NotRequired, TypedDict -from translator_tom.model_dicts.attribute import AttributeDict -from translator_tom.model_dicts.qualifier import QualifierDict -from translator_tom.model_dicts.retrieval_source import RetrievalSourceDict -from translator_tom.models.shared import CURIE, EdgeID +from translator_tom.model_dicts.analysis import AnalysisDict +from translator_tom.model_dicts.attribute import ( + AttributeConstraintDict, + AttributeConstraintDictUtil, + AttributeDict, + AttributeDictUtil, +) +from translator_tom.model_dicts.auxiliary_graph import AuxiliaryGraphsDict +from translator_tom.model_dicts.qualifier import ( + QualifierConstraintDict, + QualifierConstraintDictUtil, + QualifierDict, + QualifierDictUtil, +) +from translator_tom.model_dicts.result import ResultDict +from translator_tom.model_dicts.retrieval_source import ( + RetrievalSourceDict, + RetrievalSourceDictUtil, +) +from translator_tom.models.knowledge_graph import Edge, KnowledgeGraph, Node +from translator_tom.models.retrieval_source import ResourceRoleEnum +from translator_tom.models.shared import CURIE, AuxGraphID, EdgeID, Infores from translator_tom.utils.biolink import Biolink +from translator_tom.utils.dict_util_base import DictUtil +from translator_tom.utils.hash import tomhash __all__ = [ "EdgeDict", + "EdgeDictUtil", "KnowledgeGraphDict", + "KnowledgeGraphDictUtil", "NodeDict", + "NodeDictUtil", ] @@ -22,6 +48,37 @@ class NodeDict(TypedDict): is_set: NotRequired[bool | None] +class NodeDictUtil(DictUtil[NodeDict]): + """Utility methods for `NodeDict`, mirroring those on the `Node` model.""" + + _model = Node + + @classmethod + def hash(cls, obj: NodeDict) -> str: + """Hash matching `Node.hash` (identity by name and is_set only).""" + # Categories/attributes are excluded: a node's identity is really its KG key. + return tomhash((obj.get("name"), obj.get("is_set"))) + + @staticmethod + def meets_constraints( + node: NodeDict, constraints: list[AttributeConstraintDict] + ) -> bool: + """Check if all constraints are satisfied by the node's attributes.""" + return AttributeConstraintDictUtil.set_met_by(constraints, node["attributes"]) + + @staticmethod + def update(node: NodeDict, other: NodeDict) -> None: + """Update the node in-place with another node.""" + node["name"] = other.get("name") or node.get("name") + node["categories"] = list(set(node["categories"]) | set(other["categories"])) + + if other["attributes"]: + attrs = {AttributeDictUtil.hash(attr): attr for attr in node["attributes"]} + for attr in other["attributes"]: + attrs[AttributeDictUtil.hash(attr)] = attr + node["attributes"] = list(attrs.values()) + + class EdgeDict(TypedDict): predicate: Biolink.Predicate subject: CURIE @@ -31,6 +88,300 @@ class EdgeDict(TypedDict): sources: list[RetrievalSourceDict] +class EdgeDictUtil(DictUtil[EdgeDict]): + """Utility methods for `EdgeDict`, mirroring those on the `Edge` model.""" + + _model = Edge + + @staticmethod + def attributes_list(edge: EdgeDict) -> list[AttributeDict]: + """Get the attributes as a guaranteed list, even if they are represented as None.""" + attributes = edge.get("attributes") + return attributes if attributes is not None else [] + + @staticmethod + def qualifiers_list(edge: EdgeDict) -> list[QualifierDict]: + """Get the qualifiers as a guaranteed list, even if they are represented as None.""" + qualifiers = edge.get("qualifiers") + return qualifiers if qualifiers is not None else [] + + @staticmethod + def primary_knowledge_source(edge: EdgeDict) -> RetrievalSourceDict: + """The primary knowledge source of the edge.""" + for source in edge["sources"]: + if source["resource_role"] == ResourceRoleEnum.primary_knowledge_source: + return source + + raise ValueError( + f"Edge {edge['subject']} -{edge['predicate']}-> {edge['object']} has no " + "primary_knowledge_source!" + ) + + @staticmethod + def last_downstream_source(edge: EdgeDict) -> RetrievalSourceDict | None: + """Get the last/most downstream source in the chain.""" + upstreams = set( + itertools.chain( + *[ + source.get("upstream_resource_ids") or [] + for source in edge["sources"] + ] + ) + ) + return next( + iter( + source + for source in edge["sources"] + if source["resource_id"] not in upstreams + ), + None, + ) + + @staticmethod + def is_self_edge(edge: EdgeDict) -> bool: + """Check if the edge is a self-edge.""" + return edge["subject"] == edge["object"] + + @staticmethod + def support_graphs(edge: EdgeDict) -> list[AuxGraphID]: + """Get the support graph IDs referenced by this edge.""" + support_graphs = list[AuxGraphID]() + for attr in EdgeDictUtil.attributes_list(edge): + if attr["attribute_type_id"] == Biolink("support_graphs"): + support_graphs.extend(cast("list[AuxGraphID]", attr["value"])) + return support_graphs + + @classmethod + def hash(cls, obj: EdgeDict) -> str: + """Hash matching `Edge.hash` (subject/object/predicate, qualifiers, primary KS).""" + return tomhash( + ( + obj["subject"], + obj["object"], + obj["predicate"], + frozenset(QualifierDictUtil.hash(q) for q in cls.qualifiers_list(obj)), + cls.primary_knowledge_source(obj)["resource_id"], + ) + ) + + @staticmethod + def update(edge: EdgeDict, other: EdgeDict) -> None: + """Update the edge in-place with another edge.""" + edge_attrs = edge.get("attributes") + other_attrs = other.get("attributes") + if (not edge_attrs) and other_attrs: + edge["attributes"] = other_attrs + elif edge_attrs and other_attrs: + attrs = {AttributeDictUtil.hash(attr): attr for attr in edge_attrs} + kl_at = (Biolink("knowledge_level"), Biolink("agent_type")) + for attr in other_attrs: + # Avoid multiple KL/AT + if attr["attribute_type_id"] in kl_at: + continue + attrs[AttributeDictUtil.hash(attr)] = attr + edge["attributes"] = list(attrs.values()) + + if (not edge["sources"]) and other["sources"]: + edge["sources"] = other["sources"] + elif edge["sources"] and other["sources"]: + sources = { + RetrievalSourceDictUtil.hash(source): source + for source in edge["sources"] + } + new_sources = { + RetrievalSourceDictUtil.hash(source): source + for source in other["sources"] + } + + # Roll in upstream_resource_ids from overlapping sources; merge into + # new_source since it replaces the old one below. + for source_hash, source in sources.items(): + new_source = new_sources.get(source_hash) + if new_source is not None: + RetrievalSourceDictUtil.update(new_source, source) + sources.update(new_sources) + edge["sources"] = list(sources.values()) + + @staticmethod + def meets_attribute_constraints( + edge: EdgeDict, constraints: list[AttributeConstraintDict] + ) -> bool: + """Check if all attribute constraints are satisfied by the edge's attributes.""" + return AttributeConstraintDictUtil.set_met_by( + constraints, EdgeDictUtil.attributes_list(edge) + ) + + @staticmethod + def meets_qualifier_constraints( + edge: EdgeDict, constraints: list[QualifierConstraintDict] + ) -> bool: + """Check if the edge satisfies the qualifier constraints.""" + return QualifierConstraintDictUtil.set_met_by( + constraints, EdgeDictUtil.qualifiers_list(edge) + ) + + @staticmethod + def append_aggregator(edge: EdgeDict, source: Infores) -> None: + """Append an aggregator source to the present chain with appropriate upstreams.""" + last_downstream = EdgeDictUtil.last_downstream_source(edge) + if last_downstream is None: + raise ValueError("Provenance chain is invalid.") + edge["sources"].append( + { + "resource_id": source, + "resource_role": "aggregator_knowledge_source", + "upstream_resource_ids": [last_downstream["resource_id"]], + } + ) + + class KnowledgeGraphDict(TypedDict): nodes: dict[CURIE, NodeDict] edges: dict[EdgeID, EdgeDict] + + +class KnowledgeGraphDictUtil(DictUtil[KnowledgeGraphDict]): + """Utility methods for `KnowledgeGraphDict`, mirroring those on the `KnowledgeGraph` model.""" + + _model = KnowledgeGraph + + @staticmethod + def new() -> KnowledgeGraphDict: + """Return an empty instance, without having to pass required containers.""" + return {"nodes": {}, "edges": {}} + + @staticmethod + def normalize(knowledge_graph: KnowledgeGraphDict) -> dict[EdgeID, EdgeID]: + """Normalize the kgraph edge IDs and return a mapping of old:new.""" + mapping = dict[EdgeID, EdgeID]() + + for edge_id in list(knowledge_graph["edges"].keys()): + edge = knowledge_graph["edges"].pop(edge_id) + new_id = EdgeDictUtil.hash(edge) + mapping[edge_id] = new_id + knowledge_graph["edges"][new_id] = edge + + return mapping + + @staticmethod + def update( + knowledge_graph: KnowledgeGraphDict, + other: KnowledgeGraphDict, + pre_normalized: Literal["neither", "both", "self", "other"] = "neither", + ) -> dict[EdgeID, EdgeID]: + """Update the kgraph in-place using the other. + + Args: + knowledge_graph: The kgraph to update. + other: The other kgraph. + pre_normalized: Option to call out pre-normalized KGs to skip redundant normalization. + + Returns: + A mapping of old:new EdgeIDs if normalization was done. + """ + mapping = dict[EdgeID, EdgeID]() + if pre_normalized in ("neither", "other"): + mapping.update(KnowledgeGraphDictUtil.normalize(knowledge_graph)) + if pre_normalized in ("neither", "self"): + # Normalize a shallow copy of the other dict so as not to modify the original. + other = {"nodes": dict(other["nodes"]), "edges": dict(other["edges"])} + mapping.update(KnowledgeGraphDictUtil.normalize(other)) + + for node_id, node in other["nodes"].items(): + if node_id in knowledge_graph["nodes"]: + NodeDictUtil.update(knowledge_graph["nodes"][node_id], node) + continue + knowledge_graph["nodes"][node_id] = node + + for edge_id, edge in other["edges"].items(): + if edge_id in knowledge_graph["edges"]: + EdgeDictUtil.update(knowledge_graph["edges"][edge_id], edge) + continue + knowledge_graph["edges"][edge_id] = edge + + return mapping + + @staticmethod + def _walk_results( + aux_graphs: AuxiliaryGraphsDict, results: list[ResultDict] + ) -> tuple[set[EdgeID], set[CURIE]]: + """Walk results to find immediately bound edges and nodes.""" + bound_edges = set[EdgeID]() + bound_nodes = set[CURIE]() + for result in results: + for node_binding_set in result["node_bindings"].values(): + bound_nodes.update(binding["id"] for binding in node_binding_set) + for analysis in result["analyses"]: + for aux_id in analysis.get("support_graphs") or []: + bound_edges.update(aux_graphs[aux_id]["edges"]) + if "edge_bindings" in analysis: + analysis = cast("AnalysisDict", analysis) + for edge_binding_set in analysis["edge_bindings"].values(): + bound_edges.update( + binding["id"] for binding in edge_binding_set + ) + else: + for path_binding in itertools.chain( + *(analysis["path_bindings"].values()) + ): + if path_binding["id"] in aux_graphs: + bound_edges.update(aux_graphs[path_binding["id"]]["edges"]) + return bound_edges, bound_nodes + + @staticmethod + def prune( + knowledge_graph: KnowledgeGraphDict, + aux_graphs: AuxiliaryGraphsDict, + results: list[ResultDict], + ) -> None: + """Remove any unused nodes or edges. + + Args: + knowledge_graph: The kgraph to prune. + aux_graphs: Auxiliary graphs using this KG. + results: Results list using this KG. + + Raises: + KeyError: If nodes/edges are referenced that aren't present in the KG. + """ + bound_edges, bound_nodes = KnowledgeGraphDictUtil._walk_results( + aux_graphs, results + ) + + checked_edges = set[EdgeID]() + edges_to_check = list(bound_edges) + while len(edges_to_check) > 0: + edge_id = edges_to_check.pop() + + # Avoid infinite loops if edge and aux graph reference each other + if edge_id in checked_edges: + continue + checked_edges.add(edge_id) + + edge = knowledge_graph["edges"][edge_id] + + bound_edges.add(edge_id) + bound_nodes.add(edge["subject"]) + bound_nodes.add(edge["object"]) + + edge_aux_graphs = next( + ( + attr + for attr in EdgeDictUtil.attributes_list(edge) + if attr["attribute_type_id"] == "biolink:support_graphs" + ), + None, + ) + if edge_aux_graphs is None: + continue + # Support graphs always have a value of type list[str], but the attribute + # value is generally typed Any. + for aux_graph_id in cast("list[str]", edge_aux_graphs["value"]): + edges_to_check.extend(aux_graphs[aux_graph_id]["edges"]) + + knowledge_graph["edges"] = { + edge_id: knowledge_graph["edges"][edge_id] for edge_id in bound_edges + } + knowledge_graph["nodes"] = { + curie: knowledge_graph["nodes"][curie] for curie in bound_nodes + } diff --git a/src/translator_tom/model_dicts/log_entry.py b/src/translator_tom/model_dicts/log_entry.py index 9a9a74f..35bed7c 100644 --- a/src/translator_tom/model_dicts/log_entry.py +++ b/src/translator_tom/model_dicts/log_entry.py @@ -1,10 +1,13 @@ from __future__ import annotations +import datetime + from typing_extensions import NotRequired, TypedDict -from translator_tom.models.log_entry import LogLevel +from translator_tom.models.log_entry import LogEntry, LogLevel +from translator_tom.utils.dict_util_base import DictUtil -__all__ = ["LogEntryDict"] +__all__ = ["LogEntryDict", "LogEntryDictUtil"] class LogEntryDict(TypedDict): @@ -12,3 +15,33 @@ class LogEntryDict(TypedDict): level: NotRequired[LogLevel | None] code: NotRequired[str | None] message: str + + +class LogEntryDictUtil(DictUtil[LogEntryDict]): + """Utility methods for `LogEntryDict`, mirroring those on the `LogEntry` model.""" + + _model = LogEntry + + @staticmethod + def timestamp_dt(log_entry: LogEntryDict) -> datetime.datetime: + """Return the timestamp parsed as a timezone-aware `datetime`.""" + # datetime.fromisoformat() only accepts 'Z' as of Python 3.11; normalize for 3.10. + ts = log_entry["timestamp"] + if ts.endswith("Z"): + ts = f"{ts[:-1]}+00:00" + return datetime.datetime.fromisoformat(ts) + + @staticmethod + def new( + message: str, level: LogLevel | None = None, code: str | None = None + ) -> LogEntryDict: + """Return a new LogEntry dict with a timestamp from now.""" + log_entry: LogEntryDict = { + "timestamp": datetime.datetime.now().astimezone().isoformat(), + "message": message, + } + if level is not None: + log_entry["level"] = level + if code is not None: + log_entry["code"] = code + return log_entry diff --git a/src/translator_tom/model_dicts/message.py b/src/translator_tom/model_dicts/message.py index 9ea5e7e..4827dd2 100644 --- a/src/translator_tom/model_dicts/message.py +++ b/src/translator_tom/model_dicts/message.py @@ -1,17 +1,31 @@ from __future__ import annotations +import copy +from typing import Literal, cast + from typing_extensions import NotRequired, TypedDict -from translator_tom.model_dicts.auxiliary_graph import AuxiliaryGraphDict -from translator_tom.model_dicts.knowledge_graph import KnowledgeGraphDict +from translator_tom.model_dicts.auxiliary_graph import ( + AuxiliaryGraphDict, + AuxiliaryGraphDictUtil, + AuxiliaryGraphsDict, +) +from translator_tom.model_dicts.knowledge_graph import ( + KnowledgeGraphDict, + KnowledgeGraphDictUtil, +) from translator_tom.model_dicts.query_graph import ( PathfinderQueryGraphDict, + PathfinderQueryGraphDictUtil, QueryGraphDict, + QueryGraphDictUtil, ) -from translator_tom.model_dicts.result import ResultDict -from translator_tom.models.shared import AuxGraphID +from translator_tom.model_dicts.result import ResultDict, ResultDictUtil +from translator_tom.models.message import Message +from translator_tom.models.shared import AuxGraphID, EdgeID +from translator_tom.utils.dict_util_base import DictUtil -__all__ = ["MessageDict"] +__all__ = ["MessageDict", "MessageDictUtil"] class MessageDict(TypedDict): @@ -19,3 +33,109 @@ class MessageDict(TypedDict): query_graph: NotRequired[QueryGraphDict | PathfinderQueryGraphDict | None] knowledge_graph: NotRequired[KnowledgeGraphDict | None] auxiliary_graphs: NotRequired[dict[AuxGraphID, AuxiliaryGraphDict] | None] + + +def _query_graph_hash( + query_graph: QueryGraphDict | PathfinderQueryGraphDict | None, +) -> str | None: + """Hash a query graph for identity comparison, ignoring extra keys (like model `==`).""" + if query_graph is None: + return None + if "paths" in query_graph: + return PathfinderQueryGraphDictUtil.hash( + cast("PathfinderQueryGraphDict", query_graph) + ) + return QueryGraphDictUtil.hash(query_graph) + + +class MessageDictUtil(DictUtil[MessageDict]): + """Utility methods for `MessageDict`, mirroring those on the `Message` model.""" + + _model = Message + + @staticmethod + def results_list(message: MessageDict) -> list[ResultDict]: + """Get the results as a guaranteed list, even if they are represented as None.""" + results = message.get("results") + return results if results is not None else [] + + @staticmethod + def auxiliary_graphs_dict(message: MessageDict) -> AuxiliaryGraphsDict: + """Get the auxiliary_graphs as a guaranteed dict, even if they are represented as None.""" + auxiliary_graphs = message.get("auxiliary_graphs") + return auxiliary_graphs if auxiliary_graphs is not None else {} + + @staticmethod + def normalize(message: MessageDict) -> dict[EdgeID, EdgeID]: + """Normalize the knowledge_graph and update results and auxiliary_graphs accordingly.""" + knowledge_graph = message.get("knowledge_graph") + if knowledge_graph is None: + return {} + + mapping = KnowledgeGraphDictUtil.normalize(knowledge_graph) + + AuxiliaryGraphDictUtil.normalize_aux_dict( + MessageDictUtil.auxiliary_graphs_dict(message), mapping + ) + ResultDictUtil.normalize_list(MessageDictUtil.results_list(message), mapping) + + return mapping + + @staticmethod + def update( + message: MessageDict, + other: MessageDict, + pre_normalized: Literal["neither", "both", "self", "other"] = "neither", + ) -> dict[EdgeID, EdgeID]: + """Update one message in-place using the other. + + Returns a mapping of old:new EdgeIDs if normalization was done. + """ + # Compare by hash (like the model's `==`) so extra/non-schema keys are ignored. + if _query_graph_hash(message.get("query_graph")) != _query_graph_hash( + other.get("query_graph") + ): + raise NotImplementedError("Query graph merging not yet supported.") + + mapping = dict[EdgeID, EdgeID]() + if pre_normalized in ("neither", "other"): + mapping.update(MessageDictUtil.normalize(message)) + if pre_normalized in ("neither", "self"): + # Normalize a deep copy of the other dict so as not to modify the original. + other = copy.deepcopy(other) + mapping.update(MessageDictUtil.normalize(other)) + + msg_kg = message.get("knowledge_graph") + other_kg = other.get("knowledge_graph") + if (not msg_kg) and other_kg: + message["knowledge_graph"] = other_kg + elif msg_kg and other_kg: + KnowledgeGraphDictUtil.update(msg_kg, other_kg) + + msg_results = message.get("results") + other_results = other.get("results") + if (not msg_results) and other_results: + message["results"] = other_results + elif msg_results and other_results: + ResultDictUtil.merge_results(msg_results, other_results) + + msg_aux = message.get("auxiliary_graphs") + other_aux = other.get("auxiliary_graphs") + if (not msg_aux) and other_aux: + message["auxiliary_graphs"] = other_aux + elif msg_aux and other_aux: + AuxiliaryGraphDictUtil.merge_dictionaries(msg_aux, other_aux) + + return mapping + + @staticmethod + def prune_kg(message: MessageDict) -> None: + """Prune the knowledge_graph.""" + knowledge_graph = message.get("knowledge_graph") + if knowledge_graph is None: + return + KnowledgeGraphDictUtil.prune( + knowledge_graph, + MessageDictUtil.auxiliary_graphs_dict(message), + MessageDictUtil.results_list(message), + ) diff --git a/src/translator_tom/model_dicts/meta_attribute.py b/src/translator_tom/model_dicts/meta_attribute.py index 68caa27..9dd8fd3 100644 --- a/src/translator_tom/model_dicts/meta_attribute.py +++ b/src/translator_tom/model_dicts/meta_attribute.py @@ -2,14 +2,52 @@ from typing_extensions import NotRequired, TypedDict +from translator_tom.models.meta_attribute import MetaAttribute from translator_tom.models.shared import CURIE +from translator_tom.utils.dict_util_base import DictUtil +from translator_tom.utils.hash import tomhash -__all__ = ["MetaAttributeDict"] +__all__ = ["MetaAttributeDict", "MetaAttributeDictUtil"] class MetaAttributeDict(TypedDict): attribute_type_id: CURIE attribute_source: NotRequired[str | None] original_attribute_names: NotRequired[list[str] | None] - constraint_use: NotRequired[bool | None] + constraint_use: NotRequired[bool] constraint_name: NotRequired[str | None] + + +class MetaAttributeDictUtil(DictUtil[MetaAttributeDict]): + """Utility methods for `MetaAttributeDict`, mirroring those on the `MetaAttribute` model.""" + + _model = MetaAttribute + + @staticmethod + def original_attribute_names_list(meta_attribute: MetaAttributeDict) -> list[str]: + """Get the original attribute names as a guaranteed list, even if they are represented as None.""" + original_attribute_names = meta_attribute.get("original_attribute_names") + return original_attribute_names if original_attribute_names is not None else [] + + @classmethod + def hash(cls, obj: MetaAttributeDict) -> str: + """Hash matching `MetaAttribute.hash` (identity plus constraint usability).""" + return tomhash( + ( + obj["attribute_type_id"], + obj.get("attribute_source"), + obj.get("constraint_use", cls._default("constraint_use")), + ) + ) + + @staticmethod + def merge_attribute_lists( + old: list[MetaAttributeDict], new: list[MetaAttributeDict] + ) -> None: + """Merge the new attributes into the existing attributes.""" + attrs = {MetaAttributeDictUtil.hash(attr): attr for attr in old} + for attr in new: + attrs[MetaAttributeDictUtil.hash(attr)] = attr + + old.clear() + old.extend(attrs.values()) diff --git a/src/translator_tom/model_dicts/meta_knowledge_graph.py b/src/translator_tom/model_dicts/meta_knowledge_graph.py index 53ce1aa..8c551b4 100644 --- a/src/translator_tom/model_dicts/meta_knowledge_graph.py +++ b/src/translator_tom/model_dicts/meta_knowledge_graph.py @@ -2,15 +2,38 @@ from typing_extensions import NotRequired, TypedDict -from translator_tom.model_dicts.meta_attribute import MetaAttributeDict -from translator_tom.model_dicts.meta_qualifier import MetaQualifierDict +from translator_tom.model_dicts.attribute import ( + AttributeConstraintDict, + AttributeConstraintDictUtil, +) +from translator_tom.model_dicts.meta_attribute import ( + MetaAttributeDict, + MetaAttributeDictUtil, +) +from translator_tom.model_dicts.meta_qualifier import ( + MetaQualifierDict, + MetaQualifierDictUtil, +) +from translator_tom.model_dicts.qualifier import ( + QualifierConstraintDict, + QualifierConstraintDictUtil, +) +from translator_tom.models.meta_knowledge_graph import ( + MetaEdge, + MetaKnowledgeGraph, + MetaNode, +) from translator_tom.models.shared import KnowledgeType from translator_tom.utils.biolink import Biolink +from translator_tom.utils.dict_util_base import DictUtil __all__ = [ "MetaEdgeDict", + "MetaEdgeDictUtil", "MetaKnowledgeGraphDict", + "MetaKnowledgeGraphDictUtil", "MetaNodeDict", + "MetaNodeDictUtil", ] @@ -19,6 +42,32 @@ class MetaNodeDict(TypedDict): attributes: NotRequired[list[MetaAttributeDict] | None] +class MetaNodeDictUtil(DictUtil[MetaNodeDict]): + """Utility methods for `MetaNodeDict`, mirroring those on the `MetaNode` model.""" + + _model = MetaNode + + @staticmethod + def attributes_list(meta_node: MetaNodeDict) -> list[MetaAttributeDict]: + """Get the meta attributes as a guaranteed list, even if they are represented as None.""" + attributes = meta_node.get("attributes") + return attributes if attributes is not None else [] + + @staticmethod + def update(meta_node: MetaNodeDict, other: MetaNodeDict) -> None: + """Update the meta node in-place with another meta node.""" + meta_node["id_prefixes"] = list( + set(meta_node["id_prefixes"]) | set(other["id_prefixes"]) + ) + + node_attrs = meta_node.get("attributes") + other_attrs = other.get("attributes") + if (not node_attrs) and other_attrs: + meta_node["attributes"] = other_attrs + elif node_attrs and other_attrs: + MetaAttributeDictUtil.merge_attribute_lists(node_attrs, other_attrs) + + class MetaEdgeDict(TypedDict): subject: Biolink.Entity predicate: Biolink.Predicate @@ -29,6 +78,113 @@ class MetaEdgeDict(TypedDict): association: NotRequired[Biolink.Entity | None] +class MetaEdgeDictUtil(DictUtil[MetaEdgeDict]): + """Utility methods for `MetaEdgeDict`, mirroring those on the `MetaEdge` model.""" + + _model = MetaEdge + + @staticmethod + def knowledge_types_list(meta_edge: MetaEdgeDict) -> list[KnowledgeType]: + """Get the knowledge types as a guaranteed list, even if they are represented as None.""" + knowledge_types = meta_edge.get("knowledge_types") + return knowledge_types if knowledge_types is not None else [] + + @staticmethod + def attributes_list(meta_edge: MetaEdgeDict) -> list[MetaAttributeDict]: + """Get the meta attributes as a guaranteed list, even if they are represented as None.""" + attributes = meta_edge.get("attributes") + return attributes if attributes is not None else [] + + @staticmethod + def qualifiers_list(meta_edge: MetaEdgeDict) -> list[MetaQualifierDict]: + """Get the meta qualifiers as a guaranteed list, even if they are represented as None.""" + qualifiers = meta_edge.get("qualifiers") + return qualifiers if qualifiers is not None else [] + + @staticmethod + def update(meta_edge: MetaEdgeDict, other: MetaEdgeDict) -> None: + """Update the meta edge in-place with another meta edge.""" + edge_kt = meta_edge.get("knowledge_types") + other_kt = other.get("knowledge_types") + if (not edge_kt) and other_kt: + meta_edge["knowledge_types"] = other_kt + elif edge_kt and other_kt: + meta_edge["knowledge_types"] = list( + set(MetaEdgeDictUtil.knowledge_types_list(meta_edge)) + | set(MetaEdgeDictUtil.knowledge_types_list(other)) + ) + + edge_attrs = meta_edge.get("attributes") + other_attrs = other.get("attributes") + if (not edge_attrs) and other_attrs: + meta_edge["attributes"] = other_attrs + elif edge_attrs and other_attrs: + attrs = {MetaAttributeDictUtil.hash(attr): attr for attr in edge_attrs} + kl_at = (Biolink("knowledge_level"), Biolink("agent_type")) + for attr in other_attrs: + # Avoid multiple KL/AT + if attr["attribute_type_id"] in kl_at: + continue + attrs[MetaAttributeDictUtil.hash(attr)] = attr + meta_edge["attributes"] = list(attrs.values()) + + other_quals = other.get("qualifiers") + if not other_quals: + return + edge_quals = meta_edge.get("qualifiers") + if not edge_quals: + meta_edge["qualifiers"] = other_quals + return + + quals_by_type = {qual["qualifier_type_id"]: qual for qual in edge_quals} + new_quals_by_type = {qual["qualifier_type_id"]: qual for qual in other_quals} + for type_id, qual in new_quals_by_type.items(): + if type_id in quals_by_type: + merged = list( + set( + MetaQualifierDictUtil.applicable_values_list( + quals_by_type[type_id] + ) + ) + | set(MetaQualifierDictUtil.applicable_values_list(qual)) + ) + if len(merged) > 0: + quals_by_type[type_id]["applicable_values"] = merged + else: + quals_by_type[type_id] = qual + + meta_edge["qualifiers"] = list(quals_by_type.values()) + + @staticmethod + def meets_attribute_constraints( + meta_edge: MetaEdgeDict, constraints: list[AttributeConstraintDict] + ) -> bool: + """Check if all attribute constraints are satisfied by the meta edge's attributes.""" + return AttributeConstraintDictUtil.set_met_by( + constraints, MetaEdgeDictUtil.attributes_list(meta_edge) + ) + + @staticmethod + def meets_qualifier_constraints( + meta_edge: MetaEdgeDict, constraints: list[QualifierConstraintDict] + ) -> bool: + """Check if the meta edge satisfies the qualifier constraints.""" + return QualifierConstraintDictUtil.set_met_by( + constraints, MetaEdgeDictUtil.qualifiers_list(meta_edge) + ) + + class MetaKnowledgeGraphDict(TypedDict): nodes: dict[Biolink.Entity, MetaNodeDict] edges: list[MetaEdgeDict] + + +class MetaKnowledgeGraphDictUtil(DictUtil[MetaKnowledgeGraphDict]): + """Utility methods for `MetaKnowledgeGraphDict`, mirroring the `MetaKnowledgeGraph` model.""" + + _model = MetaKnowledgeGraph + + @staticmethod + def new() -> MetaKnowledgeGraphDict: + """Return an empty instance, without having to pass required containers.""" + return {"nodes": {}, "edges": []} diff --git a/src/translator_tom/model_dicts/meta_qualifier.py b/src/translator_tom/model_dicts/meta_qualifier.py index b5a7cbb..0741e0b 100644 --- a/src/translator_tom/model_dicts/meta_qualifier.py +++ b/src/translator_tom/model_dicts/meta_qualifier.py @@ -2,11 +2,25 @@ from typing_extensions import NotRequired, TypedDict +from translator_tom.models.meta_qualifier import MetaQualifier from translator_tom.utils.biolink import Biolink +from translator_tom.utils.dict_util_base import DictUtil -__all__ = ["MetaQualifierDict"] +__all__ = ["MetaQualifierDict", "MetaQualifierDictUtil"] class MetaQualifierDict(TypedDict): qualifier_type_id: Biolink.Qualifier applicable_values: NotRequired[list[str] | None] + + +class MetaQualifierDictUtil(DictUtil[MetaQualifierDict]): + """Utility methods for `MetaQualifierDict`, mirroring those on the `MetaQualifier` model.""" + + _model = MetaQualifier + + @staticmethod + def applicable_values_list(meta_qualifier: MetaQualifierDict) -> list[str]: + """Get the applicable values as a guaranteed list, even if they are represented as None.""" + applicable_values = meta_qualifier.get("applicable_values") + return applicable_values if applicable_values is not None else [] diff --git a/src/translator_tom/model_dicts/node_binding.py b/src/translator_tom/model_dicts/node_binding.py index d507d7a..657faa6 100644 --- a/src/translator_tom/model_dicts/node_binding.py +++ b/src/translator_tom/model_dicts/node_binding.py @@ -2,13 +2,33 @@ from typing_extensions import NotRequired, TypedDict -from translator_tom.model_dicts.attribute import AttributeDict +from translator_tom.model_dicts.attribute import AttributeDict, AttributeDictUtil +from translator_tom.models.node_binding import NodeBinding from translator_tom.models.shared import CURIE +from translator_tom.utils.dict_util_base import DictUtil +from translator_tom.utils.hash import tomhash -__all__ = ["NodeBindingDict"] +__all__ = ["NodeBindingDict", "NodeBindingDictUtil"] class NodeBindingDict(TypedDict): id: CURIE query_id: NotRequired[CURIE | None] attributes: list[AttributeDict] + + +class NodeBindingDictUtil(DictUtil[NodeBindingDict]): + """Utility methods for `NodeBindingDict`, mirroring those on the `NodeBinding` model.""" + + _model = NodeBinding + + @classmethod + def hash(cls, obj: NodeBindingDict) -> str: + """Hash matching `NodeBinding.hash` (bound node id, query id, attributes).""" + return tomhash( + ( + obj["id"], + obj.get("query_id"), + frozenset(AttributeDictUtil.hash(a) for a in obj["attributes"]), + ) + ) diff --git a/src/translator_tom/model_dicts/path_binding.py b/src/translator_tom/model_dicts/path_binding.py index 0e006f8..1b5b1a9 100644 --- a/src/translator_tom/model_dicts/path_binding.py +++ b/src/translator_tom/model_dicts/path_binding.py @@ -2,10 +2,18 @@ from typing_extensions import TypedDict +from translator_tom.models.path_binding import PathBinding from translator_tom.models.shared import AuxGraphID +from translator_tom.utils.dict_util_base import DictUtil -__all__ = ["PathBindingDict"] +__all__ = ["PathBindingDict", "PathBindingDictUtil"] class PathBindingDict(TypedDict): id: AuxGraphID + + +class PathBindingDictUtil(DictUtil[PathBindingDict]): + """Registration-only util for `PathBindingDict`.""" + + _model = PathBinding diff --git a/src/translator_tom/model_dicts/path_constraint.py b/src/translator_tom/model_dicts/path_constraint.py index 26440a0..840fd0e 100644 --- a/src/translator_tom/model_dicts/path_constraint.py +++ b/src/translator_tom/model_dicts/path_constraint.py @@ -2,10 +2,26 @@ from typing_extensions import NotRequired, TypedDict +from translator_tom.models.path_constraint import PathConstraint from translator_tom.utils.biolink import Biolink +from translator_tom.utils.dict_util_base import DictUtil -__all__ = ["PathConstraintDict"] +__all__ = ["PathConstraintDict", "PathConstraintDictUtil"] class PathConstraintDict(TypedDict): intermediate_categories: NotRequired[list[Biolink.Entity] | None] + + +class PathConstraintDictUtil(DictUtil[PathConstraintDict]): + """Utility methods for `PathConstraintDict`, mirroring those on the `PathConstraint` model.""" + + _model = PathConstraint + + @staticmethod + def intermediate_categories_list( + path_constraint: PathConstraintDict, + ) -> list[Biolink.Entity]: + """Get the intermediate_categories as a guaranteed list, even if they are represented as None.""" + intermediate_categories = path_constraint.get("intermediate_categories") + return intermediate_categories if intermediate_categories is not None else [] diff --git a/src/translator_tom/model_dicts/qualifier.py b/src/translator_tom/model_dicts/qualifier.py index 73a8078..0359a34 100644 --- a/src/translator_tom/model_dicts/qualifier.py +++ b/src/translator_tom/model_dicts/qualifier.py @@ -1,12 +1,21 @@ from __future__ import annotations +import itertools +from collections.abc import Iterable +from typing import cast + from typing_extensions import TypedDict +from translator_tom.model_dicts.meta_qualifier import MetaQualifierDict +from translator_tom.models.qualifier import Qualifier, QualifierConstraint from translator_tom.utils.biolink import Biolink +from translator_tom.utils.dict_util_base import DictUtil __all__ = [ "QualifierConstraintDict", + "QualifierConstraintDictUtil", "QualifierDict", + "QualifierDictUtil", ] @@ -15,5 +24,116 @@ class QualifierDict(TypedDict): qualifier_value: str +class QualifierDictUtil(DictUtil[QualifierDict]): + """Registration-only util for `QualifierDict`.""" + + _model = Qualifier + + +def _qualifier_values( + qualifier: QualifierDict | MetaQualifierDict, +) -> set[str] | None: + """The values a qualifier dict contributes when matching a constraint. + + A `QualifierDict` (has `qualifier_value`) contributes its single value; a + `MetaQualifierDict` contributes its applicable values, where None means any + value is allowed. + """ + if "qualifier_value" in qualifier: + return {cast("QualifierDict", qualifier)["qualifier_value"]} + applicable = qualifier.get("applicable_values") + return set(applicable) if applicable is not None else None + + class QualifierConstraintDict(TypedDict): qualifier_set: list[QualifierDict] + + +class QualifierConstraintDictUtil(DictUtil[QualifierConstraintDict]): + """Utility methods for `QualifierConstraintDict`, mirroring those on the `QualifierConstraint` model.""" + + _model = QualifierConstraint + + @staticmethod + def new() -> QualifierConstraintDict: + """Return an empty instance, without having to pass required containers.""" + return {"qualifier_set": []} + + @staticmethod + def met_by( + constraint: QualifierConstraintDict, + qualifiers: Iterable[QualifierDict] | Iterable[MetaQualifierDict], + ) -> bool: + """Check that the given qualifiers satisfy the constraint.""" + qualifier_pairs: list[tuple[Biolink.Qualifier, set[str] | None]] = [ + (qualifier["qualifier_type_id"], _qualifier_values(qualifier)) + for qualifier in qualifiers + ] + + for constr in constraint["qualifier_set"]: + applicable_types = set(Biolink.get_descendants(constr["qualifier_type_id"])) + allowed_values: set[str] | None = None + met = False + for qual_type, available_values in qualifier_pairs: + if qual_type not in applicable_types: + continue + if allowed_values is None: + # expand values once a type matches + allowed_values = set( + itertools.chain.from_iterable( + Biolink.get_descendant_values(t, constr["qualifier_value"]) + for t in applicable_types + ) + ) + # available_values None means a MetaQualifier allowing all values. + if available_values is None or allowed_values & available_values: + met = True + break + if not met: + return False + + return True + + @staticmethod + def set_met_by( + constraints: list[QualifierConstraintDict], + qualifiers: list[QualifierDict] | list[MetaQualifierDict], + ) -> bool: + """Check if the given set of constraints are met by the given qualifiers.""" + if len(constraints) == 0: + return True + elif len(qualifiers) == 0: + return False + + return any( + QualifierConstraintDictUtil.met_by(constraint, qualifiers) + for constraint in constraints + ) + + @staticmethod + def get_inverse(constraint: QualifierConstraintDict) -> QualifierConstraintDict: + """Return a (SPO) inverse of the constraint, for reversing edges.""" + new_qualifier_set = list[QualifierDict]() + for qualifier in constraint["qualifier_set"]: + new_qualifier = cast("QualifierDict", {**qualifier}) + type_id = qualifier["qualifier_type_id"] + value = qualifier["qualifier_value"] + if "object" in type_id: + new_qualifier["qualifier_type_id"] = type_id.replace( + "object", "subject" + ) + elif "subject" in type_id: + new_qualifier["qualifier_type_id"] = type_id.replace( + "subject", "object" + ) + elif inverse := ( + type_id == "biolink:qualified_predicate" and Biolink.get_inverse(value) + ): + new_qualifier["qualifier_value"] = inverse + else: + raise ValueError( + f"Cannot inverse qualifier because its value is non-inversible predicate {value}" + ) + new_qualifier_set.append(new_qualifier) + + return {"qualifier_set": new_qualifier_set} diff --git a/src/translator_tom/model_dicts/query.py b/src/translator_tom/model_dicts/query.py index 1c36c46..a0203fd 100644 --- a/src/translator_tom/model_dicts/query.py +++ b/src/translator_tom/model_dicts/query.py @@ -5,8 +5,10 @@ from translator_tom.model_dicts.message import MessageDict from translator_tom.model_dicts.workflow_operations import OperationDict from translator_tom.models.log_entry import LogLevel +from translator_tom.models.query import Query +from translator_tom.utils.dict_util_base import DictUtil -__all__ = ["QueryDict"] +__all__ = ["QueryDict", "QueryDictUtil"] class QueryDict(TypedDict): @@ -15,3 +17,20 @@ class QueryDict(TypedDict): workflow: NotRequired[list[OperationDict] | None] submitter: NotRequired[str | None] bypass_cache: NotRequired[bool] + + +class QueryDictUtil(DictUtil[QueryDict]): + """Utility methods for `QueryDict`, mirroring those on the `Query` model.""" + + _model = Query + + @staticmethod + def workflow_list(query: QueryDict) -> list[OperationDict]: + """Get the workflow operations as a guaranteed list, even if they are represented as None.""" + workflow = query.get("workflow") + return workflow if workflow is not None else [] + + @staticmethod + def new() -> QueryDict: + """Return an empty instance, without having to pass required containers.""" + return {"message": {}} diff --git a/src/translator_tom/model_dicts/query_graph.py b/src/translator_tom/model_dicts/query_graph.py index 91cc596..de8402f 100644 --- a/src/translator_tom/model_dicts/query_graph.py +++ b/src/translator_tom/model_dicts/query_graph.py @@ -1,11 +1,26 @@ from __future__ import annotations +from collections.abc import Mapping + from typing_extensions import NotRequired, TypedDict -from translator_tom.model_dicts.attribute import AttributeConstraintDict +from translator_tom.model_dicts.attribute import ( + AttributeConstraintDict, + AttributeConstraintDictUtil, +) from translator_tom.model_dicts.path_constraint import PathConstraintDict -from translator_tom.model_dicts.qualifier import QualifierConstraintDict -from translator_tom.models.query_graph import SetInterpretation +from translator_tom.model_dicts.qualifier import ( + QualifierConstraintDict, + QualifierConstraintDictUtil, +) +from translator_tom.models.query_graph import ( + PathfinderQueryGraph, + QEdge, + QNode, + QPath, + QueryGraph, + SetInterpretation, +) from translator_tom.models.shared import ( CURIE, KnowledgeType, @@ -14,14 +29,20 @@ QPathID, ) from translator_tom.utils.biolink import Biolink +from translator_tom.utils.dict_util_base import DictUtil, register_union_discriminator __all__ = [ "BaseQueryGraphDict", "PathfinderQueryGraphDict", + "PathfinderQueryGraphDictUtil", "QEdgeDict", + "QEdgeDictUtil", "QNodeDict", + "QNodeDictUtil", "QPathDict", + "QPathDictUtil", "QueryGraphDict", + "QueryGraphDictUtil", ] @@ -33,6 +54,36 @@ class QNodeDict(TypedDict): constraints: NotRequired[list[AttributeConstraintDict] | None] +class QNodeDictUtil(DictUtil[QNodeDict]): + """Utility methods for `QNodeDict`, mirroring those on the `QNode` model.""" + + _model = QNode + + @staticmethod + def ids_list(qnode: QNodeDict) -> list[CURIE]: + """Get the IDs as a guaranteed list, even if they are represented as None.""" + ids = qnode.get("ids") + return ids if ids is not None else [] + + @staticmethod + def categories_list(qnode: QNodeDict) -> list[Biolink.Entity]: + """Get the categories as a guaranteed list, even if they are represented as None.""" + categories = qnode.get("categories") + return categories if categories is not None else [] + + @staticmethod + def member_ids_list(qnode: QNodeDict) -> list[CURIE]: + """Get the member_ids as a guaranteed list, even if they are represented as None.""" + member_ids = qnode.get("member_ids") + return member_ids if member_ids is not None else [] + + @staticmethod + def constraints_list(qnode: QNodeDict) -> list[AttributeConstraintDict]: + """Get the attribute constraints as a guaranteed list, even if they are represented as None.""" + constraints = qnode.get("constraints") + return constraints if constraints is not None else [] + + class QEdgeDict(TypedDict): knowledge_type: NotRequired[KnowledgeType | None] predicates: NotRequired[list[Biolink.Predicate] | None] @@ -42,6 +93,69 @@ class QEdgeDict(TypedDict): qualifier_constraints: NotRequired[list[QualifierConstraintDict] | None] +class QEdgeDictUtil(DictUtil[QEdgeDict]): + """Utility methods for `QEdgeDict`, mirroring those on the `QEdge` model.""" + + _model = QEdge + + @staticmethod + def predicates_list(qedge: QEdgeDict) -> list[Biolink.Predicate]: + """Get the predicates as a guaranteed list, even if they are represented as None.""" + predicates = qedge.get("predicates") + return predicates if predicates is not None else [] + + @staticmethod + def attribute_constraints_list(qedge: QEdgeDict) -> list[AttributeConstraintDict]: + """Get the attribute_constraints as a guaranteed list, even if they are represented as None.""" + attribute_constraints = qedge.get("attribute_constraints") + return attribute_constraints if attribute_constraints is not None else [] + + @staticmethod + def qualifier_constraints_list(qedge: QEdgeDict) -> list[QualifierConstraintDict]: + """Get the qualifier_constraints as a guaranteed list, even if they are represented as None.""" + qualifier_constraints = qedge.get("qualifier_constraints") + return qualifier_constraints if qualifier_constraints is not None else [] + + @staticmethod + def get_inverse(qedge: QEdgeDict) -> QEdgeDict: + """Get an inverse copy of the QEdge.""" + inverse_predicates = list[Biolink.Predicate]() + failed_predicates = list[Biolink.Predicate]() + for predicate in QEdgeDictUtil.predicates_list(qedge): + inverse = Biolink.get_inverse(predicate) + if inverse is None: + failed_predicates.append(predicate) + continue + inverse_predicates.append(inverse) + + if len(failed_predicates) > 0: + raise ValueError(f"Cannot invert predicates {failed_predicates}.") + + # Keep dict minimal as in model behavior + inverted: QEdgeDict = { + "subject": qedge["object"], + "object": qedge["subject"], + } + knowledge_type = qedge.get("knowledge_type") + if knowledge_type is not None: + inverted["knowledge_type"] = knowledge_type + if inverse_predicates: + inverted["predicates"] = inverse_predicates + inverse_attribute_constraints = [ + AttributeConstraintDictUtil.get_inverse(ac) + for ac in QEdgeDictUtil.attribute_constraints_list(qedge) + ] + if inverse_attribute_constraints: + inverted["attribute_constraints"] = inverse_attribute_constraints + inverse_qualifier_constraints = [ + QualifierConstraintDictUtil.get_inverse(qc) + for qc in QEdgeDictUtil.qualifier_constraints_list(qedge) + ] + if inverse_qualifier_constraints: + inverted["qualifier_constraints"] = inverse_qualifier_constraints + return inverted + + class QPathDict(TypedDict): subject: QNodeID object: QNodeID @@ -49,6 +163,24 @@ class QPathDict(TypedDict): constraints: NotRequired[list[PathConstraintDict] | None] +class QPathDictUtil(DictUtil[QPathDict]): + """Utility methods for `QPathDict`, mirroring those on the `QPath` model.""" + + _model = QPath + + @staticmethod + def predicates_list(qpath: QPathDict) -> list[Biolink.Predicate]: + """Get the predicates as a guaranteed list, even if they are represented as None.""" + predicates = qpath.get("predicates") + return predicates if predicates is not None else [] + + @staticmethod + def constraints_list(qpath: QPathDict) -> list[PathConstraintDict]: + """Get the constraints as a guaranteed list, even if they are represented as None.""" + constraints = qpath.get("constraints") + return constraints if constraints is not None else [] + + class BaseQueryGraphDict(TypedDict): nodes: dict[QNodeID, QNodeDict] @@ -57,5 +189,30 @@ class QueryGraphDict(BaseQueryGraphDict): edges: dict[QEdgeID, QEdgeDict] +class QueryGraphDictUtil(DictUtil[QueryGraphDict]): + """Registration-only util for `QueryGraphDict`.""" + + _model = QueryGraph + + class PathfinderQueryGraphDict(BaseQueryGraphDict): paths: dict[QPathID, QPathDict] + + +class PathfinderQueryGraphDictUtil(DictUtil[PathfinderQueryGraphDict]): + """Registration-only util for `PathfinderQueryGraphDict`.""" + + _model = PathfinderQueryGraph + + +def _discriminate_query_graph( + value: Mapping[str, object], +) -> type[QueryGraph | PathfinderQueryGraph]: + """Pick the concrete query-graph model for a raw dict (`paths` -> Pathfinder).""" + return PathfinderQueryGraph if "paths" in value else QueryGraph + + +# Message.query_graph is a union QueryGraph | PathfinderQueryGraph, requires explicit discriminator +register_union_discriminator( + (QueryGraph, PathfinderQueryGraph), _discriminate_query_graph +) diff --git a/src/translator_tom/model_dicts/response.py b/src/translator_tom/model_dicts/response.py index fcc01e4..2fbe39c 100644 --- a/src/translator_tom/model_dicts/response.py +++ b/src/translator_tom/model_dicts/response.py @@ -5,8 +5,11 @@ from translator_tom.model_dicts.log_entry import LogEntryDict from translator_tom.model_dicts.message import MessageDict from translator_tom.model_dicts.workflow_operations import OperationDict +from translator_tom.models.response import Response +from translator_tom.utils.config import TRAPI_CONFIG +from translator_tom.utils.dict_util_base import DictUtil -__all__ = ["ResponseDict"] +__all__ = ["ResponseDict", "ResponseDictUtil"] class ResponseDict(TypedDict): @@ -17,3 +20,25 @@ class ResponseDict(TypedDict): workflow: NotRequired[list[OperationDict] | None] schema_version: NotRequired[str | None] biolink_version: NotRequired[str | None] + + +class ResponseDictUtil(DictUtil[ResponseDict]): + """Utility methods for `ResponseDict`, mirroring those on the `Response` model.""" + + _model = Response + + @staticmethod + def workflow_list(response: ResponseDict) -> list[OperationDict]: + """Get the workflow operations as a guaranteed list, even if they are represented as None.""" + workflow = response.get("workflow") + return workflow if workflow is not None else [] + + @staticmethod + def new() -> ResponseDict: + """Return an empty instance, without having to pass required containers.""" + # logs defaults to [] and is dropped by exclude_defaults, matching Response.new(). + return { + "message": {}, + "schema_version": TRAPI_CONFIG.schema_version, + "biolink_version": TRAPI_CONFIG.biolink_version, + } diff --git a/src/translator_tom/model_dicts/result.py b/src/translator_tom/model_dicts/result.py index dd66c67..2247b70 100644 --- a/src/translator_tom/model_dicts/result.py +++ b/src/translator_tom/model_dicts/result.py @@ -1,14 +1,145 @@ from __future__ import annotations +import itertools +from typing import cast + from typing_extensions import TypedDict -from translator_tom.model_dicts.analysis import AnalysisDict, PathfinderAnalysisDict -from translator_tom.model_dicts.node_binding import NodeBindingDict -from translator_tom.models.shared import QNodeID +from translator_tom.model_dicts.analysis import ( + AnalysisDict, + AnalysisDictUtil, + PathfinderAnalysisDict, + PathfinderAnalysisDictUtil, +) +from translator_tom.model_dicts.node_binding import ( + NodeBindingDict, + NodeBindingDictUtil, +) +from translator_tom.models.result import Result +from translator_tom.models.shared import CURIE, EdgeID, QNodeID +from translator_tom.utils.dict_util_base import DictUtil +from translator_tom.utils.hash import tomhash -__all__ = ["ResultDict"] +__all__ = ["ResultDict", "ResultDictUtil"] class ResultDict(TypedDict): node_bindings: dict[QNodeID, list[NodeBindingDict]] analyses: list[AnalysisDict | PathfinderAnalysisDict] + + +def _analysis_hash(analysis: AnalysisDict | PathfinderAnalysisDict) -> str: + """Hash an analysis dict, dispatching on its structural shape.""" + if "path_bindings" in analysis: + return PathfinderAnalysisDictUtil.hash(cast("PathfinderAnalysisDict", analysis)) + return AnalysisDictUtil.hash(analysis) + + +def _update_analysis( + existing: AnalysisDict | PathfinderAnalysisDict, + other: AnalysisDict | PathfinderAnalysisDict, +) -> None: + """Update one analysis dict with another of the same structural shape.""" + if "path_bindings" in existing: + PathfinderAnalysisDictUtil.update( + cast("PathfinderAnalysisDict", existing), + cast("PathfinderAnalysisDict", other), + ) + else: + AnalysisDictUtil.update(existing, cast("AnalysisDict", other)) + + +class ResultDictUtil(DictUtil[ResultDict]): + """Utility methods for `ResultDict`, mirroring those on the `Result` model.""" + + _model = Result + + @classmethod + def hash(cls, obj: ResultDict) -> str: + """Hash matching `Result.hash` (node bindings only).""" + return tomhash( + { + qnode_id: frozenset(NodeBindingDictUtil.hash(b) for b in bindings) + for qnode_id, bindings in obj["node_bindings"].items() + } + ) + + @staticmethod + def normalize(result: ResultDict, mapping: dict[EdgeID, EdgeID]) -> None: + """Normalize the result given a mapping of old:new EdgeIDs.""" + for analysis in result["analyses"]: + if "edge_bindings" not in analysis: + continue + analysis = cast("AnalysisDict", analysis) + for binding in itertools.chain( + *(bindings for bindings in analysis["edge_bindings"].values()) + ): + binding["id"] = mapping.get(binding["id"], binding["id"]) + + @staticmethod + def normalize_list( + results: list[ResultDict], mapping: dict[EdgeID, EdgeID] + ) -> None: + """Normalize a result list given a mapping of old:new EdgeIDs.""" + for result in results: + ResultDictUtil.normalize(result, mapping) + + @staticmethod + def update(result: ResultDict, other: ResultDict) -> None: + """Update the result in-place with another result.""" + if not other["analyses"]: + return + if not result["analyses"]: + result["analyses"] = other["analyses"] + return + + by_hash = {_analysis_hash(ana): ana for ana in result["analyses"]} + for analysis in other["analyses"]: + existing = by_hash.get(_analysis_hash(analysis)) + if existing is not None: + _update_analysis(existing, analysis) + else: + result["analyses"].append(analysis) + + @staticmethod + def merge_results( + results: list[ResultDict], new: list[ResultDict] | None = None + ) -> list[ResultDict]: + """Merge the given results in-place. + + If new results are provided, merge them into the first list. + Does not mutate `new`. + """ + if new is None: + new = [] + merged = dict[str, ResultDict]() + for result in (*results, *new): + result_hash = ResultDictUtil.hash(result) + if result_hash in merged: + ResultDictUtil.update(merged[result_hash], result) + else: + merged[result_hash] = result + + results.clear() + results.extend(merged.values()) + return results + + @staticmethod + def merge_analyses_by_resource_id(result: ResultDict) -> None: + """Merge any of the analyses on this result by resource_id. + + Useful when a service unintentionally adds multiple analyses to a single result, + combining all of those analyses. + """ + merged: dict[tuple[bool, CURIE], AnalysisDict | PathfinderAnalysisDict] = {} + for analysis in result["analyses"]: + # The bool distinguishes Analysis vs PathfinderAnalysis (mirrors the + # model keying by type), so only same-shape analyses ever merge. + key = ("path_bindings" in analysis, analysis["resource_id"]) + existing = merged.get(key) + if existing is None: + merged[key] = analysis + else: + _update_analysis(existing, analysis) + + result["analyses"] = list(merged.values()) diff --git a/src/translator_tom/model_dicts/retrieval_source.py b/src/translator_tom/model_dicts/retrieval_source.py index ef1005b..6ae47b7 100644 --- a/src/translator_tom/model_dicts/retrieval_source.py +++ b/src/translator_tom/model_dicts/retrieval_source.py @@ -2,10 +2,12 @@ from typing_extensions import NotRequired, TypedDict -from translator_tom.models.retrieval_source import ResourceRole +from translator_tom.models.retrieval_source import ResourceRole, RetrievalSource from translator_tom.models.shared import Infores +from translator_tom.utils.dict_util_base import DictUtil +from translator_tom.utils.hash import tomhash -__all__ = ["RetrievalSourceDict"] +__all__ = ["RetrievalSourceDict", "RetrievalSourceDictUtil"] class RetrievalSourceDict(TypedDict): @@ -13,3 +15,35 @@ class RetrievalSourceDict(TypedDict): resource_role: ResourceRole upstream_resource_ids: NotRequired[list[Infores] | None] source_record_urls: NotRequired[list[str] | None] + + +class RetrievalSourceDictUtil(DictUtil[RetrievalSourceDict]): + """Utility methods for `RetrievalSourceDict`, mirroring those on the `RetrievalSource` model.""" + + _model = RetrievalSource + + @staticmethod + def upstream_resource_ids_list(source: RetrievalSourceDict) -> list[Infores]: + """Get the upstream resource IDs as a guaranteed list, even if they are represented as None.""" + upstream_resource_ids = source.get("upstream_resource_ids") + return upstream_resource_ids if upstream_resource_ids is not None else [] + + @staticmethod + def source_record_urls_list(source: RetrievalSourceDict) -> list[str]: + """Get the source record URLs as a guaranteed list, even if they are represented as None.""" + source_record_urls = source.get("source_record_urls") + return source_record_urls if source_record_urls is not None else [] + + @classmethod + def hash(cls, obj: RetrievalSourceDict) -> str: + """Hash matching `RetrievalSource.hash` (resource identity and role only).""" + return tomhash((obj["resource_id"], obj["resource_role"])) + + @staticmethod + def update(source: RetrievalSourceDict, other: RetrievalSourceDict) -> None: + """Update the first source in-place, merging information from the second.""" + other_upstream = other.get("upstream_resource_ids") + if other_upstream: + source["upstream_resource_ids"] = list( + set(source.get("upstream_resource_ids") or []) | set(other_upstream) + ) diff --git a/src/translator_tom/model_dicts/workflow_operations.py b/src/translator_tom/model_dicts/workflow_operations.py index ed2ee45..1d301ad 100644 --- a/src/translator_tom/model_dicts/workflow_operations.py +++ b/src/translator_tom/model_dicts/workflow_operations.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import Literal +from collections.abc import Mapping +from typing import ClassVar, Literal from pydantic import JsonValue from typing_extensions import NotRequired, TypedDict @@ -8,67 +9,171 @@ from translator_tom.models.shared import Infores, QEdgeID, QNodeID from translator_tom.models.workflow_operations import ( AboveOrBelow, + AllowList, + AnnotateEdgesParameters, + AnnotateNodesParameters, AscendingOrDescending, + BaseOperation, + DenyList, + EnrichResultsParameters, + FillAllowListParameters, + FillDenyListParameters, + FilterKgraphContinuousKedgeAttributeParameters, + FilterKgraphDiscreteKedgeAttributeParameters, + FilterKgraphDiscreteKnodeAttributeParameters, + FilterKgraphParametersBase, + FilterKgraphPercentileParameters, + FilterKgraphStdDevParameters, + FilterKgraphTopNParameters, + FilterResultsTopNParameters, + OperationAnnotate, + OperationAnnotateEdges, + OperationAnnotateNodes, + OperationBind, + OperationCompleteResults, + OperationEnrichResults, + OperationFill, + OperationFilterKgraph, + OperationFilterKgraphContinuousKedgeAttribute, + OperationFilterKgraphDiscreteKedgeAttribute, + OperationFilterKgraphDiscreteKnodeAttribute, + OperationFilterKgraphOrphans, + OperationFilterKgraphPercentile, + OperationFilterKgraphStdDev, + OperationFilterKgraphTopN, + OperationFilterResults, + OperationFilterResultsTopN, + OperationLookup, + OperationLookupAndScore, + OperationOverlay, + OperationOverlayComputeJaccard, + OperationOverlayComputeNgd, + OperationOverlayConnectKnodes, + OperationOverlayFisherExactTest, + OperationRestate, + OperationScore, + OperationSortResults, + OperationSortResultsEdgeAttribute, + OperationSortResultsNodeAttribute, + OperationSortResultsScore, + OverlayComputeJaccardParameters, + OverlayComputeNgdParameters, + OverlayFisherExactTestParameters, PlusOrMinus, + SortResultNodeAttributeParameters, + SortResultsEdgeAttributeParameters, + SortResultsScoreParameters, TopOrBottom, ) +from translator_tom.utils.dict_util_base import DictUtil, register_union_discriminator __all__ = [ "AllowListDict", + "AllowListDictUtil", "AnnotateEdgesParametersDict", + "AnnotateEdgesParametersDictUtil", "AnnotateNodesParametersDict", + "AnnotateNodesParametersDictUtil", "BaseOperationDict", + "BaseOperationDictUtil", "DenyListDict", + "DenyListDictUtil", "EnrichResultsParametersDict", + "EnrichResultsParametersDictUtil", "FillAllowListParametersDict", + "FillAllowListParametersDictUtil", "FillDenyListParametersDict", + "FillDenyListParametersDictUtil", "FilterKgraphContinuousKedgeAttributeParametersDict", + "FilterKgraphContinuousKedgeAttributeParametersDictUtil", "FilterKgraphDiscreteKedgeAttributeParametersDict", + "FilterKgraphDiscreteKedgeAttributeParametersDictUtil", "FilterKgraphDiscreteKnodeAttributeParametersDict", + "FilterKgraphDiscreteKnodeAttributeParametersDictUtil", "FilterKgraphParametersBaseDict", + "FilterKgraphParametersBaseDictUtil", "FilterKgraphPercentileParametersDict", + "FilterKgraphPercentileParametersDictUtil", "FilterKgraphStdDevParametersDict", + "FilterKgraphStdDevParametersDictUtil", "FilterKgraphTopNParametersDict", + "FilterKgraphTopNParametersDictUtil", "FilterResultsTopNParametersDict", + "FilterResultsTopNParametersDictUtil", "OperationAnnotateDict", + "OperationAnnotateDictUtil", "OperationAnnotateEdgesDict", + "OperationAnnotateEdgesDictUtil", "OperationAnnotateNodesDict", + "OperationAnnotateNodesDictUtil", "OperationBindDict", + "OperationBindDictUtil", "OperationCompleteResultsDict", + "OperationCompleteResultsDictUtil", "OperationDict", "OperationEnrichResultsDict", + "OperationEnrichResultsDictUtil", "OperationFillDict", + "OperationFillDictUtil", "OperationFilterKgraphContinuousKedgeAttributeDict", + "OperationFilterKgraphContinuousKedgeAttributeDictUtil", "OperationFilterKgraphDict", + "OperationFilterKgraphDictUtil", "OperationFilterKgraphDiscreteKedgeAttributeDict", + "OperationFilterKgraphDiscreteKedgeAttributeDictUtil", "OperationFilterKgraphDiscreteKnodeAttributeDict", + "OperationFilterKgraphDiscreteKnodeAttributeDictUtil", "OperationFilterKgraphOrphansDict", + "OperationFilterKgraphOrphansDictUtil", "OperationFilterKgraphPercentileDict", + "OperationFilterKgraphPercentileDictUtil", "OperationFilterKgraphStdDevDict", + "OperationFilterKgraphStdDevDictUtil", "OperationFilterKgraphTopNDict", + "OperationFilterKgraphTopNDictUtil", "OperationFilterResultsDict", + "OperationFilterResultsDictUtil", "OperationFilterResultsTopNDict", + "OperationFilterResultsTopNDictUtil", "OperationLookupAndScoreDict", + "OperationLookupAndScoreDictUtil", "OperationLookupDict", + "OperationLookupDictUtil", "OperationOverlayComputeJaccardDict", + "OperationOverlayComputeJaccardDictUtil", "OperationOverlayComputeNgdDict", + "OperationOverlayComputeNgdDictUtil", "OperationOverlayConnectKnodesDict", + "OperationOverlayConnectKnodesDictUtil", "OperationOverlayDict", + "OperationOverlayDictUtil", "OperationOverlayFisherExactTestDict", - "OperationParametersDict", + "OperationOverlayFisherExactTestDictUtil", "OperationRestateDict", + "OperationRestateDictUtil", "OperationScoreDict", + "OperationScoreDictUtil", "OperationSortResultsDict", + "OperationSortResultsDictUtil", "OperationSortResultsEdgeAttributeDict", + "OperationSortResultsEdgeAttributeDictUtil", "OperationSortResultsNodeAttributeDict", + "OperationSortResultsNodeAttributeDictUtil", "OperationSortResultsScoreDict", + "OperationSortResultsScoreDictUtil", "OverlayComputeJaccardParametersDict", + "OverlayComputeJaccardParametersDictUtil", "OverlayComputeNgdParametersDict", + "OverlayComputeNgdParametersDictUtil", "OverlayFisherExactTestParametersDict", + "OverlayFisherExactTestParametersDictUtil", "RunnerParametersDict", "SortResultNodeAttributeParametersDict", + "SortResultNodeAttributeParametersDictUtil", "SortResultsEdgeAttributeParametersDict", + "SortResultsEdgeAttributeParametersDictUtil", "SortResultsScoreParametersDict", + "SortResultsScoreParametersDictUtil", ] @@ -76,10 +181,22 @@ class AllowListDict(TypedDict): allowlist: list[Infores] +class AllowListDictUtil(DictUtil[AllowListDict]): + """Registration-only util for `AllowListDict`.""" + + _model = AllowList + + class DenyListDict(TypedDict): denylist: list[Infores] +class DenyListDictUtil(DictUtil[DenyListDict]): + """Registration-only util for `DenyListDict`.""" + + _model = DenyList + + RunnerParametersDict = AllowListDict | DenyListDict @@ -91,57 +208,169 @@ class BaseOperationDict(TypedDict): runner_parameters: NotRequired[RunnerParametersDict | None] +class BaseOperationDictUtil(DictUtil[BaseOperationDict]): + """Utility methods for `BaseOperationDict`, mirroring those on the `BaseOperation` model.""" + + _model = BaseOperation + _unique: ClassVar[bool] = False + + @classmethod + def unique(cls) -> bool: + """Whether the operation may produce different results depending on the agent.""" + return cls._unique + + class OperationAnnotateDict(BaseOperationDict): id: Literal["annotate"] parameters: NotRequired[dict[str, JsonValue] | None] +class OperationAnnotateDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationAnnotateDict`, mirroring the `OperationAnnotate` model.""" + + _model = OperationAnnotate + _unique = True + + class AnnotateEdgesParametersDict(OperationParametersDict): attributes: NotRequired[list[str] | None] +class AnnotateEdgesParametersDictUtil(DictUtil[AnnotateEdgesParametersDict]): + """Utility methods for `AnnotateEdgesParametersDict`, mirroring the model.""" + + _model = AnnotateEdgesParameters + + @staticmethod + def attributes_list(parameters: AnnotateEdgesParametersDict) -> list[str]: + """Get the attributes as a guaranteed list, even if they are represented as None.""" + attributes = parameters.get("attributes") + return attributes if attributes is not None else [] + + class OperationAnnotateEdgesDict(BaseOperationDict): id: Literal["annotate_edges"] parameters: NotRequired[AnnotateEdgesParametersDict | None] +class OperationAnnotateEdgesDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationAnnotateEdgesDict`, mirroring the model.""" + + _model = OperationAnnotateEdges + _unique = True + + class AnnotateNodesParametersDict(OperationParametersDict): attributes: list[str] | None +class AnnotateNodesParametersDictUtil(DictUtil[AnnotateNodesParametersDict]): + """Utility methods for `AnnotateNodesParametersDict`, mirroring the model.""" + + _model = AnnotateNodesParameters + + @staticmethod + def attributes_list(parameters: AnnotateNodesParametersDict) -> list[str]: + """Get the attributes as a guaranteed list, even if they are represented as None.""" + attributes = parameters.get("attributes") + return attributes if attributes is not None else [] + + class OperationAnnotateNodesDict(BaseOperationDict): id: Literal["annotate_nodes"] parameters: NotRequired[AnnotateNodesParametersDict | None] +class OperationAnnotateNodesDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationAnnotateNodesDict`, mirroring the model.""" + + _model = OperationAnnotateNodes + _unique = True + + class OperationBindDict(BaseOperationDict): id: Literal["bind"] parameters: NotRequired[dict[str, JsonValue] | None] +class OperationBindDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationBindDict`, mirroring the `OperationBind` model.""" + + _model = OperationBind + + class OperationCompleteResultsDict(BaseOperationDict): id: Literal["complete_results"] parameters: NotRequired[dict[str, JsonValue] | None] +class OperationCompleteResultsDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationCompleteResultsDict`, mirroring the model.""" + + _model = OperationCompleteResults + + class EnrichResultsParametersDict(OperationParametersDict): - pvalue_threshold: NotRequired[int | float | None] + pvalue_threshold: NotRequired[int | float] qnode_keys: NotRequired[list[QNodeID] | None] +class EnrichResultsParametersDictUtil(DictUtil[EnrichResultsParametersDict]): + """Utility methods for `EnrichResultsParametersDict`, mirroring the model.""" + + _model = EnrichResultsParameters + + @staticmethod + def qnode_keys_list(parameters: EnrichResultsParametersDict) -> list[QNodeID]: + """Return a guaranteed list of qnode_keys, empty if it is not defined.""" + qnode_keys = parameters.get("qnode_keys") + return qnode_keys if qnode_keys is not None else [] + + class OperationEnrichResultsDict(BaseOperationDict): id: Literal["enrich_results"] parameters: NotRequired[EnrichResultsParametersDict | None] +class OperationEnrichResultsDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationEnrichResultsDict`, mirroring the model.""" + + _model = OperationEnrichResults + _unique = True + + class FillAllowListParametersDict(AllowListDict): qedge_keys: NotRequired[list[QEdgeID] | None] +class FillAllowListParametersDictUtil(DictUtil[FillAllowListParametersDict]): + """Utility methods for `FillAllowListParametersDict`, mirroring the model.""" + + _model = FillAllowListParameters + + @staticmethod + def qedge_keys_list(parameters: FillAllowListParametersDict) -> list[QEdgeID]: + """Return a guaranteed list of qedge_keys, empty if it is not defined.""" + qedge_keys = parameters.get("qedge_keys") + return qedge_keys if qedge_keys is not None else [] + + class FillDenyListParametersDict(DenyListDict): qedge_keys: NotRequired[list[QEdgeID] | None] +class FillDenyListParametersDictUtil(DictUtil[FillDenyListParametersDict]): + """Utility methods for `FillDenyListParametersDict`, mirroring the model.""" + + _model = FillDenyListParameters + + @staticmethod + def qedge_keys_list(parameters: FillDenyListParametersDict) -> list[QEdgeID]: + """Return a guaranteed list of qedge_keys, empty if it is not defined.""" + qedge_keys = parameters.get("qedge_keys") + return qedge_keys if qedge_keys is not None else [] + + class OperationFillDict(BaseOperationDict): id: Literal["fill"] parameters: NotRequired[ @@ -149,14 +378,45 @@ class OperationFillDict(BaseOperationDict): ] +class OperationFillDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationFillDict`, mirroring the `OperationFill` model.""" + + _model = OperationFill + _unique = True + + class OperationFilterKgraphDict(BaseOperationDict): id: Literal["filter_kgraph"] parameters: NotRequired[dict[str, JsonValue] | None] +class OperationFilterKgraphDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationFilterKgraphDict`, mirroring the model.""" + + _model = OperationFilterKgraph + + class FilterKgraphParametersBaseDict(OperationParametersDict): qedge_keys: list[QEdgeID] | None - qnode_keys: NotRequired[list[QNodeID] | None] + qnode_keys: NotRequired[list[QNodeID]] + + +class FilterKgraphParametersBaseDictUtil(DictUtil[FilterKgraphParametersBaseDict]): + """Utility methods for `FilterKgraphParametersBaseDict`, mirroring the model.""" + + _model = FilterKgraphParametersBase + + @staticmethod + def qedge_keys_list(parameters: FilterKgraphParametersBaseDict) -> list[QEdgeID]: + """Return a guaranteed list of qedge_keys, empty if it is not defined.""" + qedge_keys = parameters.get("qedge_keys") + return qedge_keys if qedge_keys is not None else [] + + @staticmethod + def qnode_keys_list(parameters: FilterKgraphParametersBaseDict) -> list[QNodeID]: + """Return a guaranteed list of qnode_keys, empty if it is not defined.""" + qnode_keys = parameters.get("qnode_keys") + return qnode_keys if qnode_keys is not None else [] class FilterKgraphContinuousKedgeAttributeParametersDict( @@ -167,52 +427,118 @@ class FilterKgraphContinuousKedgeAttributeParametersDict( remove_above_or_below: AboveOrBelow +class FilterKgraphContinuousKedgeAttributeParametersDictUtil( + FilterKgraphParametersBaseDictUtil +): + """Utility methods for `FilterKgraphContinuousKedgeAttributeParametersDict`.""" + + _model = FilterKgraphContinuousKedgeAttributeParameters + + class OperationFilterKgraphContinuousKedgeAttributeDict(BaseOperationDict): id: Literal["filter_kgraph_continuous_kedge_attribute"] parameters: FilterKgraphContinuousKedgeAttributeParametersDict +class OperationFilterKgraphContinuousKedgeAttributeDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationFilterKgraphContinuousKedgeAttributeDict`.""" + + _model = OperationFilterKgraphContinuousKedgeAttribute + + class FilterKgraphDiscreteKedgeAttributeParametersDict(FilterKgraphParametersBaseDict): edge_attribute: str remove_value: JsonValue +class FilterKgraphDiscreteKedgeAttributeParametersDictUtil( + FilterKgraphParametersBaseDictUtil +): + """Utility methods for `FilterKgraphDiscreteKedgeAttributeParametersDict`.""" + + _model = FilterKgraphDiscreteKedgeAttributeParameters + + class OperationFilterKgraphDiscreteKedgeAttributeDict(BaseOperationDict): id: Literal["filter_kgraph_discrete_kedge_attribute"] parameters: FilterKgraphDiscreteKedgeAttributeParametersDict +class OperationFilterKgraphDiscreteKedgeAttributeDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationFilterKgraphDiscreteKedgeAttributeDict`.""" + + _model = OperationFilterKgraphDiscreteKedgeAttribute + + class FilterKgraphDiscreteKnodeAttributeParametersDict(FilterKgraphParametersBaseDict): node_attribute: str remove_value: JsonValue +class FilterKgraphDiscreteKnodeAttributeParametersDictUtil( + FilterKgraphParametersBaseDictUtil +): + """Utility methods for `FilterKgraphDiscreteKnodeAttributeParametersDict`.""" + + _model = FilterKgraphDiscreteKnodeAttributeParameters + + class OperationFilterKgraphDiscreteKnodeAttributeDict(BaseOperationDict): id: Literal["filter_kgraph_discrete_knode_attribute"] parameters: FilterKgraphDiscreteKnodeAttributeParametersDict +class OperationFilterKgraphDiscreteKnodeAttributeDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationFilterKgraphDiscreteKnodeAttributeDict`.""" + + _model = OperationFilterKgraphDiscreteKnodeAttribute + + class OperationFilterKgraphOrphansDict(BaseOperationDict): id: Literal["filter_kgraph_orphans"] parameters: NotRequired[dict[str, JsonValue] | None] +class OperationFilterKgraphOrphansDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationFilterKgraphOrphansDict`, mirroring the model.""" + + _model = OperationFilterKgraphOrphans + + class FilterKgraphPercentileParametersDict(FilterKgraphParametersBaseDict): edge_attribute: str - threshold: NotRequired[float | None] + threshold: NotRequired[float] remove_above_or_below: NotRequired[AboveOrBelow] +class FilterKgraphPercentileParametersDictUtil(FilterKgraphParametersBaseDictUtil): + """Utility methods for `FilterKgraphPercentileParametersDict`.""" + + _model = FilterKgraphPercentileParameters + + class OperationFilterKgraphPercentileDict(BaseOperationDict): id: Literal["filter_kgraph_percentile"] parameters: FilterKgraphPercentileParametersDict +class OperationFilterKgraphPercentileDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationFilterKgraphPercentileDict`, mirroring the model.""" + + _model = OperationFilterKgraphPercentile + + class FilterKgraphStdDevParametersDict(FilterKgraphParametersBaseDict): edge_attribute: str - num_sigma: NotRequired[float | None] - remove_above_or_below: NotRequired[AboveOrBelow | None] - plus_or_minus_std_dev: NotRequired[PlusOrMinus | None] + num_sigma: NotRequired[float] + remove_above_or_below: NotRequired[AboveOrBelow] + plus_or_minus_std_dev: NotRequired[PlusOrMinus] + + +class FilterKgraphStdDevParametersDictUtil(FilterKgraphParametersBaseDictUtil): + """Utility methods for `FilterKgraphStdDevParametersDict`.""" + + _model = FilterKgraphStdDevParameters class OperationFilterKgraphStdDevDict(BaseOperationDict): @@ -220,10 +546,22 @@ class OperationFilterKgraphStdDevDict(BaseOperationDict): parameters: FilterKgraphStdDevParametersDict +class OperationFilterKgraphStdDevDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationFilterKgraphStdDevDict`, mirroring the model.""" + + _model = OperationFilterKgraphStdDev + + class FilterKgraphTopNParametersDict(FilterKgraphParametersBaseDict): edge_attribute: str - max_edges: NotRequired[int | None] - keep_top_or_bottom: NotRequired[TopOrBottom | None] + max_edges: NotRequired[int] + keep_top_or_bottom: NotRequired[TopOrBottom] + + +class FilterKgraphTopNParametersDictUtil(FilterKgraphParametersBaseDictUtil): + """Utility methods for `FilterKgraphTopNParametersDict`.""" + + _model = FilterKgraphTopNParameters class OperationFilterKgraphTopNDict(BaseOperationDict): @@ -231,61 +569,137 @@ class OperationFilterKgraphTopNDict(BaseOperationDict): parameters: FilterKgraphTopNParametersDict +class OperationFilterKgraphTopNDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationFilterKgraphTopNDict`, mirroring the model.""" + + _model = OperationFilterKgraphTopN + + class OperationFilterResultsDict(BaseOperationDict): id: Literal["filter_results"] parameters: NotRequired[dict[str, JsonValue] | None] +class OperationFilterResultsDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationFilterResultsDict`, mirroring the model.""" + + _model = OperationFilterResults + + class FilterResultsTopNParametersDict(OperationParametersDict): max_results: int +class FilterResultsTopNParametersDictUtil(DictUtil[FilterResultsTopNParametersDict]): + """Registration-only util for `FilterResultsTopNParametersDict`.""" + + _model = FilterResultsTopNParameters + + class OperationFilterResultsTopNDict(BaseOperationDict): id: Literal["filter_results_top_n"] parameters: FilterResultsTopNParametersDict +class OperationFilterResultsTopNDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationFilterResultsTopNDict`, mirroring the model.""" + + _model = OperationFilterResultsTopN + + class OperationLookupDict(BaseOperationDict): id: Literal["lookup"] parameters: NotRequired[dict[str, JsonValue] | None] +class OperationLookupDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationLookupDict`, mirroring the `OperationLookup` model.""" + + _model = OperationLookup + _unique = True + + class OperationLookupAndScoreDict(BaseOperationDict): id: Literal["lookup_and_score"] parameters: NotRequired[dict[str, JsonValue] | None] +class OperationLookupAndScoreDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationLookupAndScoreDict`, mirroring the model.""" + + _model = OperationLookupAndScore + _unique = True + + class OperationOverlayDict(BaseOperationDict): id: Literal["overlay"] parameters: NotRequired[dict[str, JsonValue] | None] +class OperationOverlayDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationOverlayDict`, mirroring the `OperationOverlay` model.""" + + _model = OperationOverlay + + class OverlayComputeJaccardParametersDict(OperationParametersDict): intermediate_node_key: QNodeID end_node_keys: list[QNodeID] virtual_relation_label: QEdgeID +class OverlayComputeJaccardParametersDictUtil( + DictUtil[OverlayComputeJaccardParametersDict] +): + """Registration-only util for `OverlayComputeJaccardParametersDict`.""" + + _model = OverlayComputeJaccardParameters + + class OperationOverlayComputeJaccardDict(BaseOperationDict): id: Literal["overlay_compute_jaccard"] parameters: OverlayComputeJaccardParametersDict +class OperationOverlayComputeJaccardDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationOverlayComputeJaccardDict`, mirroring the model.""" + + _model = OperationOverlayComputeJaccard + + class OverlayComputeNgdParametersDict(OperationParametersDict): virtual_relation_label: str qnode_keys: list[QNodeID] +class OverlayComputeNgdParametersDictUtil(DictUtil[OverlayComputeNgdParametersDict]): + """Registration-only util for `OverlayComputeNgdParametersDict`.""" + + _model = OverlayComputeNgdParameters + + class OperationOverlayComputeNgdDict(BaseOperationDict): id: Literal["overlay_compute_ngd"] parameters: OverlayComputeNgdParametersDict +class OperationOverlayComputeNgdDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationOverlayComputeNgdDict`, mirroring the model.""" + + _model = OperationOverlayComputeNgd + + class OperationOverlayConnectKnodesDict(BaseOperationDict): id: Literal["overlay_connect_knodes"] parameters: NotRequired[dict[str, JsonValue] | None] +class OperationOverlayConnectKnodesDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationOverlayConnectKnodesDict`, mirroring the model.""" + + _model = OperationOverlayConnectKnodes + + class OverlayFisherExactTestParametersDict(OperationParametersDict): subject_qnode_key: QNodeID object_qnode_key: QNodeID @@ -293,57 +707,139 @@ class OverlayFisherExactTestParametersDict(OperationParametersDict): rel_edge_key: QEdgeID | None +class OverlayFisherExactTestParametersDictUtil( + DictUtil[OverlayFisherExactTestParametersDict] +): + """Registration-only util for `OverlayFisherExactTestParametersDict`.""" + + _model = OverlayFisherExactTestParameters + + class OperationOverlayFisherExactTestDict(BaseOperationDict): id: Literal["overlay_fisher_exact_test"] parameters: OverlayFisherExactTestParametersDict +class OperationOverlayFisherExactTestDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationOverlayFisherExactTestDict`, mirroring the model.""" + + _model = OperationOverlayFisherExactTest + + class OperationRestateDict(BaseOperationDict): id: Literal["restate"] parameters: NotRequired[dict[str, JsonValue] | None] +class OperationRestateDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationRestateDict`, mirroring the `OperationRestate` model.""" + + _model = OperationRestate + _unique = True + + class OperationScoreDict(BaseOperationDict): id: Literal["score"] parameters: NotRequired[dict[str, JsonValue] | None] +class OperationScoreDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationScoreDict`, mirroring the `OperationScore` model.""" + + _model = OperationScore + _unique = True + + class OperationSortResultsDict(BaseOperationDict): id: Literal["sort_results"] parameters: NotRequired[dict[str, JsonValue] | None] +class OperationSortResultsDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationSortResultsDict`, mirroring the model.""" + + _model = OperationSortResults + + class SortResultsEdgeAttributeParametersDict(OperationParametersDict): edge_attribute: str ascending_or_descending: AscendingOrDescending qedge_keys: list[QEdgeID] +class SortResultsEdgeAttributeParametersDictUtil( + DictUtil[SortResultsEdgeAttributeParametersDict] +): + """Registration-only util for `SortResultsEdgeAttributeParametersDict`.""" + + _model = SortResultsEdgeAttributeParameters + + class OperationSortResultsEdgeAttributeDict(BaseOperationDict): id: Literal["sort_results_edge_attribute"] parameters: SortResultsEdgeAttributeParametersDict +class OperationSortResultsEdgeAttributeDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationSortResultsEdgeAttributeDict`, mirroring the model.""" + + _model = OperationSortResultsEdgeAttribute + + class SortResultNodeAttributeParametersDict(OperationParametersDict): node_attribute: str ascending_or_descending: AscendingOrDescending qnode_keys: list[QNodeID] | None +class SortResultNodeAttributeParametersDictUtil( + DictUtil[SortResultNodeAttributeParametersDict] +): + """Utility methods for `SortResultNodeAttributeParametersDict`, mirroring the model.""" + + _model = SortResultNodeAttributeParameters + + @staticmethod + def qnode_keys_list( + parameters: SortResultNodeAttributeParametersDict, + ) -> list[QNodeID]: + """Return a guaranteed list of qnode_keys, empty if it is not defined.""" + qnode_keys = parameters.get("qnode_keys") + return qnode_keys if qnode_keys is not None else [] + + class OperationSortResultsNodeAttributeDict(BaseOperationDict): id: Literal["sort_results_node_attribute"] parameters: SortResultNodeAttributeParametersDict +class OperationSortResultsNodeAttributeDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationSortResultsNodeAttributeDict`, mirroring the model.""" + + _model = OperationSortResultsNodeAttribute + + class SortResultsScoreParametersDict(OperationParametersDict): ascending_or_descending: AscendingOrDescending +class SortResultsScoreParametersDictUtil(DictUtil[SortResultsScoreParametersDict]): + """Registration-only util for `SortResultsScoreParametersDict`.""" + + _model = SortResultsScoreParameters + + class OperationSortResultsScoreDict(BaseOperationDict): id: Literal["sort_results_score"] parameters: SortResultsScoreParametersDict +class OperationSortResultsScoreDictUtil(BaseOperationDictUtil): + """Utility methods for `OperationSortResultsScoreDict`, mirroring the model.""" + + _model = OperationSortResultsScore + + OperationDict = ( OperationAnnotateDict | OperationAnnotateEdgesDict @@ -376,3 +872,25 @@ class OperationSortResultsScoreDict(BaseOperationDict): | OperationSortResultsNodeAttributeDict | OperationSortResultsScoreDict ) + + +def _discriminate_runner_parameters( + value: Mapping[str, object], +) -> type[AllowList | DenyList]: + """Pick the concrete runner-parameters model (`allowlist` key -> AllowList).""" + return AllowList if "allowlist" in value else DenyList + + +def _discriminate_fill_parameters( + value: Mapping[str, object], +) -> type[FillAllowListParameters | FillDenyListParameters]: + """Pick the concrete fill-parameters model (`allowlist` key -> FillAllowList).""" + return FillAllowListParameters if "allowlist" in value else FillDenyListParameters + + +# `BaseOperation.runner_parameters` and `OperationFill.parameters` are structural +# (non-tagged) unions, so hashing an operation needs explicit discriminators. +register_union_discriminator((AllowList, DenyList), _discriminate_runner_parameters) +register_union_discriminator( + (FillAllowListParameters, FillDenyListParameters), _discriminate_fill_parameters +) diff --git a/src/translator_tom/utils/dict_util_base.py b/src/translator_tom/utils/dict_util_base.py index 2e05304..fd55486 100644 --- a/src/translator_tom/utils/dict_util_base.py +++ b/src/translator_tom/utils/dict_util_base.py @@ -96,12 +96,7 @@ def _tag_literals(annotation: Any) -> tuple[Any, ...]: def _container_kind(annotation: Any) -> Literal["scalar", "list", "dict"]: - """Classify a field's outermost container (after stripping `Annotated`/`Optional`). - - Distinguishes a single nested model (`scalar`, e.g. `Message.query_graph`) from a - `list`/`dict` of them, since all three serialize to `dict`/`list` and can't be told - apart from the runtime value alone. - """ + """Classify a field's outermost container (after stripping `Annotated`/`Optional`).""" node = annotation while getattr(node, "__metadata__", None) is not None: node = node.__origin__ @@ -153,7 +148,7 @@ def from_json(cls, json: str | bytes) -> _TD: @overload @classmethod - def to_json(cls, obj: _TD) -> str: ... + def to_json(cls, obj: _TD) -> bytes: ... @overload @classmethod @@ -167,8 +162,7 @@ def to_json(cls, obj: _TD, as_str: Literal[False]) -> bytes: ... def to_json(cls, obj: _TD, as_str: bool = False) -> str | bytes: """Serialize a dict to JSON. - Dicts are expected to already be in canonical (None-omitted) form, so keys - are serialized as-is rather than filtered. + Dicts are expected to already be in canonical form. """ json = orjson.dumps(obj) if as_str: @@ -266,16 +260,12 @@ def _hash_field(cls, key: str, value: Any) -> Any: @classmethod def _field_defaults(cls) -> dict[str, Any]: - """Map each model field to its default (used for keys omitted from a dict). - - `to_dict` uses `exclude_defaults`, so a default-valued field is absent from - the serialized dict; hashing restores the default to match the model, whose - `hash()` reads live field values. Required fields (no default) map to None, - but they are always present in a valid serialization so the fallback is unused. - """ + """Map each model field to its default (for keys omitted from a dict).""" cached = cls.__dict__.get("_field_defaults_cache") if cached is not None: return cached + # to_dict omits default-valued fields; restoring the default lets a dict hash + # like the model, whose hash reads live (always-present) field values. defaults: dict[str, Any] = {} for name, field in cls._model.model_fields.items(): default = field.get_default(call_default_factory=True) @@ -283,6 +273,11 @@ def _field_defaults(cls) -> dict[str, Any]: cls._field_defaults_cache = defaults return defaults + @classmethod + def _default(cls, field_name: str) -> Any: + """Return the mirrored model's default for `field_name` (its model attribute name).""" + return cls._field_defaults()[field_name] + @classmethod def hash(cls, obj: _TD) -> str: """Hash the dict into a hex string, matching the corresponding model's `hash()`. diff --git a/tests/test_model_dicts/test_analysis_dicts.py b/tests/test_model_dicts/test_analysis_dicts.py new file mode 100644 index 0000000..ffe3ad7 --- /dev/null +++ b/tests/test_model_dicts/test_analysis_dicts.py @@ -0,0 +1,137 @@ +"""Parity tests for the analysis `*DictUtil` classes.""" + +from __future__ import annotations + +from translator_tom.model_dicts.analysis import ( + AnalysisDictUtil, + BaseAnalysisDictUtil, + PathfinderAnalysisDictUtil, +) +from translator_tom.models.analysis import ( + Analysis, + BaseAnalysis, + PathfinderAnalysis, +) +from translator_tom.models.attribute import Attribute +from translator_tom.models.edge_binding import EdgeBinding +from translator_tom.models.path_binding import PathBinding + + +def _eb(edge_id: str) -> EdgeBinding: + return EdgeBinding(id=edge_id, attributes=[]) + + +class TestBaseAnalysis: + def test_list_accessors(self): + a = BaseAnalysis( + resource_id="infores:x", + support_graphs=["a0"], + attributes=[Attribute(attribute_type_id="biolink:x", value=1)], + ) + assert BaseAnalysisDictUtil.support_graphs_list(a.to_dict()) == ["a0"] + assert len(BaseAnalysisDictUtil.attributes_list(a.to_dict())) == 1 + + def test_hash_parity(self): + a = BaseAnalysis( + resource_id="infores:x", + score=0.5, + support_graphs=["a0", "a1"], + scoring_method="method", + ) + assert BaseAnalysisDictUtil.hash(a.to_dict()) == a.hash() + + +class TestAnalysisHashParity: + def test_minimal(self): + a = Analysis(resource_id="infores:x", edge_bindings={}) + assert AnalysisDictUtil.hash(a.to_dict()) == a.hash() + + def test_with_bindings(self): + a = Analysis( + resource_id="infores:x", + score=0.9, + edge_bindings={"e0": [_eb("kg0"), _eb("kg1")]}, + ) + assert AnalysisDictUtil.hash(a.to_dict()) == a.hash() + + +class TestAnalysisUpdate: + def test_merges_edge_bindings(self): + a = Analysis( + resource_id="infores:x", + support_graphs=["a0"], + edge_bindings={"e0": [_eb("kg0")]}, + ) + other = Analysis( + resource_id="infores:x", + support_graphs=["a1"], + edge_bindings={"e0": [_eb("kg1")], "e1": [_eb("kg2")]}, + ) + a_dict = a.to_dict() + a.update(other) + AnalysisDictUtil.update(a_dict, other.to_dict()) + # Bindings are merged via a set in the model (order not meaningful); the + # analysis hash folds them into frozensets, so parity is order-independent. + assert AnalysisDictUtil.hash(a_dict) == a.hash() + assert set(a_dict["support_graphs"]) == set(a.support_graphs or []) + + +class TestPathfinderAnalysis: + def test_hash_parity(self): + a = PathfinderAnalysis( + resource_id="infores:x", + path_bindings={"p0": [PathBinding(id="a0"), PathBinding(id="a1")]}, + ) + assert PathfinderAnalysisDictUtil.hash(a.to_dict()) == a.hash() + + def test_update_parity(self): + a = PathfinderAnalysis( + resource_id="infores:x", path_bindings={"p0": [PathBinding(id="a0")]} + ) + other = PathfinderAnalysis( + resource_id="infores:x", + path_bindings={"p0": [PathBinding(id="a1")], "p1": [PathBinding(id="a2")]}, + ) + a_dict = a.to_dict() + a.update(other) + PathfinderAnalysisDictUtil.update(a_dict, other.to_dict()) + assert PathfinderAnalysisDictUtil.hash(a_dict) == a.hash() + + +class TestAnalysisUpdateBranches: + """Dedup edge cases: intra-key overlap (existing wins), deepcopy isolation, base merge.""" + + def test_intra_key_overlap_existing_wins(self): + a = Analysis(resource_id="infores:x", edge_bindings={"e0": [_eb("kg0")]}) + other = Analysis( + resource_id="infores:x", edge_bindings={"e0": [_eb("kg0"), _eb("kg1")]} + ) + a_dict = a.to_dict() + AnalysisDictUtil.update(a_dict, other.to_dict()) + assert {b["id"] for b in a_dict["edge_bindings"]["e0"]} == {"kg0", "kg1"} + + def test_incoming_bindings_are_deepcopied(self): + # a dropped copy.deepcopy would let a later mutation of `other` leak into the result + a = Analysis(resource_id="infores:x", edge_bindings={"e0": [_eb("kg0")]}) + other = Analysis(resource_id="infores:x", edge_bindings={"e1": [_eb("kg1")]}) + a_dict, other_dict = a.to_dict(), other.to_dict() + AnalysisDictUtil.update(a_dict, other_dict) + other_dict["edge_bindings"]["e1"][0]["id"] = "MUTATED" + assert a_dict["edge_bindings"]["e1"][0]["id"] == "kg1" + + def test_update_base_merges_attributes(self): + a = Analysis( + resource_id="infores:x", + edge_bindings={}, + attributes=[Attribute(attribute_type_id="biolink:x", value=1)], + ) + other = Analysis( + resource_id="infores:x", + edge_bindings={}, + attributes=[Attribute(attribute_type_id="biolink:y", value=2)], + ) + a_dict = a.to_dict() + a.update(other) + AnalysisDictUtil.update(a_dict, other.to_dict()) + assert AnalysisDictUtil.hash(a_dict) == a.hash() + assert len(a_dict.get("attributes", [])) == 2 diff --git a/tests/test_model_dicts/test_attribute_dicts.py b/tests/test_model_dicts/test_attribute_dicts.py new file mode 100644 index 0000000..eb8a58b --- /dev/null +++ b/tests/test_model_dicts/test_attribute_dicts.py @@ -0,0 +1,278 @@ +"""Tests for the `*DictUtil` sibling classes in `model_dicts/attribute.py`. + +The util classes reimplement the utility methods of the `Attribute` and +`AttributeConstraint` Pydantic models for their `TypedDict` equivalents. Tests +assert parity by comparing dict-util results against the models operating on the +same data. +""" + +from __future__ import annotations + +import pytest + +from translator_tom.model_dicts.attribute import ( + AttributeConstraintDictUtil, + AttributeDict, + AttributeDictUtil, +) +from translator_tom.models.attribute import Attribute, AttributeConstraint +from translator_tom.models.meta_attribute import MetaAttribute + +# ============================================================================ +# AttributeDictUtil.attributes_list +# ============================================================================ + + +class TestAttributeDictUtilListAccessor: + def test_missing_key_returns_empty(self): + attr: AttributeDict = {"attribute_type_id": "biolink:foo", "value": 1} + assert AttributeDictUtil.attributes_list(attr) == [] + + def test_explicit_none_returns_empty(self): + attr: AttributeDict = { + "attribute_type_id": "biolink:foo", + "value": 1, + "attributes": None, + } + assert AttributeDictUtil.attributes_list(attr) == [] + + def test_populated_returns_value(self): + sub: AttributeDict = {"attribute_type_id": "biolink:bar", "value": 2} + attr: AttributeDict = { + "attribute_type_id": "biolink:foo", + "value": 1, + "attributes": [sub], + } + assert AttributeDictUtil.attributes_list(attr) == [sub] + + +# ============================================================================ +# AttributeDictUtil.hash — parity with Attribute.hash (incl. nested recursion) +# ============================================================================ + + +class TestAttributeHashParity: + def test_scalar_only(self): + a = Attribute(attribute_type_id="biolink:foo", value=1) + assert AttributeDictUtil.hash(a.to_dict()) == a.hash() + + def test_all_scalar_fields(self): + a = Attribute( + attribute_type_id="biolink:foo", + original_attribute_name="foo", + value=[1, 2, 3], + value_type_id="biolink:bar", + attribute_source="infores:x", + value_url="http://example.com", + description="d", + ) + assert AttributeDictUtil.hash(a.to_dict()) == a.hash() + + def test_nested_attributes(self): + sub = Attribute(attribute_type_id="biolink:sub", value="x") + a = Attribute(attribute_type_id="biolink:foo", value=1, attributes=[sub]) + assert AttributeDictUtil.hash(a.to_dict()) == a.hash() + + def test_deeply_nested_attributes(self): + leaf = Attribute(attribute_type_id="biolink:leaf", value=3) + mid = Attribute(attribute_type_id="biolink:mid", value=2, attributes=[leaf]) + a = Attribute(attribute_type_id="biolink:foo", value=1, attributes=[mid]) + assert AttributeDictUtil.hash(a.to_dict()) == a.hash() + + def test_nested_order_independent(self): + # Attribute.hash folds sub-attributes into a frozenset, so order shouldn't matter. + s1 = Attribute(attribute_type_id="biolink:sub", value="x") + s2 = Attribute(attribute_type_id="biolink:sub", value="y") + a = Attribute(attribute_type_id="biolink:foo", value=1, attributes=[s1, s2]) + b = Attribute(attribute_type_id="biolink:foo", value=1, attributes=[s2, s1]) + assert AttributeDictUtil.hash(a.to_dict()) == AttributeDictUtil.hash(b.to_dict()) + + +# ============================================================================ +# AttributeDictUtil.merge_attribute_lists — parity with Attribute.merge_attribute_lists +# ============================================================================ + + +class TestMergeAttributeLists: + def _assert_parity( + self, old: list[Attribute], new: list[Attribute] + ) -> None: + old_dicts = [m.to_dict() for m in old] + new_dicts = [m.to_dict() for m in new] + Attribute.merge_attribute_lists(old, new) + AttributeDictUtil.merge_attribute_lists(old_dicts, new_dicts) + assert old_dicts == [m.to_dict() for m in old] + + def test_union_with_overlap(self): + self._assert_parity( + [ + Attribute(attribute_type_id="biolink:a", value=1), + Attribute(attribute_type_id="biolink:b", value=2), + ], + [ + Attribute(attribute_type_id="biolink:b", value=2), + Attribute(attribute_type_id="biolink:c", value=3), + ], + ) + + def test_dedupes_within_new(self): + self._assert_parity( + [Attribute(attribute_type_id="biolink:a", value=1)], + [ + Attribute(attribute_type_id="biolink:b", value=2), + Attribute(attribute_type_id="biolink:b", value=2), + ], + ) + + def test_empty_new_leaves_old(self): + self._assert_parity([Attribute(attribute_type_id="biolink:a", value=1)], []) + + def test_empty_old_takes_new(self): + self._assert_parity([], [Attribute(attribute_type_id="biolink:a", value=1)]) + + +# ============================================================================ +# AttributeConstraintDictUtil.hash — parity, incl. the nullable `not` alias +# ============================================================================ + + +class TestAttributeConstraintHashParity: + @pytest.mark.parametrize("negated", [True, False]) + def test_negated_states(self, negated: bool): + c = AttributeConstraint( + id="biolink:foo", name="Foo", operator="==", value=1, negated=negated + ) + assert AttributeConstraintDictUtil.hash(c.to_dict()) == c.hash() + + def test_default_negated(self): + c = AttributeConstraint(id="biolink:foo", name="Foo", operator="==", value=1) + assert AttributeConstraintDictUtil.hash(c.to_dict()) == c.hash() + + def test_with_units(self): + c = AttributeConstraint( + id="biolink:foo", + name="Foo", + operator=">", + value=5, + unit_id="UO:0000221", + unit_name="gram", + ) + assert AttributeConstraintDictUtil.hash(c.to_dict()) == c.hash() + + +# ============================================================================ +# AttributeConstraintDictUtil.met_by — parity with AttributeConstraint.met_by +# ============================================================================ + + +def _con( + operator: str, value: object, *, negated: bool = False +) -> AttributeConstraint: + return AttributeConstraint( + id="biolink:foo", + name="Foo", + operator=operator, # type: ignore[arg-type] + value=value, + negated=negated, + ) + + +def _attr(value: object, type_id: str = "biolink:foo") -> Attribute: + return Attribute(attribute_type_id=type_id, value=value) + + +def _assert_met_by_parity( + con: AttributeConstraint, attribute: Attribute | MetaAttribute +) -> None: + assert AttributeConstraintDictUtil.met_by( + con.to_dict(), attribute.to_dict() + ) == con.met_by(attribute) + + +class TestAttributeConstraintMetByAttribute: + @pytest.mark.parametrize( + ("operator", "con_value", "attr_value"), + [ + ("==", 1, 1), + ("==", 1, 2), + ("==", 2, [1, 2, 3]), + ("==", [2, 3], 2), + ("==", [4, 5], [1, 2, 3]), + ("===", [1, 2, 3], [1, 2, 3]), + ("===", [1, 2, 3], [3, 2, 1]), + ("===", [1], 1), + (">", 5, 10), + (">", 5, 3), + ("<", 5, 3), + ("<", 5, 10), + ("matches", "^bio", "biolink:x"), + ("matches", "^xyz", "biolink:x"), + ], + ) + def test_operators(self, operator: str, con_value: object, attr_value: object): + _assert_met_by_parity(_con(operator, con_value), _attr(attr_value)) + + def test_negated_flips(self): + _assert_met_by_parity(_con("==", 1, negated=True), _attr(1)) + _assert_met_by_parity(_con("==", 1, negated=True), _attr(2)) + + def test_type_id_mismatch(self): + _assert_met_by_parity(_con("==", 1), _attr(1, type_id="biolink:other")) + + +class TestAttributeConstraintMetByMetaAttribute: + @pytest.mark.parametrize("constraint_use", [True, False]) + def test_constraint_use_states(self, constraint_use: bool): + _assert_met_by_parity( + _con("==", 1), + MetaAttribute( + attribute_type_id="biolink:foo", constraint_use=constraint_use + ), + ) + + def test_type_id_mismatch(self): + _assert_met_by_parity( + _con("==", 1), + MetaAttribute(attribute_type_id="biolink:other", constraint_use=True), + ) + + +# ============================================================================ +# AttributeConstraintDictUtil.set_met_by — parity with AttributeConstraint.set_met_by +# ============================================================================ + + +class TestAttributeConstraintSetMetBy: + def _assert_parity( + self, + constraints: list[AttributeConstraint], + attributes: list[Attribute], + ) -> None: + result = AttributeConstraintDictUtil.set_met_by( + [c.to_dict() for c in constraints], [a.to_dict() for a in attributes] + ) + assert result == AttributeConstraint.set_met_by(constraints, attributes) + + def test_empty_constraints_is_true(self): + self._assert_parity([], [_attr(1)]) + + def test_constraints_but_no_attributes_is_false(self): + self._assert_parity([_con("==", 1)], []) + + def test_all_constraints_met(self): + self._assert_parity( + [_con("==", 1), _con("==", 2, negated=True)], + [_attr(1)], + ) + + def test_one_constraint_unmet(self): + self._assert_parity( + [_con("==", 1), _con("==", 99)], + [_attr(1)], + ) + + def test_multiple_attributes_grouped_by_type(self): + self._assert_parity( + [_con("==", 1)], + [_attr(0), _attr(1), _attr(2, type_id="biolink:other")], + ) diff --git a/tests/test_model_dicts/test_auxiliary_graph_dicts.py b/tests/test_model_dicts/test_auxiliary_graph_dicts.py new file mode 100644 index 0000000..386a825 --- /dev/null +++ b/tests/test_model_dicts/test_auxiliary_graph_dicts.py @@ -0,0 +1,91 @@ +"""Tests for `AuxiliaryGraphDictUtil`, asserting parity with the model.""" + +from __future__ import annotations + +from translator_tom.model_dicts.auxiliary_graph import ( + AuxiliaryGraphDict, + AuxiliaryGraphDictUtil, +) +from translator_tom.models.attribute import Attribute +from translator_tom.models.auxiliary_graph import AuxiliaryGraph + + +def _aux(*edges: str, attrs: list[Attribute] | None = None) -> AuxiliaryGraph: + return AuxiliaryGraph(edges=list(edges), attributes=attrs or []) + + +class TestHashParity: + def test_no_attributes(self): + a = _aux("e0", "e1") + assert AuxiliaryGraphDictUtil.hash(a.to_dict()) == a.hash() + + def test_with_attributes(self): + a = _aux("e0", attrs=[Attribute(attribute_type_id="biolink:x", value=1)]) + assert AuxiliaryGraphDictUtil.hash(a.to_dict()) == a.hash() + + def test_edges_unordered(self): + a = _aux("e0", "e1") + b = _aux("e1", "e0") + assert AuxiliaryGraphDictUtil.hash(a.to_dict()) == AuxiliaryGraphDictUtil.hash( + b.to_dict() + ) + + +class TestNormalize: + def test_parity(self): + model = _aux("e0", "e1", "e2") + aux_dict = model.to_dict() + mapping = {"e0": "n0", "e2": "n2"} + model.normalize(mapping) + AuxiliaryGraphDictUtil.normalize(aux_dict, mapping) + assert aux_dict["edges"] == model.edges + + def test_normalize_aux_dict_parity(self): + model_map = {"a0": _aux("e0"), "a1": _aux("e1")} + dict_map: dict[str, AuxiliaryGraphDict] = { + k: v.to_dict() for k, v in model_map.items() + } + mapping = {"e0": "n0"} + for v in model_map.values(): + v.normalize(mapping) + AuxiliaryGraphDictUtil.normalize_aux_dict(dict_map, mapping) + assert dict_map == {k: v.to_dict() for k, v in model_map.items()} + + +class TestUpdate: + def test_takes_other_attributes_when_empty(self): + model = _aux("e0") + other = _aux("e0", attrs=[Attribute(attribute_type_id="biolink:x", value=1)]) + aux_dict = model.to_dict() + model.update(other) + AuxiliaryGraphDictUtil.update(aux_dict, other.to_dict()) + assert aux_dict == model.to_dict() + + def test_merges_attributes(self): + model = _aux("e0", attrs=[Attribute(attribute_type_id="biolink:x", value=1)]) + other = _aux("e0", attrs=[Attribute(attribute_type_id="biolink:y", value=2)]) + aux_dict = model.to_dict() + model.update(other) + AuxiliaryGraphDictUtil.update(aux_dict, other.to_dict()) + assert aux_dict == model.to_dict() + + +class TestMergeDictionaries: + def test_parity(self): + old_models = { + "a0": _aux("e0", attrs=[Attribute(attribute_type_id="biolink:x", value=1)]), + "a1": _aux("e1"), + } + new_models = { + "a1": _aux("e1", attrs=[Attribute(attribute_type_id="biolink:y", value=2)]), + "a2": _aux("e2"), + } + old_dicts: dict[str, AuxiliaryGraphDict] = { + k: v.to_dict() for k, v in old_models.items() + } + new_dicts: dict[str, AuxiliaryGraphDict] = { + k: v.to_dict() for k, v in new_models.items() + } + AuxiliaryGraph.merge_dictionaries(old_models, new_models) + AuxiliaryGraphDictUtil.merge_dictionaries(old_dicts, new_dicts) + assert old_dicts == {k: v.to_dict() for k, v in old_models.items()} diff --git a/tests/test_model_dicts/test_binding_dicts.py b/tests/test_model_dicts/test_binding_dicts.py new file mode 100644 index 0000000..eb36c61 --- /dev/null +++ b/tests/test_model_dicts/test_binding_dicts.py @@ -0,0 +1,44 @@ +"""Hash-parity tests for the binding `*DictUtil` classes (edge/node/path).""" + +from __future__ import annotations + +from translator_tom.model_dicts.edge_binding import EdgeBindingDictUtil +from translator_tom.model_dicts.node_binding import NodeBindingDictUtil +from translator_tom.model_dicts.path_binding import PathBindingDictUtil +from translator_tom.models.attribute import Attribute +from translator_tom.models.edge_binding import EdgeBinding +from translator_tom.models.node_binding import NodeBinding +from translator_tom.models.path_binding import PathBinding + + +def _attrs() -> list[Attribute]: + return [ + Attribute(attribute_type_id="biolink:score", value=0.9), + Attribute(attribute_type_id="biolink:source", value="infores:x"), + ] + + +class TestEdgeBindingHashParity: + def test_no_attributes(self): + eb = EdgeBinding(id="e0", attributes=[]) + assert EdgeBindingDictUtil.hash(eb.to_dict()) == eb.hash() + + def test_with_attributes(self): + eb = EdgeBinding(id="e0", attributes=_attrs()) + assert EdgeBindingDictUtil.hash(eb.to_dict()) == eb.hash() + + +class TestNodeBindingHashParity: + def test_minimal(self): + nb = NodeBinding(id="CHEBI:1", attributes=[]) + assert NodeBindingDictUtil.hash(nb.to_dict()) == nb.hash() + + def test_with_query_id_and_attributes(self): + nb = NodeBinding(id="CHEBI:1", query_id="MONDO:1", attributes=_attrs()) + assert NodeBindingDictUtil.hash(nb.to_dict()) == nb.hash() + + +class TestPathBindingHashParity: + def test_base_hash(self): + pb = PathBinding(id="a0") + assert PathBindingDictUtil.hash(pb.to_dict()) == pb.hash() diff --git a/tests/test_model_dicts/test_knowledge_graph_dicts.py b/tests/test_model_dicts/test_knowledge_graph_dicts.py new file mode 100644 index 0000000..8fc4ce7 --- /dev/null +++ b/tests/test_model_dicts/test_knowledge_graph_dicts.py @@ -0,0 +1,350 @@ +"""Parity tests for the knowledge-graph `*DictUtil` classes.""" + +from __future__ import annotations + +import pytest + +from translator_tom.model_dicts.knowledge_graph import ( + EdgeDictUtil, + KnowledgeGraphDictUtil, + NodeDictUtil, +) +from translator_tom.models.analysis import Analysis +from translator_tom.models.attribute import Attribute, AttributeConstraint +from translator_tom.models.auxiliary_graph import AuxiliaryGraph +from translator_tom.models.edge_binding import EdgeBinding +from translator_tom.models.knowledge_graph import Edge, KnowledgeGraph, Node +from translator_tom.models.node_binding import NodeBinding +from translator_tom.models.result import Result +from translator_tom.models.retrieval_source import RetrievalSource + + +def _source( + role: str = "primary_knowledge_source", + rid: str = "infores:p", + upstream: list[str] | None = None, +) -> RetrievalSource: + return RetrievalSource( + resource_id=rid, + resource_role=role, # type: ignore[arg-type] + upstream_resource_ids=upstream, + ) + + +def _edge(subject: str, obj: str, **kw: object) -> Edge: + kw.setdefault("sources", [_source()]) + return Edge(predicate="biolink:related_to", subject=subject, object=obj, **kw) # type: ignore[arg-type] + + +def _node(*categories: str, attributes: list[Attribute] | None = None, **kw: object) -> Node: + return Node( + categories=list(categories) or ["biolink:NamedThing"], + attributes=attributes or [], + **kw, # type: ignore[arg-type] + ) + + +# ============================================================================ +# Node +# ============================================================================ + + +class TestNode: + def test_hash_parity(self): + node = _node("biolink:Gene", name="BRCA1", is_set=False) + assert NodeDictUtil.hash(node.to_dict()) == node.hash() + + def test_hash_ignores_categories_and_attributes(self): + a = _node("biolink:Gene", name="X") + b = _node( + "biolink:Disease", + name="X", + attributes=[Attribute(attribute_type_id="biolink:z", value=1)], + ) + assert NodeDictUtil.hash(a.to_dict()) == NodeDictUtil.hash(b.to_dict()) + + def test_meets_constraints_parity(self): + node = _node( + "biolink:Gene", + attributes=[Attribute(attribute_type_id="biolink:foo", value=1)], + ) + constraints = [ + AttributeConstraint(id="biolink:foo", name="Foo", operator="==", value=1) + ] + assert NodeDictUtil.meets_constraints( + node.to_dict(), [c.to_dict() for c in constraints] + ) == node.meets_constraints(constraints) + + def test_update_parity(self): + node = _node( + "biolink:Gene", + name=None, + attributes=[Attribute(attribute_type_id="biolink:a", value=1)], + ) + other = _node( + "biolink:Disease", + name="Named", + attributes=[Attribute(attribute_type_id="biolink:b", value=2)], + ) + node_dict = node.to_dict() + node.update(other) + NodeDictUtil.update(node_dict, other.to_dict()) + assert node_dict == node.to_dict() + + +# ============================================================================ +# Edge +# ============================================================================ + + +class TestEdge: + def test_list_accessors(self): + edge = _edge("n0", "n1") + assert EdgeDictUtil.attributes_list(edge.to_dict()) == [] + assert EdgeDictUtil.qualifiers_list(edge.to_dict()) == [] + + def test_hash_parity(self): + edge = _edge("n0", "n1") + assert EdgeDictUtil.hash(edge.to_dict()) == edge.hash() + + def test_primary_knowledge_source_parity(self): + edge = _edge( + "n0", + "n1", + sources=[ + _source(role="aggregator_knowledge_source", rid="infores:a"), + _source(rid="infores:p"), + ], + ) + assert ( + EdgeDictUtil.primary_knowledge_source(edge.to_dict()) + == edge.primary_knowledge_source.to_dict() + ) + + def test_primary_knowledge_source_raises(self): + edge = _edge( + "n0", "n1", sources=[_source(role="aggregator_knowledge_source")] + ) + with pytest.raises(ValueError, match="no .*primary_knowledge_source"): + EdgeDictUtil.primary_knowledge_source(edge.to_dict()) + + def test_last_downstream_source_parity(self): + edge = _edge( + "n0", + "n1", + sources=[ + _source(rid="infores:p"), + _source( + role="aggregator_knowledge_source", + rid="infores:a", + upstream=["infores:p"], + ), + ], + ) + result = EdgeDictUtil.last_downstream_source(edge.to_dict()) + expected = edge.last_downstream_source + assert result == (expected.to_dict() if expected is not None else None) + + def test_is_self_edge_parity(self): + assert EdgeDictUtil.is_self_edge(_edge("n0", "n0").to_dict()) is True + assert EdgeDictUtil.is_self_edge(_edge("n0", "n1").to_dict()) is False + + def test_support_graphs_parity(self): + edge = _edge( + "n0", + "n1", + attributes=[ + Attribute( + attribute_type_id="biolink:support_graphs", value=["a0", "a1"] + ) + ], + ) + assert EdgeDictUtil.support_graphs(edge.to_dict()) == edge.support_graphs + + def test_update_parity(self): + edge = _edge( + "n0", + "n1", + attributes=[Attribute(attribute_type_id="biolink:a", value=1)], + ) + other = _edge( + "n0", + "n1", + attributes=[Attribute(attribute_type_id="biolink:b", value=2)], + sources=[_source(role="aggregator_knowledge_source", rid="infores:a2")], + ) + edge_dict = edge.to_dict() + edge.update(other) + EdgeDictUtil.update(edge_dict, other.to_dict()) + assert edge_dict == edge.to_dict() + + def test_meets_attribute_constraints_parity(self): + edge = _edge( + "n0", + "n1", + attributes=[Attribute(attribute_type_id="biolink:foo", value=5)], + ) + constraints = [ + AttributeConstraint(id="biolink:foo", name="Foo", operator=">", value=1) + ] + assert EdgeDictUtil.meets_attribute_constraints( + edge.to_dict(), [c.to_dict() for c in constraints] + ) == edge.meets_attribute_constraints(constraints) + + def test_append_aggregator_parity(self): + edge = _edge("n0", "n1", sources=[_source(rid="infores:p")]) + edge_dict = edge.to_dict() + edge.append_aggregator("infores:agg") + EdgeDictUtil.append_aggregator(edge_dict, "infores:agg") + assert edge_dict["sources"] == edge.to_dict()["sources"] + + +# ============================================================================ +# KnowledgeGraph +# ============================================================================ + + +class TestKnowledgeGraph: + def test_new(self): + assert KnowledgeGraphDictUtil.new() == KnowledgeGraph.new().to_dict() + assert KnowledgeGraphDictUtil.new() == {"nodes": {}, "edges": {}} + + def test_normalize_parity(self): + kg = KnowledgeGraph( + nodes={"n0": _node("biolink:Gene")}, + edges={"e0": _edge("n0", "n1")}, + ) + kg_dict = kg.to_dict() + model_mapping = kg.normalize() + dict_mapping = KnowledgeGraphDictUtil.normalize(kg_dict) + assert dict_mapping == model_mapping + assert kg_dict == kg.to_dict() + + def test_update_parity(self): + kg = KnowledgeGraph( + nodes={"n0": _node("biolink:Gene", name="A")}, + edges={"e0": _edge("n0", "n1")}, + ) + other = KnowledgeGraph( + nodes={"n2": _node("biolink:Disease")}, + edges={"e1": _edge("n2", "n3")}, + ) + kg_dict = kg.to_dict() + model_mapping = kg.update(other) + dict_mapping = KnowledgeGraphDictUtil.update(kg_dict, other.to_dict()) + assert dict_mapping == model_mapping + assert kg_dict == kg.to_dict() + + def test_prune_parity(self): + kg = KnowledgeGraph( + nodes={ + "n0": _node("biolink:Gene"), + "n1": _node("biolink:Gene"), + "n2": _node("biolink:Gene"), # unused + }, + edges={ + "e0": _edge("n0", "n1"), + "e1": _edge("n1", "n2"), # unused + }, + ) + result = Result( + node_bindings={"qn0": [NodeBinding(id="n0", attributes=[])]}, + analyses=[ + Analysis( + resource_id="infores:x", + edge_bindings={"qe0": [EdgeBinding(id="e0", attributes=[])]}, + ) + ], + ) + aux_graphs: dict[str, AuxiliaryGraph] = {} + kg_dict = kg.to_dict() + kg.prune(aux_graphs, [result]) + KnowledgeGraphDictUtil.prune(kg_dict, {}, [result.to_dict()]) + assert kg_dict == kg.to_dict() + + +class TestEdgeUpdateBranches: + """The subtle Edge.update branches: overlapping-source upstream roll-up + KL/AT skip.""" + + def test_merges_overlapping_source_upstreams(self): + # same (resource_id, role) -> same source hash -> upstreams rolled up (set union) + edge = _edge( + "n0", "n1", sources=[_source(rid="infores:p", upstream=["infores:a"])] + ) + other = _edge( + "n0", "n1", sources=[_source(rid="infores:p", upstream=["infores:b"])] + ) + edge_dict = edge.to_dict() + edge.update(other) + EdgeDictUtil.update(edge_dict, other.to_dict()) + dict_up = { + s["resource_id"]: set(s.get("upstream_resource_ids") or []) + for s in edge_dict["sources"] + } + model_up = { + s.resource_id: set(s.upstream_resource_ids or []) for s in edge.sources + } + assert dict_up == model_up + assert dict_up["infores:p"] == {"infores:a", "infores:b"} + + def test_update_skips_knowledge_level_and_agent_type(self): + edge = _edge( + "n0", "n1", attributes=[Attribute(attribute_type_id="biolink:a", value=1)] + ) + other = _edge( + "n0", + "n1", + attributes=[ + Attribute(attribute_type_id="biolink:knowledge_level", value="ka"), + Attribute(attribute_type_id="biolink:agent_type", value="manual"), + Attribute(attribute_type_id="biolink:b", value=2), + ], + ) + edge_dict = edge.to_dict() + edge.update(other) + EdgeDictUtil.update(edge_dict, other.to_dict()) + assert edge_dict == edge.to_dict() + types = {a["attribute_type_id"] for a in edge_dict.get("attributes", [])} + assert "biolink:knowledge_level" not in types + assert "biolink:agent_type" not in types + + +class TestPruneSupportGraphWalk: + def test_prune_follows_support_graph_edges(self): + kg = KnowledgeGraph( + nodes={ + "n0": _node("biolink:Gene"), + "n1": _node("biolink:Gene"), + "n2": _node("biolink:Gene"), + }, + edges={ + "e0": _edge( + "n0", + "n1", + attributes=[ + Attribute( + attribute_type_id="biolink:support_graphs", value=["aux0"] + ) + ], + ), + "e1": _edge("n1", "n2"), # reachable only via aux0's support graph + "e_orphan": _edge("n2", "n0"), # unused + }, + ) + result = Result( + node_bindings={"qn0": [NodeBinding(id="n0", attributes=[])]}, + analyses=[ + Analysis( + resource_id="infores:x", + edge_bindings={"qe0": [EdgeBinding(id="e0", attributes=[])]}, + ) + ], + ) + aux = {"aux0": AuxiliaryGraph(edges=["e1"], attributes=[])} + kg_dict = kg.to_dict() + kg.prune(aux, [result]) + KnowledgeGraphDictUtil.prune( + kg_dict, {k: v.to_dict() for k, v in aux.items()}, [result.to_dict()] + ) + assert kg_dict == kg.to_dict() + assert set(kg_dict["edges"]) == {"e0", "e1"} # e_orphan pruned diff --git a/tests/test_model_dicts/test_log_entry_dicts.py b/tests/test_model_dicts/test_log_entry_dicts.py new file mode 100644 index 0000000..6bc5f37 --- /dev/null +++ b/tests/test_model_dicts/test_log_entry_dicts.py @@ -0,0 +1,42 @@ +"""Tests for `LogEntryDictUtil`, asserting parity with the `LogEntry` model.""" + +from __future__ import annotations + +from translator_tom.model_dicts.log_entry import LogEntryDict, LogEntryDictUtil +from translator_tom.models.log_entry import LogEntry + + +class TestTimestampDt: + def test_z_suffix_parity(self): + entry: LogEntryDict = {"timestamp": "2020-09-03T18:13:49Z", "message": "hi"} + model = LogEntry(timestamp="2020-09-03T18:13:49Z", message="hi") + assert LogEntryDictUtil.timestamp_dt(entry) == model.timestamp_dt + + def test_offset_parity(self): + entry: LogEntryDict = { + "timestamp": "2020-09-03T18:13:49-04:00", + "message": "hi", + } + model = LogEntry(timestamp="2020-09-03T18:13:49-04:00", message="hi") + assert LogEntryDictUtil.timestamp_dt(entry) == model.timestamp_dt + + +class TestNew: + def test_minimal_omits_none(self): + entry = LogEntryDictUtil.new("a message") + assert entry["message"] == "a message" + assert "level" not in entry + assert "code" not in entry + # timestamp must be valid and round-trip through timestamp_dt + assert LogEntryDictUtil.timestamp_dt(entry) is not None + + def test_with_level_and_code(self): + entry = LogEntryDictUtil.new("msg", level="ERROR", code="KPNotAvailable") + assert entry["level"] == "ERROR" + assert entry["code"] == "KPNotAvailable" + + def test_matches_model_new_shape(self): + entry = LogEntryDictUtil.new("msg", level="INFO") + model_dict = LogEntry.new("msg", level="INFO").to_dict() + # Same keys (timestamps differ by construction time). + assert set(entry) == set(model_dict) diff --git a/tests/test_model_dicts/test_message_dicts.py b/tests/test_model_dicts/test_message_dicts.py new file mode 100644 index 0000000..3d9b638 --- /dev/null +++ b/tests/test_model_dicts/test_message_dicts.py @@ -0,0 +1,158 @@ +"""Parity tests for `MessageDictUtil`.""" + +from __future__ import annotations + +import pytest + +from translator_tom.model_dicts.message import MessageDict, MessageDictUtil +from translator_tom.models.analysis import Analysis +from translator_tom.models.auxiliary_graph import AuxiliaryGraph +from translator_tom.models.edge_binding import EdgeBinding +from translator_tom.models.knowledge_graph import Edge, KnowledgeGraph, Node +from translator_tom.models.message import Message +from translator_tom.models.node_binding import NodeBinding +from translator_tom.models.query_graph import QNode, QueryGraph +from translator_tom.models.result import Result +from translator_tom.models.retrieval_source import RetrievalSource + + +def _edge(subject: str, obj: str) -> Edge: + return Edge( + predicate="biolink:treats", + subject=subject, + object=obj, + sources=[ + RetrievalSource( + resource_id="infores:x", resource_role="primary_knowledge_source" + ) + ], + ) + + +def _message() -> Message: + return Message( + knowledge_graph=KnowledgeGraph( + nodes={ + "CHEBI:1": Node(categories=["biolink:ChemicalEntity"], attributes=[]), + "MONDO:1": Node(categories=["biolink:Disease"], attributes=[]), + }, + edges={"kg0": _edge("CHEBI:1", "MONDO:1")}, + ), + results=[ + Result( + node_bindings={"n0": [NodeBinding(id="CHEBI:1", attributes=[])]}, + analyses=[ + Analysis( + resource_id="infores:x", + edge_bindings={"e0": [EdgeBinding(id="kg0", attributes=[])]}, + ) + ], + ) + ], + auxiliary_graphs={"a0": AuxiliaryGraph(edges=["kg0"], attributes=[])}, + ) + + +class TestListAccessors: + def test_results_list(self): + m = _message() + assert len(MessageDictUtil.results_list(m.to_dict())) == 1 + assert MessageDictUtil.results_list({}) == [] + + def test_auxiliary_graphs_dict(self): + m = _message() + assert set(MessageDictUtil.auxiliary_graphs_dict(m.to_dict())) == {"a0"} + assert MessageDictUtil.auxiliary_graphs_dict({}) == {} + + +class TestHashParity: + def test_full_message(self): + m = _message() + assert MessageDictUtil.hash(m.to_dict()) == m.hash() + + +class TestNormalize: + def test_parity(self): + m = _message() + m_dict = m.to_dict() + model_mapping = m.normalize() + dict_mapping = MessageDictUtil.normalize(m_dict) + assert dict_mapping == model_mapping + assert m_dict == m.to_dict() + + +class TestPruneKg: + def test_parity(self): + m = _message() + # Add an unused node/edge that pruning should drop. + assert m.knowledge_graph is not None + m.knowledge_graph.nodes["ORPHAN:1"] = Node( + categories=["biolink:Gene"], attributes=[] + ) + m.knowledge_graph.edges["orphan"] = _edge("ORPHAN:1", "ORPHAN:2") + m_dict = m.to_dict() + m.prune_kg() + MessageDictUtil.prune_kg(m_dict) + assert m_dict == m.to_dict() + + def test_none_kg_is_noop(self): + message: MessageDict = {} + MessageDictUtil.prune_kg(message) + assert message == {} + + +class TestUpdate: + def test_merges_and_hash_parity(self): + m = _message() + other = Message( + results=[ + Result( + node_bindings={"n0": [NodeBinding(id="DRUGBANK:2", attributes=[])]}, + analyses=[Analysis(resource_id="infores:y", edge_bindings={})], + ) + ], + ) + m_dict = m.to_dict() + model_mapping = m.update(other) + dict_mapping = MessageDictUtil.update(m_dict, other.to_dict()) + assert dict_mapping == model_mapping + # Message.hash covers kg/results(node-bindings)/aux, robust to analysis ordering. + assert MessageDictUtil.hash(m_dict) == m.hash() + assert len(MessageDictUtil.results_list(m_dict)) == len(m.results_list) + + def test_mismatched_query_graph_raises(self): + m: MessageDict = {"query_graph": {"nodes": {"n0": {}}, "edges": {}}} + other: MessageDict = {"query_graph": {"nodes": {"n1": {}}, "edges": {}}} + with pytest.raises(NotImplementedError): + MessageDictUtil.update(m, other) + + def test_merges_kg_and_auxiliary_graphs(self): + # `other` carries a kg + aux (not just results) to exercise the + # KnowledgeGraphDictUtil.update and merge_dictionaries branches. + m = _message() + other = Message( + knowledge_graph=KnowledgeGraph( + nodes={ + "CHEBI:2": Node(categories=["biolink:ChemicalEntity"], attributes=[]) + }, + edges={"kg1": _edge("CHEBI:2", "MONDO:2")}, + ), + auxiliary_graphs={"a1": AuxiliaryGraph(edges=["kg1"], attributes=[])}, + ) + m_dict = m.to_dict() + model_mapping = m.update(other) + dict_mapping = MessageDictUtil.update(m_dict, other.to_dict()) + assert dict_mapping == model_mapping + assert m_dict == m.to_dict() + + def test_query_graphs_equal_but_extra_key_do_not_raise(self): + # Query graphs equal by hash but differing by an extra key: the model ignores + # extras (hash-based `==`), so MessageDictUtil.update must not raise either. + qg = QueryGraph(nodes={"n0": QNode()}, edges={}) + qg_extra = QueryGraph(nodes={"n0": QNode()}, edges={}, foo="bar") + m = Message(query_graph=qg) + other = Message(query_graph=qg_extra) + m_dict = m.to_dict() + m.update(other) + MessageDictUtil.update(m_dict, other.to_dict()) + assert MessageDictUtil.hash(m_dict) == m.hash() diff --git a/tests/test_model_dicts/test_meta_dicts.py b/tests/test_model_dicts/test_meta_dicts.py new file mode 100644 index 0000000..5b16ec7 --- /dev/null +++ b/tests/test_model_dicts/test_meta_dicts.py @@ -0,0 +1,133 @@ +"""Tests for the `*DictUtil` classes in `model_dicts/meta_attribute.py` and +`model_dicts/meta_qualifier.py`, asserting parity with their Pydantic models. +""" + +from __future__ import annotations + +import pytest + +from translator_tom.model_dicts.meta_attribute import ( + MetaAttributeDict, + MetaAttributeDictUtil, +) +from translator_tom.model_dicts.meta_qualifier import ( + MetaQualifierDict, + MetaQualifierDictUtil, +) +from translator_tom.models.meta_attribute import MetaAttribute +from translator_tom.models.meta_qualifier import MetaQualifier + +# ============================================================================ +# MetaAttributeDictUtil +# ============================================================================ + + +class TestMetaAttributeListAccessor: + def test_missing_returns_empty(self): + attr: MetaAttributeDict = {"attribute_type_id": "biolink:foo"} + assert MetaAttributeDictUtil.original_attribute_names_list(attr) == [] + + def test_none_returns_empty(self): + attr: MetaAttributeDict = { + "attribute_type_id": "biolink:foo", + "original_attribute_names": None, + } + assert MetaAttributeDictUtil.original_attribute_names_list(attr) == [] + + def test_populated(self): + attr: MetaAttributeDict = { + "attribute_type_id": "biolink:foo", + "original_attribute_names": ["col_a", "col_b"], + } + assert MetaAttributeDictUtil.original_attribute_names_list(attr) == [ + "col_a", + "col_b", + ] + + +class TestMetaAttributeHashParity: + @pytest.mark.parametrize("constraint_use", [True, False]) + def test_constraint_use_states(self, constraint_use: bool): + m = MetaAttribute( + attribute_type_id="biolink:foo", + attribute_source="infores:x", + constraint_use=constraint_use, + ) + assert MetaAttributeDictUtil.hash(m.to_dict()) == m.hash() + + def test_minimal(self): + m = MetaAttribute(attribute_type_id="biolink:foo") + assert MetaAttributeDictUtil.hash(m.to_dict()) == m.hash() + + def test_hash_ignores_non_identity_fields(self): + # Only attribute_type_id/source/constraint_use feed the hash. + a = MetaAttribute( + attribute_type_id="biolink:foo", original_attribute_names=["x"] + ) + b = MetaAttribute( + attribute_type_id="biolink:foo", original_attribute_names=["y"] + ) + assert MetaAttributeDictUtil.hash(a.to_dict()) == MetaAttributeDictUtil.hash( + b.to_dict() + ) + + +class TestMetaAttributeMerge: + def _assert_parity( + self, old: list[MetaAttribute], new: list[MetaAttribute] + ) -> None: + old_dicts = [m.to_dict() for m in old] + new_dicts = [m.to_dict() for m in new] + MetaAttribute.merge_attribute_lists(old, new) + MetaAttributeDictUtil.merge_attribute_lists(old_dicts, new_dicts) + assert old_dicts == [m.to_dict() for m in old] + + def test_union_with_overlap(self): + self._assert_parity( + [ + MetaAttribute(attribute_type_id="biolink:a", constraint_use=True), + MetaAttribute(attribute_type_id="biolink:b"), + ], + [ + MetaAttribute(attribute_type_id="biolink:b"), + MetaAttribute(attribute_type_id="biolink:c"), + ], + ) + + +# ============================================================================ +# MetaQualifierDictUtil +# ============================================================================ + + +class TestMetaQualifierListAccessor: + def test_missing_returns_empty(self): + mq: MetaQualifierDict = {"qualifier_type_id": "biolink:subject_aspect_qualifier"} + assert MetaQualifierDictUtil.applicable_values_list(mq) == [] + + def test_none_returns_empty(self): + mq: MetaQualifierDict = { + "qualifier_type_id": "biolink:subject_aspect_qualifier", + "applicable_values": None, + } + assert MetaQualifierDictUtil.applicable_values_list(mq) == [] + + def test_populated(self): + mq: MetaQualifierDict = { + "qualifier_type_id": "biolink:subject_aspect_qualifier", + "applicable_values": ["activity", "abundance"], + } + assert MetaQualifierDictUtil.applicable_values_list(mq) == [ + "activity", + "abundance", + ] + + def test_parity_with_model(self): + m = MetaQualifier( + qualifier_type_id="biolink:subject_aspect_qualifier", + applicable_values=["activity"], + ) + assert ( + MetaQualifierDictUtil.applicable_values_list(m.to_dict()) + == m.applicable_values_list + ) diff --git a/tests/test_model_dicts/test_meta_knowledge_graph_dicts.py b/tests/test_model_dicts/test_meta_knowledge_graph_dicts.py new file mode 100644 index 0000000..8c2b5fa --- /dev/null +++ b/tests/test_model_dicts/test_meta_knowledge_graph_dicts.py @@ -0,0 +1,195 @@ +"""Parity tests for the meta-knowledge-graph `*DictUtil` classes.""" + +from __future__ import annotations + +from translator_tom.model_dicts.meta_knowledge_graph import ( + MetaEdgeDictUtil, + MetaKnowledgeGraphDictUtil, + MetaNodeDictUtil, +) +from translator_tom.models.attribute import AttributeConstraint +from translator_tom.models.meta_attribute import MetaAttribute +from translator_tom.models.meta_knowledge_graph import ( + MetaEdge, + MetaKnowledgeGraph, + MetaNode, +) +from translator_tom.models.meta_qualifier import MetaQualifier +from translator_tom.models.qualifier import Qualifier, QualifierConstraint + +# ============================================================================ +# MetaNode +# ============================================================================ + + +class TestMetaNode: + def test_attributes_list(self): + node = MetaNode( + id_prefixes=["CHEBI"], attributes=[MetaAttribute(attribute_type_id="x")] + ) + assert MetaNodeDictUtil.attributes_list(node.to_dict()) == node.to_dict()[ + "attributes" + ] + + def test_hash_parity(self): + node = MetaNode( + id_prefixes=["CHEBI", "PUBCHEM"], + attributes=[MetaAttribute(attribute_type_id="biolink:x")], + ) + assert MetaNodeDictUtil.hash(node.to_dict()) == node.hash() + + def test_update_parity(self): + node = MetaNode( + id_prefixes=["CHEBI"], + attributes=[MetaAttribute(attribute_type_id="biolink:a")], + ) + other = MetaNode( + id_prefixes=["PUBCHEM"], + attributes=[MetaAttribute(attribute_type_id="biolink:b")], + ) + node_dict = node.to_dict() + node.update(other) + MetaNodeDictUtil.update(node_dict, other.to_dict()) + assert node_dict == node.to_dict() + + +# ============================================================================ +# MetaEdge +# ============================================================================ + + +def _meta_edge(**kwargs: object) -> MetaEdge: + base: dict[str, object] = { + "subject": "biolink:Gene", + "predicate": "biolink:affects", + "object": "biolink:Disease", + } + base.update(kwargs) + return MetaEdge(**base) # type: ignore[arg-type] + + +class TestMetaEdge: + def test_list_accessors(self): + edge = _meta_edge(knowledge_types=["lookup"]) + assert MetaEdgeDictUtil.knowledge_types_list(edge.to_dict()) == ["lookup"] + assert MetaEdgeDictUtil.attributes_list(edge.to_dict()) == [] + assert MetaEdgeDictUtil.qualifiers_list(edge.to_dict()) == [] + + def test_hash_parity(self): + edge = _meta_edge( + knowledge_types=["lookup"], + attributes=[MetaAttribute(attribute_type_id="biolink:x")], + qualifiers=[ + MetaQualifier( + qualifier_type_id="biolink:subject_aspect_qualifier", + applicable_values=["activity"], + ) + ], + ) + assert MetaEdgeDictUtil.hash(edge.to_dict()) == edge.hash() + + def test_update_parity(self): + edge = _meta_edge( + knowledge_types=["lookup"], + attributes=[MetaAttribute(attribute_type_id="biolink:a")], + qualifiers=[ + MetaQualifier( + qualifier_type_id="biolink:subject_aspect_qualifier", + applicable_values=["activity"], + ) + ], + ) + other = _meta_edge( + knowledge_types=["inferred"], + attributes=[MetaAttribute(attribute_type_id="biolink:b")], + qualifiers=[ + MetaQualifier( + qualifier_type_id="biolink:subject_aspect_qualifier", + applicable_values=["abundance"], + ) + ], + ) + edge_dict = edge.to_dict() + edge.update(other) + MetaEdgeDictUtil.update(edge_dict, other.to_dict()) + # knowledge_types / applicable_values merge via sets; compare set-wise. + assert set(edge_dict["knowledge_types"]) == set(edge.knowledge_types or []) + assert MetaEdgeDictUtil.hash(edge_dict) == edge.hash() + merged_values = { + q["qualifier_type_id"]: set(q.get("applicable_values") or []) + for q in edge_dict["qualifiers"] + } + model_values = { + q.qualifier_type_id: set(q.applicable_values or []) + for q in (edge.qualifiers or []) + } + assert merged_values == model_values + + def test_meets_attribute_constraints_parity(self): + edge = _meta_edge(attributes=[MetaAttribute(attribute_type_id="biolink:foo")]) + constraints = [ + AttributeConstraint(id="biolink:foo", name="Foo", operator="==", value=1) + ] + assert MetaEdgeDictUtil.meets_attribute_constraints( + edge.to_dict(), [c.to_dict() for c in constraints] + ) == edge.meets_attribute_constraints(constraints) + + def test_meets_qualifier_constraints_parity(self): + edge = _meta_edge( + qualifiers=[ + MetaQualifier( + qualifier_type_id="biolink:subject_aspect_qualifier", + applicable_values=["activity"], + ) + ] + ) + constraints = [ + QualifierConstraint( + qualifier_set=[ + Qualifier( + qualifier_type_id="biolink:subject_aspect_qualifier", + qualifier_value="activity", + ) + ] + ) + ] + assert MetaEdgeDictUtil.meets_qualifier_constraints( + edge.to_dict(), [c.to_dict() for c in constraints] + ) == edge.meets_qualifier_constraints(constraints) + + +# ============================================================================ +# MetaKnowledgeGraph +# ============================================================================ + + +class TestMetaKnowledgeGraph: + def test_new(self): + assert MetaKnowledgeGraphDictUtil.new() == MetaKnowledgeGraph.new().to_dict() + assert MetaKnowledgeGraphDictUtil.new() == {"nodes": {}, "edges": []} + + def test_hash_parity(self): + mkg = MetaKnowledgeGraph( + nodes={"biolink:Gene": MetaNode(id_prefixes=["NCBIGene"])}, + edges=[_meta_edge()], + ) + assert MetaKnowledgeGraphDictUtil.hash(mkg.to_dict()) == mkg.hash() + + +class TestMetaEdgeUpdateKlAtSkip: + def test_update_skips_knowledge_level_and_agent_type(self): + edge = _meta_edge(attributes=[MetaAttribute(attribute_type_id="biolink:a")]) + other = _meta_edge( + attributes=[ + MetaAttribute(attribute_type_id="biolink:knowledge_level"), + MetaAttribute(attribute_type_id="biolink:agent_type"), + MetaAttribute(attribute_type_id="biolink:b"), + ] + ) + edge_dict = edge.to_dict() + edge.update(other) + MetaEdgeDictUtil.update(edge_dict, other.to_dict()) + assert edge_dict == edge.to_dict() + types = {a["attribute_type_id"] for a in edge_dict.get("attributes", [])} + assert "biolink:knowledge_level" not in types + assert "biolink:agent_type" not in types diff --git a/tests/test_model_dicts/test_path_constraint_dicts.py b/tests/test_model_dicts/test_path_constraint_dicts.py new file mode 100644 index 0000000..df4a3f8 --- /dev/null +++ b/tests/test_model_dicts/test_path_constraint_dicts.py @@ -0,0 +1,54 @@ +"""Tests for the `*DictUtil` sibling class in `model_dicts/path_constraint.py`. + +`PathConstraintDictUtil` reimplements `PathConstraint`'s utility methods for its +`TypedDict` equivalent; tests assert parity with the Pydantic behaviour. +""" + +from __future__ import annotations + +from translator_tom.model_dicts.path_constraint import ( + PathConstraintDict, + PathConstraintDictUtil, +) +from translator_tom.models.path_constraint import PathConstraint + +# ============================================================================ +# PathConstraintDictUtil.intermediate_categories_list +# ============================================================================ + + +class TestIntermediateCategoriesList: + def test_missing_key_returns_empty(self): + assert PathConstraintDictUtil.intermediate_categories_list({}) == [] + + def test_explicit_none_returns_empty(self): + constraint: PathConstraintDict = {"intermediate_categories": None} + assert PathConstraintDictUtil.intermediate_categories_list(constraint) == [] + + def test_populated_returns_value(self): + constraint: PathConstraintDict = {"intermediate_categories": ["biolink:Gene"]} + assert PathConstraintDictUtil.intermediate_categories_list(constraint) == [ + "biolink:Gene" + ] + + def test_parity_with_model(self): + model = PathConstraint(intermediate_categories=["biolink:Gene"]) + assert ( + PathConstraintDictUtil.intermediate_categories_list(model.to_dict()) + == model.intermediate_categories_list + ) + + +# ============================================================================ +# PathConstraintDictUtil.hash — parity with PathConstraint.hash +# ============================================================================ + + +class TestHashParity: + def test_empty(self): + model = PathConstraint() + assert PathConstraintDictUtil.hash(model.to_dict()) == model.hash() + + def test_populated(self): + model = PathConstraint(intermediate_categories=["biolink:Gene", "biolink:Drug"]) + assert PathConstraintDictUtil.hash(model.to_dict()) == model.hash() diff --git a/tests/test_model_dicts/test_qualifier_dicts.py b/tests/test_model_dicts/test_qualifier_dicts.py new file mode 100644 index 0000000..156b2cf --- /dev/null +++ b/tests/test_model_dicts/test_qualifier_dicts.py @@ -0,0 +1,188 @@ +"""Tests for the `*DictUtil` sibling classes in `model_dicts/qualifier.py`. + +The util classes reimplement the utility methods of the `QualifierConstraint` +Pydantic model for its `TypedDict` equivalent. Tests assert parity by comparing +dict-util results against the model operating on the same data. +""" + +from __future__ import annotations + +import pytest + +from translator_tom.model_dicts.qualifier import QualifierConstraintDictUtil +from translator_tom.models.meta_qualifier import MetaQualifier +from translator_tom.models.qualifier import Qualifier, QualifierConstraint + + +def _q(type_id: str, value: str) -> Qualifier: + return Qualifier(qualifier_type_id=type_id, qualifier_value=value) + + +def _qc(*pairs: tuple[str, str]) -> QualifierConstraint: + return QualifierConstraint(qualifier_set=[_q(t, v) for t, v in pairs]) + + +# ============================================================================ +# QualifierConstraintDictUtil.new +# ============================================================================ + + +class TestNew: + def test_matches_model(self): + assert QualifierConstraintDictUtil.new() == QualifierConstraint.new().to_dict() + assert QualifierConstraintDictUtil.new() == {"qualifier_set": []} + + +# ============================================================================ +# QualifierConstraintDictUtil.hash — parity (base hash, recurses into Qualifier) +# ============================================================================ + + +class TestHashParity: + def test_empty_set(self): + qc = QualifierConstraint(qualifier_set=[]) + assert QualifierConstraintDictUtil.hash(qc.to_dict()) == qc.hash() + + def test_populated_set(self): + qc = _qc( + ("biolink:subject_aspect_qualifier", "activity"), + ("biolink:object_direction_qualifier", "increased"), + ) + assert QualifierConstraintDictUtil.hash(qc.to_dict()) == qc.hash() + + +# ============================================================================ +# QualifierConstraintDictUtil.met_by — parity with QualifierConstraint.met_by +# ============================================================================ + + +def _assert_met_by_parity( + qc: QualifierConstraint, + qualifiers: list[Qualifier] | list[MetaQualifier], +) -> None: + dicts = [q.to_dict() for q in qualifiers] + assert QualifierConstraintDictUtil.met_by(qc.to_dict(), dicts) == qc.met_by( + qualifiers + ) + + +class TestMetByQualifiers: + def test_empty_constraint_is_met(self): + _assert_met_by_parity(QualifierConstraint(qualifier_set=[]), []) + + def test_no_qualifiers_fails_nonempty_constraint(self): + _assert_met_by_parity(_qc(("biolink:subject_aspect_qualifier", "activity")), []) + + def test_matching_qualifier(self): + _assert_met_by_parity( + _qc(("biolink:subject_aspect_qualifier", "activity")), + [_q("biolink:subject_aspect_qualifier", "activity")], + ) + + def test_type_mismatch(self): + _assert_met_by_parity( + _qc(("biolink:subject_aspect_qualifier", "activity")), + [_q("biolink:object_aspect_qualifier", "activity")], + ) + + def test_value_mismatch(self): + _assert_met_by_parity( + _qc(("biolink:subject_aspect_qualifier", "activity")), + [_q("biolink:subject_aspect_qualifier", "abundance")], + ) + + +class TestMetByMetaQualifiers: + def test_applicable_values_match(self): + _assert_met_by_parity( + _qc(("biolink:subject_aspect_qualifier", "activity")), + [ + MetaQualifier( + qualifier_type_id="biolink:subject_aspect_qualifier", + applicable_values=["activity", "abundance"], + ) + ], + ) + + def test_applicable_values_none_is_wildcard(self): + _assert_met_by_parity( + _qc(("biolink:object_aspect_qualifier", "secretion")), + [ + MetaQualifier( + qualifier_type_id="biolink:object_aspect_qualifier", + applicable_values=None, + ) + ], + ) + + def test_applicable_values_empty_is_not_wildcard(self): + _assert_met_by_parity( + _qc(("biolink:object_aspect_qualifier", "secretion")), + [ + MetaQualifier( + qualifier_type_id="biolink:object_aspect_qualifier", + applicable_values=[], + ) + ], + ) + + +# ============================================================================ +# QualifierConstraintDictUtil.set_met_by — parity with the model +# ============================================================================ + + +class TestSetMetBy: + def _assert_parity( + self, + constraints: list[QualifierConstraint], + qualifiers: list[Qualifier], + ) -> None: + result = QualifierConstraintDictUtil.set_met_by( + [c.to_dict() for c in constraints], [q.to_dict() for q in qualifiers] + ) + assert result == QualifierConstraint.set_met_by(constraints, qualifiers) + + def test_empty_constraints_is_true(self): + self._assert_parity([], [_q("biolink:subject_aspect_qualifier", "activity")]) + + def test_constraints_but_no_qualifiers_is_false(self): + self._assert_parity( + [_qc(("biolink:subject_aspect_qualifier", "activity"))], [] + ) + + def test_any_constraint_met(self): + self._assert_parity( + [ + _qc(("biolink:subject_aspect_qualifier", "abundance")), + _qc(("biolink:subject_aspect_qualifier", "activity")), + ], + [_q("biolink:subject_aspect_qualifier", "activity")], + ) + + +# ============================================================================ +# QualifierConstraintDictUtil.get_inverse — parity with QualifierConstraint.get_inverse +# ============================================================================ + + +class TestGetInverse: + def _assert_parity(self, qc: QualifierConstraint) -> None: + assert ( + QualifierConstraintDictUtil.get_inverse(qc.to_dict()) + == qc.get_inverse().to_dict() + ) + + def test_subject_to_object(self): + self._assert_parity(_qc(("biolink:subject_aspect_qualifier", "activity"))) + + def test_object_to_subject(self): + self._assert_parity(_qc(("biolink:object_direction_qualifier", "increased"))) + + def test_qualified_predicate_value_inverted(self): + self._assert_parity(_qc(("biolink:qualified_predicate", "biolink:causes"))) + + def test_uninvertible_raises(self): + qc = _qc(("biolink:qualified_predicate", "biolink:has_count")) + with pytest.raises(ValueError, match="non-inversible predicate"): + QualifierConstraintDictUtil.get_inverse(qc.to_dict()) diff --git a/tests/test_model_dicts/test_query_graph_dicts.py b/tests/test_model_dicts/test_query_graph_dicts.py new file mode 100644 index 0000000..25a25d7 --- /dev/null +++ b/tests/test_model_dicts/test_query_graph_dicts.py @@ -0,0 +1,445 @@ +"""Tests for the `*DictUtil` sibling classes in `model_dicts/query_graph.py`. + +The util classes reimplement the utility methods of the `QNode`/`QEdge`/`QPath` +Pydantic models for their `TypedDict` equivalents. Most tests assert parity with +the Pydantic behaviour by comparing against `model.get_inverse().to_dict()`. +""" + +from __future__ import annotations + +from typing import Annotated, Literal + +import pytest +from pydantic import Field + +from translator_tom.model_dicts.attribute import AttributeConstraintDictUtil +from translator_tom.model_dicts.qualifier import QualifierConstraintDictUtil +from translator_tom.model_dicts.query_graph import ( + PathfinderQueryGraphDictUtil, + QEdgeDict, + QEdgeDictUtil, + QNodeDict, + QNodeDictUtil, + QPathDict, + QPathDictUtil, + QueryGraphDictUtil, +) +from translator_tom.models.attribute import AttributeConstraint +from translator_tom.models.path_constraint import PathConstraint +from translator_tom.models.qualifier import Qualifier, QualifierConstraint +from translator_tom.models.query_graph import ( + PathfinderQueryGraph, + QEdge, + QNode, + QPath, + QueryGraph, +) +from translator_tom.utils.dict_util_base import DictUtil +from translator_tom.utils.object_base import TOMBase + +# ============================================================================ +# QNodeDictUtil — list accessors +# ============================================================================ + + +class TestQNodeDictUtilListAccessors: + def test_missing_keys_return_empty(self): + qnode: QNodeDict = {} + assert QNodeDictUtil.ids_list(qnode) == [] + assert QNodeDictUtil.categories_list(qnode) == [] + assert QNodeDictUtil.member_ids_list(qnode) == [] + assert QNodeDictUtil.constraints_list(qnode) == [] + + def test_explicit_none_returns_empty(self): + qnode: QNodeDict = {"ids": None, "categories": None} + assert QNodeDictUtil.ids_list(qnode) == [] + assert QNodeDictUtil.categories_list(qnode) == [] + + def test_populated_returns_value(self): + qnode: QNodeDict = { + "ids": ["CHEBI:1"], + "categories": ["biolink:ChemicalEntity"], + } + assert QNodeDictUtil.ids_list(qnode) == ["CHEBI:1"] + assert QNodeDictUtil.categories_list(qnode) == ["biolink:ChemicalEntity"] + + +# ============================================================================ +# QEdgeDictUtil — list accessors +# ============================================================================ + + +class TestQEdgeDictUtilListAccessors: + def test_missing_keys_return_empty(self): + qedge: QEdgeDict = {"subject": "n0", "object": "n1"} + assert QEdgeDictUtil.predicates_list(qedge) == [] + assert QEdgeDictUtil.attribute_constraints_list(qedge) == [] + assert QEdgeDictUtil.qualifier_constraints_list(qedge) == [] + + def test_populated_returns_value(self): + qedge: QEdgeDict = { + "subject": "n0", + "object": "n1", + "predicates": ["biolink:treats"], + } + assert QEdgeDictUtil.predicates_list(qedge) == ["biolink:treats"] + + +# ============================================================================ +# QEdgeDictUtil.get_inverse — parity with QEdge.get_inverse +# ============================================================================ + + +def _assert_inverse_parity(edge: QEdge) -> None: + """The dict util inverse should match the model's serialized inverse exactly.""" + edge_dict = edge.to_dict() + assert QEdgeDictUtil.get_inverse(edge_dict) == edge.get_inverse().to_dict() + + +class TestQEdgeDictUtilGetInverse: + def test_swaps_subject_and_object(self): + qedge: QEdgeDict = {"subject": "n0", "object": "n1"} + inverted = QEdgeDictUtil.get_inverse(qedge) + assert inverted["subject"] == "n1" + assert inverted["object"] == "n0" + + def test_inverts_predicates(self): + qedge: QEdgeDict = { + "subject": "n0", + "object": "n1", + "predicates": ["biolink:treats"], + } + assert QEdgeDictUtil.get_inverse(qedge)["predicates"] == ["biolink:treated_by"] + + def test_no_predicates_key_omitted(self): + qedge: QEdgeDict = {"subject": "n0", "object": "n1"} + assert "predicates" not in QEdgeDictUtil.get_inverse(qedge) + + def test_raises_when_predicate_uninvertible(self): + qedge: QEdgeDict = { + "subject": "n0", + "object": "n1", + "predicates": ["biolink:has_count"], + } + with pytest.raises(ValueError, match="Cannot invert predicates"): + QEdgeDictUtil.get_inverse(qedge) + + def test_parity_predicates(self): + _assert_inverse_parity( + QEdge(subject="n0", object="n1", predicates=["biolink:treats"]) + ) + + def test_parity_knowledge_type(self): + _assert_inverse_parity( + QEdge( + subject="n0", + object="n1", + predicates=["biolink:treats"], + knowledge_type="inferred", + ) + ) + + def test_parity_direction_scoped_attribute_constraints(self): + _assert_inverse_parity( + QEdge( + subject="n0", + object="n1", + attribute_constraints=[ + AttributeConstraint( + id="biolink:original_subject", + name="original subject", + operator="==", + value="X", + ) + ], + ) + ) + + def test_parity_qualifier_constraints(self): + _assert_inverse_parity( + QEdge( + subject="n0", + object="n1", + qualifier_constraints=[ + QualifierConstraint( + qualifier_set=[ + Qualifier( + qualifier_type_id="biolink:subject_aspect_qualifier", + qualifier_value="activity", + ) + ] + ) + ], + ) + ) + + +# ============================================================================ +# QPathDictUtil — list accessors +# ============================================================================ + + +class TestQPathDictUtilListAccessors: + def test_missing_keys_return_empty(self): + qpath: QPathDict = {"subject": "n0", "object": "n1"} + assert QPathDictUtil.predicates_list(qpath) == [] + assert QPathDictUtil.constraints_list(qpath) == [] + + def test_populated_returns_value(self): + qpath: QPathDict = { + "subject": "n0", + "object": "n1", + "predicates": ["biolink:related_to"], + } + assert QPathDictUtil.predicates_list(qpath) == ["biolink:related_to"] + + +# ============================================================================ +# hash — parity with the model's hash(), including nested-model recursion +# ============================================================================ + + +class TestHashParity: + def test_qnode_empty(self): + node = QNode() + assert QNodeDictUtil.hash(node.to_dict()) == node.hash() + + def test_qnode_scalar_fields(self): + node = QNode(ids=["CHEBI:1"], categories=["biolink:ChemicalEntity"]) + assert QNodeDictUtil.hash(node.to_dict()) == node.hash() + + def test_qnode_with_nested_attribute_constraints(self): + node = QNode( + ids=["CHEBI:1"], + constraints=[ + AttributeConstraint( + id="biolink:x", name="x", operator="==", value=[1, 2] + ) + ], + ) + assert QNodeDictUtil.hash(node.to_dict()) == node.hash() + + def test_qedge_with_all_nested_constraints(self): + edge = QEdge( + subject="n0", + object="n1", + predicates=["biolink:treats"], + attribute_constraints=[ + AttributeConstraint(id="biolink:x", name="x", operator="==", value=1) + ], + qualifier_constraints=[ + QualifierConstraint( + qualifier_set=[ + Qualifier( + qualifier_type_id="biolink:subject_aspect_qualifier", + qualifier_value="activity", + ) + ] + ) + ], + ) + assert QEdgeDictUtil.hash(edge.to_dict()) == edge.hash() + + def test_qpath_with_nested_path_constraints(self): + path = QPath( + subject="n0", + object="n1", + constraints=[ + PathConstraint(intermediate_categories=["biolink:Gene"]), + ], + ) + assert QPathDictUtil.hash(path.to_dict()) == path.hash() + + def test_hash_ignores_extra_keys(self): + node = QNode(ids=["CHEBI:1"]) + with_extra = {**node.to_dict(), "unexpected": "ignored"} + assert QNodeDictUtil.hash(with_extra) == node.hash() + + +# ============================================================================ +# I/O — JSON / MessagePack round-trips and model interop +# ============================================================================ + + +class TestIO: + def test_json_roundtrip(self): + qedge: QEdgeDict = { + "subject": "n0", + "object": "n1", + "predicates": ["biolink:treats"], + } + assert QEdgeDictUtil.from_json(QEdgeDictUtil.to_json(qedge)) == qedge + + def test_to_json_as_str(self): + qedge: QEdgeDict = {"subject": "n0", "object": "n1"} + assert isinstance(QEdgeDictUtil.to_json(qedge, as_str=True), str) + assert isinstance(QEdgeDictUtil.to_json(qedge), bytes) + + def test_msgpack_roundtrip(self): + qedge: QEdgeDict = { + "subject": "n0", + "object": "n1", + "predicates": ["biolink:treats"], + } + assert QEdgeDictUtil.from_msgpack(QEdgeDictUtil.to_msgpack(qedge)) == qedge + + def test_from_model_json_interop(self): + edge = QEdge(subject="n0", object="n1", predicates=["biolink:treats"]) + assert QEdgeDictUtil.from_json(edge.to_json()) == edge.to_dict() + + +# ============================================================================ +# Nested-util derivation (auto-derived from the model's field types) +# ============================================================================ + + +class _Inner(TOMBase): + x: int = 0 + + +class _Outer(TOMBase): + inner: _Inner | None = None + + +class _OuterDictUtil(DictUtil[dict[str, object]]): + """A util whose model nests `_Inner`, which deliberately has no util.""" + + _model = _Outer + + +class TestNestedDerivation: + def test_derived_mapping(self): + # Each entry pairs the field's container kind with a util resolver. + qnode = QNodeDictUtil._nested_fields() + assert set(qnode) == {"constraints"} + assert qnode["constraints"].kind == "list" + assert qnode["constraints"].resolve({}) is AttributeConstraintDictUtil + qedge = QEdgeDictUtil._nested_fields() + assert set(qedge) == {"attribute_constraints", "qualifier_constraints"} + assert qedge["attribute_constraints"].resolve({}) is AttributeConstraintDictUtil + assert qedge["qualifier_constraints"].resolve({}) is QualifierConstraintDictUtil + + def test_scalar_only_model_has_empty_mapping(self): + # AttributeConstraintDictUtil overrides hash, but base derivation still + # yields nothing since AttributeConstraint has no nested-model fields. + assert AttributeConstraintDictUtil._nested_fields() == {} + + def test_missing_nested_util_raises_loudly(self): + with pytest.raises(LookupError, match="_InnerDictUtil"): + _OuterDictUtil.hash({"inner": {"x": 1}}) + + +# ============================================================================ +# Union nested fields (discriminator-aware resolution) +# ============================================================================ + +# A model with a structural (non-tagged) union field: `QueryGraph | PathfinderQueryGraph` +# mirrors `Message.query_graph`, resolved via the registered structural discriminator. + + +class _QGHolder(TOMBase): + query_graph: QueryGraph | PathfinderQueryGraph | None = None + + +class _QGHolderDictUtil(DictUtil[dict[str, object]]): + _model = _QGHolder + + +# A model with a pydantic tagged union field (discriminated by `kind`), mirroring the +# `Annotated[..., Field(discriminator=...)]` shape of TRAPI's `workflow`. + + +# The discriminator is required (no default), as in real TRAPI tagged unions +# (Operation.id) — so exclude_defaults never drops the key the resolver needs. +class _Cat(TOMBase): + kind: Literal["cat"] + meow: int = 1 + + +class _Dog(TOMBase): + kind: Literal["dog"] + woof: int = 2 + + +_Pet = Annotated[_Cat | _Dog, Field(discriminator="kind")] + + +class _PetHolder(TOMBase): + pets: list[_Pet] | None = None + + +class _CatDictUtil(DictUtil[dict[str, object]]): + _model = _Cat + + +class _DogDictUtil(DictUtil[dict[str, object]]): + _model = _Dog + + +class _PetHolderDictUtil(DictUtil[dict[str, object]]): + _model = _PetHolder + + +class TestUnionHashParity: + def test_query_graph_members_hash(self): + qg = QueryGraph( + nodes={"n0": QNode(ids=["CHEBI:1"])}, + edges={"e0": QEdge(subject="n0", object="n1", predicates=["biolink:treats"])}, + ) + assert QueryGraphDictUtil.hash(qg.to_dict()) == qg.hash() + pqg = PathfinderQueryGraph( + nodes={"n0": QNode(ids=["CHEBI:1"])}, + paths={"p0": QPath(subject="n0", object="n1")}, + ) + assert PathfinderQueryGraphDictUtil.hash(pqg.to_dict()) == pqg.hash() + + def test_structural_union_end_to_end(self): + # Uses the real registered `QueryGraph | PathfinderQueryGraph` discriminator. + for qg in ( + QueryGraph(nodes={"n0": QNode()}, edges={}), + PathfinderQueryGraph( + nodes={"n0": QNode()}, paths={"p0": QPath(subject="n0", object="n1")} + ), + ): + holder = _QGHolder(query_graph=qg) + assert _QGHolderDictUtil.hash(holder.to_dict()) == holder.hash() + + def test_tagged_union_end_to_end(self): + holder = _PetHolder(pets=[_Cat(kind="cat", meow=3), _Dog(kind="dog", woof=4)]) + assert _PetHolderDictUtil.hash(holder.to_dict()) == holder.hash() + + def test_tagged_union_resolver_picks_by_tag(self): + pets = _PetHolderDictUtil._nested_fields()["pets"] + assert pets.kind == "list" + assert pets.resolve({"kind": "cat"}) is _CatDictUtil + assert pets.resolve({"kind": "dog"}) is _DogDictUtil + + +class _UnregisteredA(TOMBase): + a: int = 0 + + +class _UnregisteredB(TOMBase): + b: int = 0 + + +class _UnregisteredHolder(TOMBase): + item: _UnregisteredA | _UnregisteredB | None = None + + +class _UnregisteredHolderDictUtil(DictUtil[dict[str, object]]): + _model = _UnregisteredHolder + + +class _UnregisteredADictUtil(DictUtil[dict[str, object]]): + _model = _UnregisteredA + + +class _UnregisteredBDictUtil(DictUtil[dict[str, object]]): + _model = _UnregisteredB + + +def test_unregistered_structural_union_raises_loudly(): + # Utils exist for both members, but no discriminator is registered for the union. + with pytest.raises(LookupError, match="no discriminator"): + _UnregisteredHolderDictUtil.hash({"item": {"a": 1}}) diff --git a/tests/test_model_dicts/test_query_response_dicts.py b/tests/test_model_dicts/test_query_response_dicts.py new file mode 100644 index 0000000..1aae456 --- /dev/null +++ b/tests/test_model_dicts/test_query_response_dicts.py @@ -0,0 +1,104 @@ +"""Parity tests for `QueryDictUtil` and `ResponseDictUtil`. + +The hash-parity tests here exercise the full base-hash recursion chain: the tagged +`workflow` Operation union, the structural runner/fill-parameter unions, the +`query_graph` union, and every nested model's hash override. +""" + +from __future__ import annotations + +from translator_tom.model_dicts.query import QueryDictUtil +from translator_tom.model_dicts.response import ResponseDictUtil +from translator_tom.models.analysis import Analysis +from translator_tom.models.edge_binding import EdgeBinding +from translator_tom.models.knowledge_graph import Edge, KnowledgeGraph, Node +from translator_tom.models.message import Message +from translator_tom.models.node_binding import NodeBinding +from translator_tom.models.query import Query +from translator_tom.models.query_graph import QEdge, QNode, QueryGraph +from translator_tom.models.response import Response +from translator_tom.models.result import Result +from translator_tom.models.retrieval_source import RetrievalSource +from translator_tom.models.workflow_operations import ( + DenyList, + FillAllowListParameters, + OperationFill, + OperationScore, +) + + +def _full_message() -> Message: + return Message( + query_graph=QueryGraph( + nodes={"n0": QNode(ids=["CHEBI:1"])}, + edges={ + "e0": QEdge(subject="n0", object="n1", predicates=["biolink:treats"]) + }, + ), + knowledge_graph=KnowledgeGraph( + nodes={"CHEBI:1": Node(categories=["biolink:ChemicalEntity"], attributes=[])}, + edges={ + "kg0": Edge( + predicate="biolink:treats", + subject="CHEBI:1", + object="MONDO:1", + sources=[ + RetrievalSource( + resource_id="infores:x", + resource_role="primary_knowledge_source", + ) + ], + ) + }, + ), + results=[ + Result( + node_bindings={"n0": [NodeBinding(id="CHEBI:1", attributes=[])]}, + analyses=[ + Analysis( + resource_id="infores:x", + edge_bindings={"e0": [EdgeBinding(id="kg0", attributes=[])]}, + ) + ], + ) + ], + ) + + +def _workflow() -> list: + return [ + OperationFill( + id="fill", + parameters=FillAllowListParameters( + allowlist=["infores:a"], qedge_keys=["e0"] + ), + ), + OperationScore(id="score", runner_parameters=DenyList(denylist=["infores:b"])), + ] + + +class TestQuery: + def test_workflow_list(self): + q = Query(message=Message(), workflow=_workflow()) + assert QueryDictUtil.workflow_list(q.to_dict()) == q.to_dict()["workflow"] + assert QueryDictUtil.workflow_list({"message": {}}) == [] + + def test_new(self): + assert QueryDictUtil.new() == Query.new().to_dict() + + def test_hash_parity_full_chain(self): + q = Query(message=_full_message(), workflow=_workflow()) + assert QueryDictUtil.hash(q.to_dict()) == q.hash() + + +class TestResponse: + def test_workflow_list(self): + r = Response(message=Message(), workflow=_workflow()) + assert ResponseDictUtil.workflow_list(r.to_dict()) == r.to_dict()["workflow"] + + def test_new(self): + assert ResponseDictUtil.new() == Response.new().to_dict() + + def test_hash_parity_full_chain(self): + r = Response(message=_full_message(), workflow=_workflow()) + assert ResponseDictUtil.hash(r.to_dict()) == r.hash() diff --git a/tests/test_model_dicts/test_result_dicts.py b/tests/test_model_dicts/test_result_dicts.py new file mode 100644 index 0000000..3083b77 --- /dev/null +++ b/tests/test_model_dicts/test_result_dicts.py @@ -0,0 +1,181 @@ +"""Parity tests for `ResultDictUtil`.""" + +from __future__ import annotations + +from typing import Any + +from translator_tom.model_dicts.analysis import ( + AnalysisDictUtil, + PathfinderAnalysisDictUtil, +) +from translator_tom.model_dicts.result import ResultDict, ResultDictUtil +from translator_tom.models.analysis import Analysis +from translator_tom.models.attribute import Attribute +from translator_tom.models.edge_binding import EdgeBinding +from translator_tom.models.node_binding import NodeBinding +from translator_tom.models.result import Result + + +def _eb(edge_id: str) -> EdgeBinding: + return EdgeBinding(id=edge_id, attributes=[]) + + +def _result(qnode: str, node_id: str, analyses: list[Analysis]) -> Result: + return Result( + node_bindings={qnode: [NodeBinding(id=node_id, attributes=[])]}, + analyses=analyses, + ) + + +def _dict_analysis_hash(analysis: dict[str, Any]) -> str: + if "path_bindings" in analysis: + return PathfinderAnalysisDictUtil.hash(analysis) # type: ignore[arg-type] + return AnalysisDictUtil.hash(analysis) # type: ignore[arg-type] + + +def _dict_analysis_hashes(result: ResultDict) -> list[str]: + return sorted(_dict_analysis_hash(a) for a in result["analyses"]) + + +def _model_analysis_hashes(result: Result) -> list[str]: + return sorted(a.hash() for a in result.analyses) + + +class TestHashParity: + def test_parity(self): + r = _result( + "n0", "CHEBI:1", [Analysis(resource_id="infores:x", edge_bindings={})] + ) + assert ResultDictUtil.hash(r.to_dict()) == r.hash() + + def test_hash_ignores_analyses(self): + # Result.hash keys only on node_bindings. + a = _result("n0", "CHEBI:1", [Analysis(resource_id="infores:x", edge_bindings={})]) + b = _result("n0", "CHEBI:1", []) + assert ResultDictUtil.hash(a.to_dict()) == ResultDictUtil.hash(b.to_dict()) + + +class TestNormalize: + def test_remaps_edge_binding_ids(self): + r = _result( + "n0", + "CHEBI:1", + [Analysis(resource_id="infores:x", edge_bindings={"e0": [_eb("old")]})], + ) + r_dict = r.to_dict() + mapping = {"old": "new"} + r.normalize(mapping) + ResultDictUtil.normalize(r_dict, mapping) + assert r_dict == r.to_dict() + + def test_normalize_list_parity(self): + results = [ + _result( + "n0", + "CHEBI:1", + [Analysis(resource_id="infores:x", edge_bindings={"e0": [_eb("old")]})], + ) + ] + dicts = [r.to_dict() for r in results] + mapping = {"old": "new"} + Result.normalize_list(results, mapping) + ResultDictUtil.normalize_list(dicts, mapping) + assert dicts == [r.to_dict() for r in results] + + +class TestUpdate: + def test_merges_analyses(self): + r = _result( + "n0", + "CHEBI:1", + [Analysis(resource_id="infores:x", edge_bindings={"e0": [_eb("kg0")]})], + ) + other = _result( + "n0", + "CHEBI:1", + [ + Analysis(resource_id="infores:x", edge_bindings={"e0": [_eb("kg1")]}), + Analysis(resource_id="infores:y", edge_bindings={}), + ], + ) + r_dict = r.to_dict() + r.update(other) + ResultDictUtil.update(r_dict, other.to_dict()) + assert _dict_analysis_hashes(r_dict) == _model_analysis_hashes(r) + + def test_empty_other_is_noop(self): + r = _result("n0", "CHEBI:1", [Analysis(resource_id="infores:x", edge_bindings={})]) + r_dict = r.to_dict() + ResultDictUtil.update(r_dict, {"node_bindings": {}, "analyses": []}) + assert _dict_analysis_hashes(r_dict) == _model_analysis_hashes(r) + + +class TestMergeResults: + def test_parity(self): + results = [ + _result("n0", "CHEBI:1", [Analysis(resource_id="infores:x", edge_bindings={})]), + _result("n0", "CHEBI:2", []), + ] + new = [ + # Same node bindings as results[0] -> merges analyses. + _result("n0", "CHEBI:1", [Analysis(resource_id="infores:y", edge_bindings={})]), + ] + dicts = [r.to_dict() for r in results] + new_dicts = [r.to_dict() for r in new] + Result.merge_results(results, new) + ResultDictUtil.merge_results(dicts, new_dicts) + assert [ResultDictUtil.hash(d) for d in dicts] == [r.hash() for r in results] + for d, r in zip(dicts, results, strict=True): + assert _dict_analysis_hashes(d) == _model_analysis_hashes(r) + + +class TestMergeAnalysesByResourceId: + def test_parity(self): + r = _result( + "n0", + "CHEBI:1", + [ + Analysis(resource_id="infores:x", edge_bindings={"e0": [_eb("kg0")]}), + Analysis(resource_id="infores:x", edge_bindings={"e1": [_eb("kg1")]}), + Analysis(resource_id="infores:y", edge_bindings={}), + ], + ) + r_dict = r.to_dict() + r.merge_analyses_by_resource_id() + ResultDictUtil.merge_analyses_by_resource_id(r_dict) + assert _dict_analysis_hashes(r_dict) == _model_analysis_hashes(r) + + +class TestUpdateExistingAnalysisMerge: + def test_same_hash_analysis_merged_not_appended(self): + # identical hash-relevant fields but different attributes (not in the analysis + # hash) -> the existing-analysis merge branch of update(), attributes combined + a = _result( + "n0", + "CHEBI:1", + [ + Analysis( + resource_id="infores:x", + edge_bindings={}, + attributes=[Attribute(attribute_type_id="biolink:a", value=1)], + ) + ], + ) + other = _result( + "n0", + "CHEBI:1", + [ + Analysis( + resource_id="infores:x", + edge_bindings={}, + attributes=[Attribute(attribute_type_id="biolink:b", value=2)], + ) + ], + ) + r_dict = a.to_dict() + a.update(other) + ResultDictUtil.update(r_dict, other.to_dict()) + assert len(r_dict["analyses"]) == 1 + assert len(a.analyses) == 1 + assert _dict_analysis_hashes(r_dict) == _model_analysis_hashes(a) + assert len(r_dict["analyses"][0].get("attributes", [])) == 2 diff --git a/tests/test_model_dicts/test_retrieval_source_dicts.py b/tests/test_model_dicts/test_retrieval_source_dicts.py new file mode 100644 index 0000000..bbea91c --- /dev/null +++ b/tests/test_model_dicts/test_retrieval_source_dicts.py @@ -0,0 +1,93 @@ +"""Tests for `RetrievalSourceDictUtil`, asserting parity with the model.""" + +from __future__ import annotations + +from translator_tom.model_dicts.retrieval_source import ( + RetrievalSourceDict, + RetrievalSourceDictUtil, +) +from translator_tom.models.retrieval_source import RetrievalSource + + +class TestListAccessors: + def test_missing_returns_empty(self): + source: RetrievalSourceDict = { + "resource_id": "infores:x", + "resource_role": "primary_knowledge_source", + } + assert RetrievalSourceDictUtil.upstream_resource_ids_list(source) == [] + assert RetrievalSourceDictUtil.source_record_urls_list(source) == [] + + def test_populated(self): + source: RetrievalSourceDict = { + "resource_id": "infores:x", + "resource_role": "aggregator_knowledge_source", + "upstream_resource_ids": ["infores:y"], + "source_record_urls": ["http://example.com"], + } + assert RetrievalSourceDictUtil.upstream_resource_ids_list(source) == [ + "infores:y" + ] + assert RetrievalSourceDictUtil.source_record_urls_list(source) == [ + "http://example.com" + ] + + +class TestHashParity: + def test_minimal(self): + s = RetrievalSource( + resource_id="infores:x", resource_role="primary_knowledge_source" + ) + assert RetrievalSourceDictUtil.hash(s.to_dict()) == s.hash() + + def test_hash_ignores_non_identity_fields(self): + # Only resource_id/resource_role feed the hash. + a = RetrievalSource( + resource_id="infores:x", + resource_role="primary_knowledge_source", + upstream_resource_ids=["infores:y"], + ) + b = RetrievalSource( + resource_id="infores:x", resource_role="primary_knowledge_source" + ) + assert RetrievalSourceDictUtil.hash(a.to_dict()) == RetrievalSourceDictUtil.hash( + b.to_dict() + ) + + +class TestUpdate: + def _assert_parity( + self, source: RetrievalSource, other: RetrievalSource + ) -> None: + source_dict = source.to_dict() + source.update(other) + RetrievalSourceDictUtil.update(source_dict, other.to_dict()) + # Merged upstream ids are built from a set, so compare order-independently. + assert set(source_dict.get("upstream_resource_ids", [])) == set( + source.upstream_resource_ids or [] + ) + + def test_merges_upstream_ids(self): + self._assert_parity( + RetrievalSource( + resource_id="infores:x", + resource_role="aggregator_knowledge_source", + upstream_resource_ids=["infores:a"], + ), + RetrievalSource( + resource_id="infores:x", + resource_role="aggregator_knowledge_source", + upstream_resource_ids=["infores:b"], + ), + ) + + def test_no_other_upstream_is_noop(self): + source: RetrievalSourceDict = { + "resource_id": "infores:x", + "resource_role": "primary_knowledge_source", + } + RetrievalSourceDictUtil.update( + source, + {"resource_id": "infores:x", "resource_role": "primary_knowledge_source"}, + ) + assert "upstream_resource_ids" not in source diff --git a/tests/test_model_dicts/test_workflow_operations_dicts.py b/tests/test_model_dicts/test_workflow_operations_dicts.py new file mode 100644 index 0000000..971c482 --- /dev/null +++ b/tests/test_model_dicts/test_workflow_operations_dicts.py @@ -0,0 +1,157 @@ +"""Parity tests for the workflow-operation `*DictUtil` classes.""" + +from __future__ import annotations + +from translator_tom.model_dicts.workflow_operations import ( + AnnotateEdgesParametersDict, + AnnotateEdgesParametersDictUtil, + EnrichResultsParametersDict, + EnrichResultsParametersDictUtil, + FillAllowListParametersDict, + FillAllowListParametersDictUtil, + FilterKgraphPercentileParametersDict, + FilterKgraphPercentileParametersDictUtil, + FilterKgraphStdDevParametersDictUtil, + OperationBindDictUtil, + OperationFillDictUtil, + OperationFilterKgraphPercentileDictUtil, + OperationFilterKgraphStdDevDictUtil, + OperationScoreDictUtil, + SortResultNodeAttributeParametersDict, + SortResultNodeAttributeParametersDictUtil, +) +from translator_tom.models.workflow_operations import ( + AllowList, + DenyList, + FillAllowListParameters, + FillDenyListParameters, + FilterKgraphPercentileParameters, + FilterKgraphStdDevParameters, + OperationBind, + OperationFill, + OperationFilterKgraphPercentile, + OperationFilterKgraphStdDev, + OperationScore, +) + + +class TestUnique: + def test_matches_model(self): + assert OperationBindDictUtil.unique() == OperationBind(id="bind").unique + assert OperationScoreDictUtil.unique() == OperationScore(id="score").unique + assert OperationFillDictUtil.unique() == OperationFill(id="fill").unique + + def test_values(self): + assert OperationBindDictUtil.unique() is False + assert OperationScoreDictUtil.unique() is True + + +class TestParameterListAccessors: + def test_annotate_edges_attributes(self): + params: AnnotateEdgesParametersDict = {"attributes": ["pmids"]} + assert AnnotateEdgesParametersDictUtil.attributes_list(params) == ["pmids"] + assert AnnotateEdgesParametersDictUtil.attributes_list({}) == [] + + def test_enrich_results_qnode_keys(self): + params: EnrichResultsParametersDict = {"qnode_keys": ["n0"]} + assert EnrichResultsParametersDictUtil.qnode_keys_list(params) == ["n0"] + assert EnrichResultsParametersDictUtil.qnode_keys_list({}) == [] + + def test_fill_allowlist_qedge_keys(self): + params: FillAllowListParametersDict = { + "allowlist": ["infores:x"], + "qedge_keys": ["e0"], + } + assert FillAllowListParametersDictUtil.qedge_keys_list(params) == ["e0"] + + def test_filter_kgraph_base_accessors_inherited(self): + # FilterKgraphPercentileParametersDictUtil inherits the base accessors. + params: FilterKgraphPercentileParametersDict = { + "qedge_keys": ["e0"], + "qnode_keys": ["n0"], + "edge_attribute": "x", + } + assert FilterKgraphPercentileParametersDictUtil.qedge_keys_list(params) == [ + "e0" + ] + assert FilterKgraphPercentileParametersDictUtil.qnode_keys_list(params) == [ + "n0" + ] + + def test_sort_node_attribute_qnode_keys(self): + params: SortResultNodeAttributeParametersDict = { + "node_attribute": "x", + "ascending_or_descending": "ascending", + "qnode_keys": None, + } + assert SortResultNodeAttributeParametersDictUtil.qnode_keys_list(params) == [] + + +class TestOperationHashParity: + def test_simple_operation(self): + op = OperationScore(id="score") + assert OperationScoreDictUtil.hash(op.to_dict()) == op.hash() + + def test_runner_parameters_allowlist_union(self): + op = OperationBind(id="bind", runner_parameters=AllowList(allowlist=["infores:a"])) + assert OperationBindDictUtil.hash(op.to_dict()) == op.hash() + + def test_runner_parameters_denylist_union(self): + op = OperationBind(id="bind", runner_parameters=DenyList(denylist=["infores:a"])) + assert OperationBindDictUtil.hash(op.to_dict()) == op.hash() + + def test_fill_allowlist_parameters_union(self): + op = OperationFill( + id="fill", + runner_parameters=AllowList(allowlist=["infores:a"]), + parameters=FillAllowListParameters( + allowlist=["infores:b"], qedge_keys=["e0"] + ), + ) + assert OperationFillDictUtil.hash(op.to_dict()) == op.hash() + + def test_fill_denylist_parameters_union(self): + op = OperationFill( + id="fill", + parameters=FillDenyListParameters(denylist=["infores:b"]), + ) + assert OperationFillDictUtil.hash(op.to_dict()) == op.hash() + + +class TestFilterKgraphParamHashParity: + """Regression: `threshold`/`num_sigma` are float fields; their defaults must be + floats so exclude_defaults drops-and-restores them without an int/float hash drift. + """ + + def test_percentile_threshold(self): + # default 95.0 (dropped by exclude_defaults) + an explicit non-default value + default = FilterKgraphPercentileParameters(qedge_keys=["e0"], edge_attribute="a") + explicit = FilterKgraphPercentileParameters( + qedge_keys=["e0"], edge_attribute="a", threshold=50.0 + ) + for m in (default, explicit): + assert FilterKgraphPercentileParametersDictUtil.hash(m.to_dict()) == m.hash() + + def test_stddev_num_sigma(self): + default = FilterKgraphStdDevParameters(qedge_keys=["e0"], edge_attribute="a") + explicit = FilterKgraphStdDevParameters( + qedge_keys=["e0"], edge_attribute="a", num_sigma=2.0 + ) + for m in (default, explicit): + assert FilterKgraphStdDevParametersDictUtil.hash(m.to_dict()) == m.hash() + + def test_operation_wrappers_at_default(self): + pct = OperationFilterKgraphPercentile( + id="filter_kgraph_percentile", + parameters=FilterKgraphPercentileParameters( + qedge_keys=["e0"], edge_attribute="a" + ), + ) + assert OperationFilterKgraphPercentileDictUtil.hash(pct.to_dict()) == pct.hash() + std = OperationFilterKgraphStdDev( + id="filter_kgraph_std_dev", + parameters=FilterKgraphStdDevParameters( + qedge_keys=["e0"], edge_attribute="a" + ), + ) + assert OperationFilterKgraphStdDevDictUtil.hash(std.to_dict()) == std.hash() diff --git a/uv.lock b/uv.lock index 955ed5e..dff411e 100644 --- a/uv.lock +++ b/uv.lock @@ -2275,7 +2275,7 @@ wheels = [ [[package]] name = "translator-tom" -version = "1.4.1" +version = "1.5.0" source = { editable = "." } dependencies = [ { name = "bmt" }, From 234ee7d78857fd5c8b413cc2c4f061ffcb01e90f Mon Sep 17 00:00:00 2001 From: tokebe <43009413+tokebe@users.noreply.github.com> Date: Fri, 31 Jul 2026 20:15:21 -0400 Subject: [PATCH 5/8] optional model_dict light validation --- src/translator_tom/utils/dict_util_base.py | 53 ++++++++- tests/test_utils/test_dict_util_base.py | 124 +++++++++++++++++++++ 2 files changed, 171 insertions(+), 6 deletions(-) create mode 100644 tests/test_utils/test_dict_util_base.py diff --git a/src/translator_tom/utils/dict_util_base.py b/src/translator_tom/utils/dict_util_base.py index fd55486..7aff3a2 100644 --- a/src/translator_tom/utils/dict_util_base.py +++ b/src/translator_tom/utils/dict_util_base.py @@ -25,6 +25,7 @@ import orjson import ormsgpack +from pydantic import TypeAdapter from pydantic.fields import FieldInfo from pydantic_core import PydanticUndefined @@ -131,6 +132,8 @@ class DictUtil(Generic[_TD]): _nested_fields_cache: ClassVar[dict[str, _NestedField] | None] = None # Per-subclass cache for `_field_defaults()`. _field_defaults_cache: ClassVar[dict[str, Any] | None] = None + # Per-subclass cache for `_adapter()` (built lazily on first validating parse). + _adapter_cache: ClassVar[TypeAdapter[Any] | None] = None def __init_subclass__(cls, **kwargs: Any) -> None: """Register each concrete `*DictUtil` under the model it mirrors.""" @@ -142,9 +145,39 @@ def __init_subclass__(cls, **kwargs: Any) -> None: ##### I/O methods ##### @classmethod - def from_json(cls, json: str | bytes) -> _TD: - """Deserialize a dict from JSON.""" - return cast("_TD", orjson.loads(json)) + def _adapter(cls) -> TypeAdapter[Any]: + """Return the cached `TypeAdapter` over this util's `TypedDict`, building it lazily.""" + cached = cls.__dict__.get("_adapter_cache") + if cached is not None: + return cached + typed_dict = next( + ( + arg + for base in getattr(cls, "__orig_bases__", ()) + for arg in get_args(base) + ), + None, + ) + if typed_dict is None: + raise TypeError( + f"{cls.__name__}: cannot resolve the TypedDict type parameter for validation." + ) + adapter: TypeAdapter[Any] = TypeAdapter(typed_dict) + cls._adapter_cache = adapter + return adapter + + @classmethod + def from_json(cls, json: str | bytes, validate: bool = False) -> _TD: + """Deserialize a dict from JSON. + + With `validate=True`, data is validated using pydantic TypeAdapter and raises + `pydantic.ValidationError` if it failes validation. The returned dict is not + modified by the validation (extra keys preserved, etc). + """ + data = orjson.loads(json) + if validate: + cls._adapter().validate_python(data) + return cast("_TD", data) @overload @classmethod @@ -170,9 +203,17 @@ def to_json(cls, obj: _TD, as_str: bool = False) -> str | bytes: return json @classmethod - def from_msgpack(cls, msgpack: bytes) -> _TD: - """Deserialize a dict from MessagePack.""" - return cast("_TD", ormsgpack.unpackb(msgpack)) + def from_msgpack(cls, msgpack: bytes, validate: bool = False) -> _TD: + """Deserialize a dict from MessagePack. + + With `validate=True`, data is validated using pydantic TypeAdapter and raises + `pydantic.ValidationError` if it failes validation. The returned dict is not + modified by the validation (extra keys preserved, etc). + """ + data = ormsgpack.unpackb(msgpack) + if validate: + cls._adapter().validate_python(data) + return cast("_TD", data) @classmethod def to_msgpack(cls, obj: _TD) -> bytes: diff --git a/tests/test_utils/test_dict_util_base.py b/tests/test_utils/test_dict_util_base.py new file mode 100644 index 0000000..029de34 --- /dev/null +++ b/tests/test_utils/test_dict_util_base.py @@ -0,0 +1,124 @@ +"""Tests for the I/O + validation methods on the `DictUtil` base. + +`from_json`/`from_msgpack` optionally validate the parsed dict against the util's +`TypedDict` — recursively, and without building model instances — via a cached +`TypeAdapter`. The *parsed* dict is returned unchanged, so extra keys and original +values survive even when validation runs. +""" + +from __future__ import annotations + +import orjson +import ormsgpack +import pytest +from pydantic import TypeAdapter, ValidationError + +from translator_tom.model_dicts.attribute import ( + AttributeConstraintDictUtil, + AttributeDict, + AttributeDictUtil, +) +from translator_tom.models.attribute import Attribute + +# ============================================================================ +# Default behavior — no validation unless asked +# ============================================================================ + + +class TestNoValidation: + def test_from_json_passes_garbage_through(self): + assert AttributeDictUtil.from_json(b'{"not_a_field": 5}') == {"not_a_field": 5} + + def test_from_msgpack_passes_garbage_through(self): + packed = ormsgpack.packb({"not_a_field": 5}) + assert AttributeDictUtil.from_msgpack(packed) == {"not_a_field": 5} + + +# ============================================================================ +# from_json(validate=True) +# ============================================================================ + + +class TestFromJsonValidate: + def test_valid_passes(self): + attr: AttributeDict = {"attribute_type_id": "biolink:foo", "value": 1} + assert ( + AttributeDictUtil.from_json(orjson.dumps(attr), validate=True) == attr + ) + + def test_returns_loaded_dict_preserving_extras(self): + # The validator drops keys not in the TypedDict; from_json must still return + # them, since it returns the parsed dict, not the validator's output. + attr = {"attribute_type_id": "biolink:foo", "value": 1, "surprise": "kept"} + out = AttributeDictUtil.from_json(orjson.dumps(attr), validate=True) + # Equality proves the unknown key survived; a dropped key would fail this. + assert out == attr + + def test_missing_required_field_raises(self): + with pytest.raises(ValidationError): + AttributeDictUtil.from_json(orjson.dumps({"value": 1}), validate=True) + + def test_wrong_scalar_type_raises(self): + # attribute_type_id is a CURIE (str); an int is rejected even in lax mode. + bad = {"attribute_type_id": 123, "value": 1} + with pytest.raises(ValidationError): + AttributeDictUtil.from_json(orjson.dumps(bad), validate=True) + + def test_recursive_nested_valid_passes(self): + good = { + "attribute_type_id": "biolink:foo", + "value": 1, + "attributes": [{"attribute_type_id": "biolink:bar", "value": 2}], + } + assert AttributeDictUtil.from_json(orjson.dumps(good), validate=True) == good + + def test_recursive_nested_invalid_raises(self): + # Outer object is fine; the nested sub-attribute is missing its required key. + bad = { + "attribute_type_id": "biolink:foo", + "value": 1, + "attributes": [{"value": 2}], + } + with pytest.raises(ValidationError): + AttributeDictUtil.from_json(orjson.dumps(bad), validate=True) + + def test_model_json_round_trip_parity(self): + # A real model's JSON validates and yields exactly the model's dict form. + a = Attribute( + attribute_type_id="biolink:foo", + value=[1, 2], + attributes=[Attribute(attribute_type_id="biolink:bar", value=3)], + ) + assert AttributeDictUtil.from_json(a.to_json(), validate=True) == a.to_dict() + + +# ============================================================================ +# from_msgpack(validate=True) — mirrors from_json +# ============================================================================ + + +class TestFromMsgpackValidate: + def test_valid_passes_preserving_extras(self): + attr = {"attribute_type_id": "biolink:foo", "value": 1, "surprise": "kept"} + out = AttributeDictUtil.from_msgpack(ormsgpack.packb(attr), validate=True) + assert out == attr + + def test_invalid_raises(self): + with pytest.raises(ValidationError): + AttributeDictUtil.from_msgpack(ormsgpack.packb({"value": 1}), validate=True) + + +# ============================================================================ +# TypeAdapter caching +# ============================================================================ + + +class TestAdapterCaching: + def test_same_instance_across_calls(self): + assert AttributeDictUtil._adapter() is AttributeDictUtil._adapter() + + def test_distinct_per_subclass(self): + assert AttributeDictUtil._adapter() is not AttributeConstraintDictUtil._adapter() + + def test_adapter_is_a_type_adapter(self): + assert isinstance(AttributeDictUtil._adapter(), TypeAdapter) From 6a2b2fa2af0b4d6fbf93e212534d53db739e00b1 Mon Sep 17 00:00:00 2001 From: tokebe <43009413+tokebe@users.noreply.github.com> Date: Fri, 31 Jul 2026 22:33:55 -0400 Subject: [PATCH 6/8] use FastJsonValue in model_dicts --- src/translator_tom/model_dicts/attribute.py | 7 +- .../model_dicts/workflow_operations.py | 103 +++++++++--------- 2 files changed, 56 insertions(+), 54 deletions(-) diff --git a/src/translator_tom/model_dicts/attribute.py b/src/translator_tom/model_dicts/attribute.py index cfd8b37..61a1e89 100644 --- a/src/translator_tom/model_dicts/attribute.py +++ b/src/translator_tom/model_dicts/attribute.py @@ -3,7 +3,6 @@ import re from typing import cast -from pydantic import JsonValue from typing_extensions import NotRequired, TypedDict from translator_tom.model_dicts.meta_attribute import ( @@ -17,7 +16,7 @@ AttributeConstraint, Operator, ) -from translator_tom.models.shared import CURIE +from translator_tom.models.shared import CURIE, FastJsonValue from translator_tom.utils.dict_util_base import DictUtil from translator_tom.utils.hash import tomhash @@ -32,7 +31,7 @@ class AttributeDict(TypedDict): attribute_type_id: CURIE original_attribute_name: NotRequired[str | None] - value: JsonValue + value: FastJsonValue value_type_id: NotRequired[CURIE | None] attribute_source: NotRequired[str | None] value_url: NotRequired[str | None] @@ -88,7 +87,7 @@ def merge_attribute_lists( "name": str, "not": NotRequired[bool], "operator": Operator, - "value": JsonValue, + "value": FastJsonValue, "unit_id": NotRequired[CURIE | None], "unit_name": NotRequired[str | None], }, diff --git a/src/translator_tom/model_dicts/workflow_operations.py b/src/translator_tom/model_dicts/workflow_operations.py index 1d301ad..3f5d3a9 100644 --- a/src/translator_tom/model_dicts/workflow_operations.py +++ b/src/translator_tom/model_dicts/workflow_operations.py @@ -1,12 +1,12 @@ from __future__ import annotations from collections.abc import Mapping -from typing import ClassVar, Literal +from typing import Annotated, ClassVar, Literal -from pydantic import JsonValue +from pydantic import Field from typing_extensions import NotRequired, TypedDict -from translator_tom.models.shared import Infores, QEdgeID, QNodeID +from translator_tom.models.shared import FastJsonValue, Infores, QEdgeID, QNodeID from translator_tom.models.workflow_operations import ( AboveOrBelow, AllowList, @@ -222,7 +222,7 @@ def unique(cls) -> bool: class OperationAnnotateDict(BaseOperationDict): id: Literal["annotate"] - parameters: NotRequired[dict[str, JsonValue] | None] + parameters: NotRequired[dict[str, FastJsonValue] | None] class OperationAnnotateDictUtil(BaseOperationDictUtil): @@ -290,7 +290,7 @@ class OperationAnnotateNodesDictUtil(BaseOperationDictUtil): class OperationBindDict(BaseOperationDict): id: Literal["bind"] - parameters: NotRequired[dict[str, JsonValue] | None] + parameters: NotRequired[dict[str, FastJsonValue] | None] class OperationBindDictUtil(BaseOperationDictUtil): @@ -301,7 +301,7 @@ class OperationBindDictUtil(BaseOperationDictUtil): class OperationCompleteResultsDict(BaseOperationDict): id: Literal["complete_results"] - parameters: NotRequired[dict[str, JsonValue] | None] + parameters: NotRequired[dict[str, FastJsonValue] | None] class OperationCompleteResultsDictUtil(BaseOperationDictUtil): @@ -387,7 +387,7 @@ class OperationFillDictUtil(BaseOperationDictUtil): class OperationFilterKgraphDict(BaseOperationDict): id: Literal["filter_kgraph"] - parameters: NotRequired[dict[str, JsonValue] | None] + parameters: NotRequired[dict[str, FastJsonValue] | None] class OperationFilterKgraphDictUtil(BaseOperationDictUtil): @@ -448,7 +448,7 @@ class OperationFilterKgraphContinuousKedgeAttributeDictUtil(BaseOperationDictUti class FilterKgraphDiscreteKedgeAttributeParametersDict(FilterKgraphParametersBaseDict): edge_attribute: str - remove_value: JsonValue + remove_value: FastJsonValue class FilterKgraphDiscreteKedgeAttributeParametersDictUtil( @@ -472,7 +472,7 @@ class OperationFilterKgraphDiscreteKedgeAttributeDictUtil(BaseOperationDictUtil) class FilterKgraphDiscreteKnodeAttributeParametersDict(FilterKgraphParametersBaseDict): node_attribute: str - remove_value: JsonValue + remove_value: FastJsonValue class FilterKgraphDiscreteKnodeAttributeParametersDictUtil( @@ -496,7 +496,7 @@ class OperationFilterKgraphDiscreteKnodeAttributeDictUtil(BaseOperationDictUtil) class OperationFilterKgraphOrphansDict(BaseOperationDict): id: Literal["filter_kgraph_orphans"] - parameters: NotRequired[dict[str, JsonValue] | None] + parameters: NotRequired[dict[str, FastJsonValue] | None] class OperationFilterKgraphOrphansDictUtil(BaseOperationDictUtil): @@ -577,7 +577,7 @@ class OperationFilterKgraphTopNDictUtil(BaseOperationDictUtil): class OperationFilterResultsDict(BaseOperationDict): id: Literal["filter_results"] - parameters: NotRequired[dict[str, JsonValue] | None] + parameters: NotRequired[dict[str, FastJsonValue] | None] class OperationFilterResultsDictUtil(BaseOperationDictUtil): @@ -609,7 +609,7 @@ class OperationFilterResultsTopNDictUtil(BaseOperationDictUtil): class OperationLookupDict(BaseOperationDict): id: Literal["lookup"] - parameters: NotRequired[dict[str, JsonValue] | None] + parameters: NotRequired[dict[str, FastJsonValue] | None] class OperationLookupDictUtil(BaseOperationDictUtil): @@ -621,7 +621,7 @@ class OperationLookupDictUtil(BaseOperationDictUtil): class OperationLookupAndScoreDict(BaseOperationDict): id: Literal["lookup_and_score"] - parameters: NotRequired[dict[str, JsonValue] | None] + parameters: NotRequired[dict[str, FastJsonValue] | None] class OperationLookupAndScoreDictUtil(BaseOperationDictUtil): @@ -633,7 +633,7 @@ class OperationLookupAndScoreDictUtil(BaseOperationDictUtil): class OperationOverlayDict(BaseOperationDict): id: Literal["overlay"] - parameters: NotRequired[dict[str, JsonValue] | None] + parameters: NotRequired[dict[str, FastJsonValue] | None] class OperationOverlayDictUtil(BaseOperationDictUtil): @@ -691,7 +691,7 @@ class OperationOverlayComputeNgdDictUtil(BaseOperationDictUtil): class OperationOverlayConnectKnodesDict(BaseOperationDict): id: Literal["overlay_connect_knodes"] - parameters: NotRequired[dict[str, JsonValue] | None] + parameters: NotRequired[dict[str, FastJsonValue] | None] class OperationOverlayConnectKnodesDictUtil(BaseOperationDictUtil): @@ -728,7 +728,7 @@ class OperationOverlayFisherExactTestDictUtil(BaseOperationDictUtil): class OperationRestateDict(BaseOperationDict): id: Literal["restate"] - parameters: NotRequired[dict[str, JsonValue] | None] + parameters: NotRequired[dict[str, FastJsonValue] | None] class OperationRestateDictUtil(BaseOperationDictUtil): @@ -740,7 +740,7 @@ class OperationRestateDictUtil(BaseOperationDictUtil): class OperationScoreDict(BaseOperationDict): id: Literal["score"] - parameters: NotRequired[dict[str, JsonValue] | None] + parameters: NotRequired[dict[str, FastJsonValue] | None] class OperationScoreDictUtil(BaseOperationDictUtil): @@ -752,7 +752,7 @@ class OperationScoreDictUtil(BaseOperationDictUtil): class OperationSortResultsDict(BaseOperationDict): id: Literal["sort_results"] - parameters: NotRequired[dict[str, JsonValue] | None] + parameters: NotRequired[dict[str, FastJsonValue] | None] class OperationSortResultsDictUtil(BaseOperationDictUtil): @@ -840,38 +840,41 @@ class OperationSortResultsScoreDictUtil(BaseOperationDictUtil): _model = OperationSortResultsScore -OperationDict = ( - OperationAnnotateDict - | OperationAnnotateEdgesDict - | OperationAnnotateNodesDict - | OperationBindDict - | OperationCompleteResultsDict - | OperationEnrichResultsDict - | OperationFillDict - | OperationFilterKgraphDict - | OperationFilterKgraphContinuousKedgeAttributeDict - | OperationFilterKgraphDiscreteKedgeAttributeDict - | OperationFilterKgraphDiscreteKnodeAttributeDict - | OperationFilterKgraphOrphansDict - | OperationFilterKgraphPercentileDict - | OperationFilterKgraphStdDevDict - | OperationFilterKgraphTopNDict - | OperationFilterResultsDict - | OperationFilterResultsTopNDict - | OperationLookupDict - | OperationLookupAndScoreDict - | OperationOverlayDict - | OperationOverlayComputeJaccardDict - | OperationOverlayComputeNgdDict - | OperationOverlayConnectKnodesDict - | OperationOverlayFisherExactTestDict - | OperationRestateDict - | OperationScoreDict - | OperationSortResultsDict - | OperationSortResultsEdgeAttributeDict - | OperationSortResultsNodeAttributeDict - | OperationSortResultsScoreDict -) +OperationDict = Annotated[ + ( + OperationAnnotateDict + | OperationAnnotateEdgesDict + | OperationAnnotateNodesDict + | OperationBindDict + | OperationCompleteResultsDict + | OperationEnrichResultsDict + | OperationFillDict + | OperationFilterKgraphDict + | OperationFilterKgraphContinuousKedgeAttributeDict + | OperationFilterKgraphDiscreteKedgeAttributeDict + | OperationFilterKgraphDiscreteKnodeAttributeDict + | OperationFilterKgraphOrphansDict + | OperationFilterKgraphPercentileDict + | OperationFilterKgraphStdDevDict + | OperationFilterKgraphTopNDict + | OperationFilterResultsDict + | OperationFilterResultsTopNDict + | OperationLookupDict + | OperationLookupAndScoreDict + | OperationOverlayDict + | OperationOverlayComputeJaccardDict + | OperationOverlayComputeNgdDict + | OperationOverlayConnectKnodesDict + | OperationOverlayFisherExactTestDict + | OperationRestateDict + | OperationScoreDict + | OperationSortResultsDict + | OperationSortResultsEdgeAttributeDict + | OperationSortResultsNodeAttributeDict + | OperationSortResultsScoreDict + ), + Field(discriminator="id"), +] def _discriminate_runner_parameters( From 357ed66fc5f8e2db28882d7e55abf4d41ed4d5c1 Mon Sep 17 00:00:00 2001 From: tokebe <43009413+tokebe@users.noreply.github.com> Date: Fri, 31 Jul 2026 22:34:23 -0400 Subject: [PATCH 7/8] add model_dict validation bench --- bench/test_sd_tom_dicts.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/bench/test_sd_tom_dicts.py b/bench/test_sd_tom_dicts.py index 407c9fa..00fbbac 100644 --- a/bench/test_sd_tom_dicts.py +++ b/bench/test_sd_tom_dicts.py @@ -4,13 +4,17 @@ but driving the `*DictUtil` serdes (raw orjson/ormsgpack over the TypedDict form, no model construction) instead of the `Response` model. Run both to see the cost the model layer adds over operating on plain dicts. + +The `+val` rows re-run the `from` path with `validate=True`, adding a pydantic +`TypeAdapter` pass over the parsed data; their `from` timing minus the plain +row's is the cost of opting into validation. """ import time from utils import CORPUS_ROOT, discover_files, read_corpus_file -LABEL_WIDTH = 10 +LABEL_WIDTH = 12 VALUE_FMT = "{:>8.4f}s" @@ -72,6 +76,12 @@ def section(title: str) -> None: t_to_json = time.perf_counter() - t0 pair_row("json", t_from_json, t_to_json, file_results) + # `to` is unaffected by validation; reuse its timing for the +val rows. + t0 = time.perf_counter() + _ = ResponseDictUtil.from_json(response_json, validate=True) + t_from_json_val = time.perf_counter() - t0 + pair_row("json+val", t_from_json_val, t_to_json, file_results) + t0 = time.perf_counter() response_msgpack = ResponseDictUtil.to_msgpack(response) t_to_mp = time.perf_counter() - t0 @@ -80,6 +90,11 @@ def section(title: str) -> None: t_from_mp = time.perf_counter() - t0 pair_row("msgpack", t_from_mp, t_to_mp, file_results) + t0 = time.perf_counter() + _ = ResponseDictUtil.from_msgpack(response_msgpack, validate=True) + t_from_mp_val = time.perf_counter() - t0 + pair_row("msgpack+val", t_from_mp_val, t_to_mp, file_results) + # --- Summary --- From dd3a7844e2dfa189436f72f53af172e5d29cb993 Mon Sep 17 00:00:00 2001 From: tokebe <43009413+tokebe@users.noreply.github.com> Date: Fri, 31 Jul 2026 22:58:22 -0400 Subject: [PATCH 8/8] fix + update README --- README.md | 168 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 160 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e685232..da3468d 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Models based on Pydantic provide deserialize with basic validation, serialize, a Allows for easy FastAPI standup. -## Usage +## Model Usage The main ways you interact with a Model are as follows: @@ -45,7 +45,11 @@ query_json = """ """ query = Query.from_json(query_json) -assert len(query.message.query_graph.nodes) == 2 # True + +# Access is now statically typed and editor provides hints + completions +query_graph = query.message.query_graph +assert query_graph is not None +assert len(query_graph.nodes) == 2 # True ``` Similarly, you can validate from JSON with a FastAPI endpoint: @@ -88,10 +92,8 @@ query_dict = { } query = Query.from_dict(query_dict) -assert len(query.message.query_graph.nodes) == 2 # True query = Query(**query_dict) # Also works (less clear, not recommended) -assert len(query.message.query_graph.nodes) == 2 # True ``` ### Construction @@ -122,7 +124,6 @@ query = Query( } }, ) -assert len(query.message.query_graph.nodes) == 2 # True ``` Another way is to use `Model.model_construct()`. @@ -135,7 +136,7 @@ from translator_tom import Biolink, Curie, Message, QEdge, QNode, Query, QueryGr # Using each type provides hints and type checking, making internal TRAPI construction # safer. -query = Query( +query = Query.model_construct( submitter="TOM tester", message=Message( query_graph=QueryGraph( @@ -155,7 +156,6 @@ query = Query( ) ), ) -assert len(query.message.query_graph.nodes) == 2 # True ``` ### Convenience Methods @@ -177,7 +177,159 @@ There are many more, it's recommended to look at the models themselves as they a More in-depth utility methods include `.normalize()` for Message/KnowledgeGraph/Result/AuxiliaryGraph, `.prune()` for KnowledgeGraph, etc. -### Semantic Validation (WIP) +## TypedDict Usage + +This library also provides `TypedDict` models, which can be used for internal static typing without class instantiation overhead, at the cost of some code verbosity. + +- `*DictUtil.from_json()` and `*DictUtil.to_json()` +- `*DictUtil.from_msgpack()` and `*DictUtil.to_msgpack()` +- Direct construction: `*Dict()` + +### JSON Reading + +Unlike with models, the model_dicts don't validate by default. + +```python +from translator_tom.model_dicts import QueryDictUtil, QNodeDictUtil + + +query_json = """ +{ + "submitter": "TOM tester", + "message": { + "query_graph": { + "nodes": { + "n0": { "ids": [ "PUBCHEM.COMPOUND:726218" ] }, + "n1": { "ids": [ "NCBIGene:3778" ] } + }, + "edges": { + "e0": { + "subject": "n0", + "object": "n1", + "predicates": [ "biolink:related_to" ] + } + } + } + } +} +""" + +query = QueryDictUtil.from_json(query_json) # returns type QueryDict + +# These key accessors now have hints+completions in type-aware editors +query_graph = query["message"]["query_graph"] +assert query_graph is not None # Type narrowing +assert len(query_graph["nodes"]) == 2 # True +n0_ids = query_graph["nodes"]["n0"].get("ids") or [] # `ids` is optional +assert n0_ids == ["PUBCHEM.COMPOUND:726218"] # True + +# DictUtils also provide safe accessors: +n0 = query_graph["nodes"]["n0"] +assert QNodeDictUtil.ids_list(n0) == ["PUBCHEM.COMPOUND:726218"] # True + + +# A 'lite' version of validation may be optionally used +# This doesn't mutate the parsed dict, but throws ValidationError if it fails. +# Significantly faster than model validation; but not as thorough +query = QueryDictUtil.from_json(query_json, validate=True) +``` + + +### Casting and direct instantiation + +Oftentimes you'll just want to cast a model_dict: + +```python +from typing import cast + +from translator_tom.model_dicts import QueryDict + +query_plain = { + "submitter": "TOM tester", + "message": { + "query_graph": { + "nodes": { + "n0": {"ids": ["PUBCHEM.COMPOUND:726218"]}, + "n1": {"ids": ["NCBIGene:3778"]}, + }, + "edges": { + "e0": { + "subject": "n0", + "object": "n1", + "predicates": ["biolink:related_to"], + } + }, + } + }, +} + +# cast is free at runtime; it only tells the type checker to treat query_plain as a QueryDict. +query = cast("QueryDict", query_plain) +``` + +You can also just pass an already-existing dict to the dict constructor, although it produces a shallow copy: + +```python +from translator_tom.model_dicts import QueryDict + +# An existing dict you've annotated as a QueryDict (checked against it here). +query_plain = { + "submitter": "TOM tester", + "message": { + "query_graph": { + "nodes": { + "n0": {"ids": ["PUBCHEM.COMPOUND:726218"]}, + "n1": {"ids": ["NCBIGene:3778"]}, + }, + "edges": { + "e0": { + "subject": "n0", + "object": "n1", + "predicates": ["biolink:related_to"], + } + }, + } + }, +} + +query = QueryDict(**query_plain) +``` + +### Direct construction + +You can also use the model_dicts directly as construction guides: + +```python +from translator_tom.model_dicts import ( + MessageDict, + QEdgeDict, + QNodeDict, + QueryDict, + QueryGraphDict, +) + +# Each constructor provides key hints and type checking +query = QueryDict( + submitter="TOM tester", + message=MessageDict( + query_graph=QueryGraphDict( + nodes={ + "n0": QNodeDict(ids=["PUBCHEM.COMPOUND:726218"]), + "n1": QNodeDict(ids=["NCBIGene:3778"]), + }, + edges={ + "e0": QEdgeDict( + subject="n0", + object="n1", + predicates=["biolink:related_to"], + ) + }, + ) + ), +) +``` + +## Semantic Validation (WIP) A very WIP item is Semantic Validation: