diff --git a/src/chexus/json.py b/src/chexus/json.py index b5a24b3..ba19ea8 100644 --- a/src/chexus/json.py +++ b/src/chexus/json.py @@ -1,12 +1,23 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import json +import sys from typing import Any import numpy as np from .tree import Dataset, Group +_MAX_STORED_DATASET_VALUE_SIZE = 100 * 1024 + + +class _DatasetWasTooBig: + def __repr__(self) -> str: + return "dataset_was_too_big" + + +dataset_was_too_big = _DatasetWasTooBig() + def read_json(path: str) -> Group: """ @@ -46,7 +57,7 @@ def read_json(path: str) -> Group: "config": { "name": "slits", "values": 1, - "type": "int64" + "dtype": "int64" } }, ] @@ -66,7 +77,7 @@ def _read_group(group: dict[str, Any], parent: Group | None = None) -> Group: if not isinstance(child, dict): continue module = child.get("module") - if module is None: + if module is None and "type" in child: if child["type"] == "group": grp.children[child["name"]] = _read_group(child, parent=grp) elif module == "dataset": @@ -81,18 +92,53 @@ def _read_group(group: dict[str, Any], parent: Group | None = None) -> Group: def _read_dataset(dataset: dict[str, Any], parent: Group) -> Dataset: """Read JSON dataset""" - name = parent.name + '/' + dataset['config']["name"] - if (values := dataset["config"].get("values")) is not None: - type_from_values = type(values) + config = dataset["config"] + name = parent.name + '/' + config["name"] + if "dtype" in config: + dtype = _translate_dtype(config["dtype"]) + elif "values" in config: + dtype = np.dtype(type(config["values"])) + else: + dtype = None + + kwargs = {} + if "values" in config: + if _json_value_fits(config["values"], _MAX_STORED_DATASET_VALUE_SIZE): + kwargs["value"] = config["values"] + else: + kwargs["value"] = dataset_was_too_big + return Dataset( name=name, shape=None, - dtype=_translate_dtype(dataset["config"].get("type", type_from_values)), + dtype=dtype, attrs=_read_attrs(dataset), parent=parent, + **kwargs, ) +def _json_value_fits(value: Any, limit: int) -> bool: + if (size := _fast_json_value_size(value, limit)) is not None: + return size < limit + return sys.getsizeof(json.dumps(value)) < limit + + +def _fast_json_value_size(value: Any, limit: int) -> int | None: + if value is None or isinstance(value, str | int | float | bool): + return sys.getsizeof(value) + if not isinstance(value, list): + return None + size = sys.getsizeof(value) + for item in value: + if size >= limit: + break + if (item_size := _fast_json_value_size(item, limit - size)) is None: + return None + size += item_size + return size + + def _read_source(source: dict[str, Any], parent: Group) -> Dataset: """Read JSON source""" name = parent.name + '/' + source['config']["source"] diff --git a/tests/json_test.py b/tests/json_test.py new file mode 100644 index 0000000..171ff2b --- /dev/null +++ b/tests/json_test.py @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Scipp contributors (https://github.com/scipp) +import json +import sys +from pathlib import Path +from typing import Any + +import numpy as np + +import chexus +import chexus.json as chexus_json + + +def _read_json(tmp_path: Path, content: dict[str, Any]) -> chexus.Group: + path = tmp_path / "template.json" + path.write_text(json.dumps(content), encoding="utf-8") + return chexus.read_json(str(path)) + + +def _tree_with_dataset(config: dict[str, Any]) -> dict[str, Any]: + return {"name": "", "children": [{"module": "dataset", "config": config}]} + + +def _dataset_from_config(tmp_path: Path, config: dict[str, Any]) -> chexus.Dataset: + root = _read_json(tmp_path, _tree_with_dataset(config)) + node = root.children[config["name"]] + assert isinstance(node, chexus.Dataset) + return node + + +def test_read_json_dataset_stores_values_and_reads_dtype(tmp_path: Path): + node = _dataset_from_config( + tmp_path, + { + "name": "depends_on", + "dtype": "string", + "values": "/entry/instrument/detector/transform", + }, + ) + + assert node.value == "/entry/instrument/detector/transform" + assert np.dtype(node.dtype) == np.dtype("U") + + +def test_read_json_dataset_ignores_config_type(tmp_path: Path): + node = _dataset_from_config( + tmp_path, {"name": "value", "type": "string", "values": 1} + ) + + assert node.value == 1 + assert np.dtype(node.dtype) == np.dtype(int) + + +def test_read_json_dataset_does_not_store_large_values(tmp_path: Path, monkeypatch): + value = [["larger"], [1, 2, 3]] + monkeypatch.setattr( + chexus_json, + "_MAX_STORED_DATASET_VALUE_SIZE", + sys.getsizeof(value) + 1, + ) + node = _dataset_from_config(tmp_path, {"name": "value", "values": value}) + + assert node.value is chexus_json.dataset_was_too_big + assert np.dtype(node.dtype) == np.dtype(object) + + +def test_read_json_dataset_without_dtype_or_values_leaves_dtype_unset( + tmp_path: Path, +): + node = _dataset_from_config(tmp_path, {"name": "value"}) + + assert node.dtype is None + assert node.value is None + + +def test_read_json_ignores_child_without_module_or_type(tmp_path: Path): + root = _read_json( + tmp_path, + { + "name": "", + "children": [ + { + "config": { + "name": "jaw_3_l", + "source": "/entry/parameters/jaw_3_l", + } + } + ], + }, + ) + + assert root.children == {} + + +def test_read_json_static_depends_on_value_validates_target(tmp_path: Path): + content = { + "name": "", + "children": [ + { + "name": "detector", + "type": "group", + "children": [ + { + "module": "dataset", + "config": { + "name": "depends_on", + "dtype": "string", + "values": "/detector/transform", + }, + }, + { + "module": "dataset", + "config": { + "name": "transform", + "dtype": "double", + "values": 0.0, + }, + "attributes": [ + {"name": "transformation_type", "values": "rotation"}, + {"name": "vector", "values": [0.0, 1.0, 0.0]}, + {"name": "depends_on", "values": "."}, + ], + }, + ], + } + ], + } + root = _read_json(tmp_path, content) + + results = chexus.validate( + root, validators=[chexus.validators.depends_on_target_missing()] + ) + + result = results[chexus.validators.depends_on_target_missing] + assert result.fails == 0