From 178c7dea02a10fa1aec7a08522317bb686261a49 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 14 Jul 2026 14:13:03 +0200 Subject: [PATCH 01/12] Add version negotiation --- src/simdb/cli/remote_api.py | 30 +++++++++++++++++++++------- src/simdb/remote/__init__.py | 8 +++----- tests/cli/test_remote_api_version.py | 18 +++++++++++++++++ 3 files changed, 44 insertions(+), 12 deletions(-) create mode 100644 tests/cli/test_remote_api_version.py diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index b8312060..0e0092ec 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -36,7 +36,7 @@ from simdb.database.models import Simulation from simdb.imas.utils import SimDBUrl, imas_files from simdb.json import CustomDecoder, CustomEncoder -from simdb.remote import APIConstants +from simdb.remote import CLIENT_API_VERSIONS, APIConstants from .manifest import DataType @@ -131,6 +131,19 @@ def _read_bytes_in_chunks( yield data +def select_api_version( + server_versions: Iterable[str], + client_versions: Iterable[str] = CLIENT_API_VERSIONS, +) -> Optional[str]: + """ + Select the highest API version supported by both the server and this client. + """ + common_versions = set(server_versions) & set(client_versions) + if not common_versions: + return None + return max(common_versions, key=lambda v: Version.coerce(v.lstrip("v"))) + + def check_return(res: "requests.Response") -> None: if res.status_code != 200: try: @@ -198,7 +211,6 @@ def __init__( f"Remote '{remote}' not found. Use `simdb remote config add` to add it." ) from None - self._api_url: str = f"{self._url}/v{config.api_version}/" self._firewall: Optional[str] = config.get_string_option( f"remote.{remote}.firewall", default=None ) @@ -247,14 +259,18 @@ def __init__( endpoints = self.get_endpoints() endpoint_versions = [endpoint.split("/")[-1] for endpoint in endpoints] - if not endpoint_versions: - raise RemoteError("No compatible API version found on remote") + selected_version = select_api_version(endpoint_versions) + if selected_version is None: + raise RemoteError( + "No compatible API version found on remote: the server provides " + f"{', '.join(endpoint_versions) or 'none'} and this client supports " + f"{', '.join(CLIENT_API_VERSIONS)}." + ) - latest_version = max(endpoint_versions) if config.verbose: - print(f"Selected latest endpoint version {latest_version}") + print(f"Selected API version {selected_version}") - self._api_url += f"{latest_version}/" + self._api_url += f"{selected_version}/" self.version = Version.coerce(self.get_api_version()) self.server_version = Version.coerce(self.get_server_version()) diff --git a/src/simdb/remote/__init__.py b/src/simdb/remote/__init__.py index 02eb4593..c16fa7e9 100644 --- a/src/simdb/remote/__init__.py +++ b/src/simdb/remote/__init__.py @@ -4,11 +4,9 @@ endpoint to which simulations can be sent for staging and signing-off. """ -from semantic_version import SimpleSpec - -# Compatibility scheme for the latest API version, i.e. anything with the same major and -# minor version -COMPATIBILITY_SPEC = SimpleSpec("~=1.2.0") +# API versions supported by this client, as they appear in the server endpoint URLs. +# Update this when a new API version is added to simdb.remote.apis. +CLIENT_API_VERSIONS = ("v1", "v1.1", "v1.2") # API constants diff --git a/tests/cli/test_remote_api_version.py b/tests/cli/test_remote_api_version.py new file mode 100644 index 00000000..08b0b44b --- /dev/null +++ b/tests/cli/test_remote_api_version.py @@ -0,0 +1,18 @@ +from simdb.cli.remote_api import select_api_version + + +def test_selects_highest_common_version(): + assert ( + select_api_version(["v1", "v1.1", "v1.2", "v1.3"], ("v1", "v1.1", "v1.2")) + == "v1.2" + ) + assert select_api_version(["v1", "v1.1"], ("v1", "v1.1", "v1.2")) == "v1.1" + + +def test_no_common_version_returns_none(): + assert select_api_version(["v2"], ("v1", "v1.1", "v1.2")) is None + assert select_api_version([], ("v1", "v1.1", "v1.2")) is None + + +def test_versions_compare_semantically_not_lexicographically(): + assert select_api_version(["v1.2", "v1.10"], ("v1.2", "v1.10")) == "v1.10" From 08157736a841c5a68858dcd29ca5235461b7aef3 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 10:55:40 +0200 Subject: [PATCH 02/12] feat: local simulation push CLI command, netcdf support, and validation tests --- .github/dependabot.yml | 5 - .../b2c52ee8ff12_add_ingestion_status.py | 2 +- docs/Makefile | 8 +- docs/sphinx/conf.py | 42 +---- pyproject.toml | 3 +- src/simdb/cli/commands/simulation.py | 79 +++++++- src/simdb/cli/manifest.py | 93 ++++----- src/simdb/cli/remote_api.py | 178 +++++++++++++++++- src/simdb/imas/utils.py | 3 - src/simdb/remote/apis/files.py | 6 +- src/simdb/remote/apis/v1/simulations.py | 2 +- src/simdb/remote/apis/v1_1/simulations.py | 2 +- src/simdb/remote/apis/v1_2/simulations.py | 4 +- src/simdb/remote/models.py | 6 + src/simdb/validation/file/ids_validator.py | 2 +- src/simdb/validation/validator.py | 52 +++-- src/simdb/workers/tasks.py | 6 +- tests/validation/test_validator.py | 56 ++++++ tests/workers/test_tasks.py | 4 +- uv.lock | 140 +++----------- 20 files changed, 453 insertions(+), 240 deletions(-) create mode 100644 tests/validation/test_validator.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 001b22a5..d52cfe9a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,9 +7,4 @@ updates: # Check for updates once a week schedule: interval: "weekly" - # Group actions version bumps into a single PR - groups: - actions-deps: - patterns: - - "*" diff --git a/alembic/versions/b2c52ee8ff12_add_ingestion_status.py b/alembic/versions/b2c52ee8ff12_add_ingestion_status.py index b9861c90..05ebcd98 100644 --- a/alembic/versions/b2c52ee8ff12_add_ingestion_status.py +++ b/alembic/versions/b2c52ee8ff12_add_ingestion_status.py @@ -1,7 +1,7 @@ """Add ingestion status Revision ID: b2c52ee8ff12 -Revises: 28bee3aa2429 +Revises: 9e9a4a7cd639 Create Date: 2026-05-11 16:16:03.768893 """ diff --git a/docs/Makefile b/docs/Makefile index 44d7ed10..6d45923c 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -11,13 +11,7 @@ BUILDDIR = _build help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -.PHONY: help html Makefile - -# Copy source files from docs/ into sphinx/ before building (mirrors .readthedocs.yml) -html: Makefile - cp ../*.md $(SOURCEDIR)/ 2>/dev/null || true - cp ../*.svg $(SOURCEDIR)/ 2>/dev/null || true - @$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) +.PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py index bb2b19f8..f1cc47d4 100644 --- a/docs/sphinx/conf.py +++ b/docs/sphinx/conf.py @@ -48,8 +48,8 @@ "sphinx.ext.mathjax", "sphinx.ext.viewcode", "myst_parser", - "sphinx_immaterial", # Sphinx immaterial theme ] + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -73,7 +73,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = 'en' +language = None # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -90,46 +90,16 @@ # a list of builtin themes. # # html_theme = 'sphinx_rtd_theme' -html_theme = "sphinx_immaterial" +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # +# html_theme_options = {} html_theme_options = { - "palette": [ - { - "media": "(prefers-color-scheme: light)", - "scheme": "default", - "primary": "blue", - "accent": "light-blue", - "toggle": { - "icon": "material/lightbulb-outline", - "name": "Switch to dark mode", - }, - }, - { - "media": "(prefers-color-scheme: dark)", - "scheme": "slate", - "primary": "blue", - "accent": "light-blue", - "toggle": { - "icon": "material/lightbulb", - "name": "Switch to light mode", - }, - }, - ], - "features": [ - "navigation.expand", - "navigation.tabs", - "navigation.sections", - "navigation.top", - "search.share", - "toc.follow", - "toc.sticky", - ], - "repo_url": "https://github.com/iterorganization/SimDB", - "repo_name": "SimDB", + "page_width": "auto", + "body_max_width": "auto", } # Add any paths that contain custom static files (such as style sheets) here, diff --git a/pyproject.toml b/pyproject.toml index 5f070698..c1dbebd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dependencies = [ "sqlalchemy>=1.2.12,<2.0", "alembic~=1.13", "rich>=14.3.3", + "netcdf4>=1.7.2", ] [project.optional-dependencies] @@ -81,7 +82,7 @@ imas-validator = [ ] build-docs = [ "sphinx>=4.5", - "sphinx-immaterial>=0.11.14", + "sphinx-rtd-theme>=1.0.0", "sphinx-autodoc-typehints>=1.12.0", "myst-parser>=0.18.0", "nbsphinx>=0.8.0", diff --git a/src/simdb/cli/commands/simulation.py b/src/simdb/cli/commands/simulation.py index 5b8045a9..1fe7a5f4 100644 --- a/src/simdb/cli/commands/simulation.py +++ b/src/simdb/cli/commands/simulation.py @@ -1,5 +1,6 @@ import contextlib import sys +import time import urllib.parse from itertools import chain from pathlib import Path @@ -172,11 +173,7 @@ def simulation_info(config: Config, sim_id: str): def simulation_ingest(config: Config, manifest_file: str, alias: str): """Ingest a MANIFEST_FILE.""" - overrides = {} - if alias: - overrides["alias"] = alias - - manifest = Manifest.load_from_file(Path(manifest_file), overrides=overrides) + manifest = Manifest.load_from_file(Path(manifest_file)) simulation = Simulation(manifest, config) if alias: @@ -207,6 +204,78 @@ def parse_args(self, ctx, args): return NRequiredArgs +@simulation.command("push_local", cls=n_required_args_adaptor(1)) +@pass_config +@click.argument("remote", required=False) +@click.argument("sim_id") +@click.option("--username", help="Username used to authenticate with the remote.") +@click.option("--password", help="Password used to authenticate with the remote.") +@click.option("--replaces", help="SIM_ID of simulation to deprecate and replace.") +@click.option( + "--add-watcher", + is_flag=True, + help="Add the current user as a watcher of the simulation.", +) +def simulation_push_local( + config: Config, + remote: Optional[str], + sim_id: str, + username: Optional[str], + password: Optional[str], + replaces: Optional[str], + add_watcher: bool, +): + """Push the simulation with the given SIM_ID (UUID or alias) to the REMOTE.""" + + api = RemoteAPI(remote, username, password, config) + db = get_local_db(config) + + simulation = db.get_simulation(sim_id) + if simulation is None: + raise click.ClickException(f"Failed to find simulation: {sim_id}") + + if replaces: + simulation.set_meta("replaces", replaces) + + schemas = api.get_validation_schemas() + try: + for schema in schemas: + Validator(schema).validate(simulation) + except ValidationError as err: + raise click.ClickException(f"Simulation does not validate: {err}") from err + + api.push_local_simulation(simulation) + + click.echo("Waiting for ingestion to complete...", nl=False) + last_status = None + while True: + try: + status = api.get_ingestion_status(simulation.uuid.hex) + except Exception as err: + click.echo() + raise click.ClickException( + f"Failed to check ingestion status: {err}" + ) from err + + if status != last_status: + if last_status is not None: + click.echo(f" -> {status}", nl=False) + else: + click.echo(f" {status}", nl=False) + last_status = status + + if status in ("completed", "copy_failed", "validation_failed"): + break + + time.sleep(1) + + click.echo() + if status == "completed": + click.echo(f"Successfully pushed simulation {simulation.uuid}") + else: + raise click.ClickException(f"Simulation ingestion failed with status: {status}") + + @simulation.command("push", cls=n_required_args_adaptor(1)) @pass_config @click.argument("remote", required=False) diff --git a/src/simdb/cli/manifest.py b/src/simdb/cli/manifest.py index a64c03e2..f7a41a83 100644 --- a/src/simdb/cli/manifest.py +++ b/src/simdb/cli/manifest.py @@ -55,8 +55,6 @@ def _get_data_object_type(uri: SimDBUrl) -> "DataType": return DataType.IMAS return DataType.FILE - raise ValueError(f"URI scheme ({uri.scheme}:) not recognized") - class DataObject(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) @@ -97,6 +95,7 @@ def validate_uri(cls, v: ManifestUrl, info): elif v.scheme == "file": v = v.build( scheme="file", + host="", path=_expand_path(Path(v.path), base_path).as_posix(), ) @@ -165,7 +164,7 @@ def to_v2_data(self) -> Dict[str, Any]: class Manifest(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) - manifest_version: Literal[2] = Field(default=2) + manifest_version: Literal[2] = Field(default=2, alias="version") alias: Optional[str] = None responsible_name: Optional[str] = None inputs_raw: List[Source] = Field(default_factory=list, alias="inputs") @@ -249,46 +248,57 @@ def resolve_metadata(self, info) -> "Manifest": self._metadata.update(metadata_item) return self - def _resolve_manifest_items(self, items, factory_cls, skip_glob_check): - resolved = [] - for item in items: - if item.type == DataType.FILE and item.uri.path: - path_obj = Path(item.uri.path) - - matches = list(path_obj.parent.glob(path_obj.name)) - - if not matches and skip_glob_check: - matches = [path_obj] - - if not matches: - raise ValueError(f"No files found matching path {path_obj}") - - for p in matches: - resolved.append( - factory_cls( - uri=SimDBUrl.build(scheme="file", path=p.as_posix()) - ) - ) - else: - resolved.append(item) - return resolved - @model_validator(mode="after") def resolve_inputs_and_outputs(self, info) -> "Manifest": context = info.context or {} skip_glob_check = context.get("skip_glob_check", False) + base_path = context.get("base_path") + if not base_path: + context["base_path"] = ( + self._path.absolute().parent if self._path != Path() else Path.cwd() + ) - context.setdefault( - "base_path", - self._path.absolute().parent if self._path != Path() else Path.cwd(), - ) - - self._inputs = self._resolve_manifest_items( - self.inputs_raw, Source, skip_glob_check - ) - self._outputs = self._resolve_manifest_items( - self.outputs_raw, Sink, skip_glob_check - ) + inputs = [] + for i in self.inputs_raw: + if i.type == DataType.FILE: + if i.uri.path: + source_path = Path(i.uri.path) + if not skip_glob_check: + names = [ + p.as_posix() + for p in source_path.parent.glob(source_path.name) + ] + if not names: + raise ValueError( + f"No files found matching path {source_path}" + ) + else: + names = [source_path.as_posix()] + for name in names: + inputs.append( + Source(uri=SimDBUrl.build(scheme="file", path=name)) + ) + else: + inputs.append(i) + self._inputs = inputs + + outputs = [] + for i in self.outputs_raw: + if i.type == DataType.FILE: + if i.uri.path: + sink_path = Path(i.uri.path) + names = [ + p.as_posix() for p in sink_path.parent.glob(sink_path.name) + ] + if not names and skip_glob_check: + names = [sink_path.as_posix()] + for name in names: + outputs.append( + Sink(uri=SimDBUrl.build(scheme="file", path=name)) + ) + else: + outputs.append(i) + self._outputs = outputs return self @@ -318,18 +328,13 @@ def from_template(cls) -> "Manifest": return model @classmethod - def load_from_file( - cls, file_path: Path, overrides: Optional[dict] = None - ) -> "Manifest": + def load_from_file(cls, file_path: Path) -> "Manifest": with file_path.open() as file: try: raw_data = yaml.load(file, Loader=cls._get_loader()) except yaml.YAMLError as err: raise ValueError("badly formatted manifest") from err - if overrides: - raw_data.update(overrides) - model = cls.model_validate( raw_data, context={"base_path": file_path.absolute().parent} ) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 0e0092ec..845ac17f 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -23,12 +23,14 @@ Optional, Tuple, Union, + cast, ) -from urllib.parse import urlparse +from urllib.parse import ParseResult, urlparse import appdirs import click import requests +from netCDF4 import Dataset from requests.auth import AuthBase from semantic_version import Version @@ -37,6 +39,8 @@ from simdb.imas.utils import SimDBUrl, imas_files from simdb.json import CustomDecoder, CustomEncoder from simdb.remote import CLIENT_API_VERSIONS, APIConstants +from simdb.remote.models import FileData, SimulationPostData +from simdb.workers.tasks import _calculate_checksum from .manifest import DataType @@ -165,6 +169,104 @@ def _get_paths(file: "File") -> Iterable[Path]: return imas_files(file.uri) +def _check_file_is_imas(file: Path) -> bool: + # Check NetCDF + if file.suffix == ".nc": + with Dataset(file, "r") as ds: + if getattr(ds, "Conventions", None) == "IMAS": + return True + + children = set(file.parent.iterdir()) + + # ASCII heuristic + if any(child.suffix == ".ids" for child in children): + return True + + # HDF5 heuristic + if any(child.suffix == ".h5" for child in children) and any( + child.name == "master.h5" for child in children + ): + return True + + # MDSplus heuristic + if {p.name for p in children} >= { # noqa: SIM103 + "ids_001.tree", + "ids_001.characteristics", + "ids_001.datafile", + }: + return True + + # No IMAS data detected + return False + + +def _find_partition_for_file(file: Path, partitions: dict[str, str]): + for partition, path in partitions.items(): + try: + return partition, file.relative_to(Path(path)) + except ValueError: + pass + return "file", file + + +def _expand_directories(files: Iterable[FileData], partitions: dict[str, str]): + new_file_list = [] + for file in files: + file_uri = SimDBUrl(file.uri) + if file_uri.path is None: + raise ValueError("File has no associated path") + file_path = Path(file_uri.path) + if file_uri.scheme == "imas": + qs = dict(file_uri.query_params()) + path = qs.get("path") + if path is None: + raise ValueError("IMAS uri has not path set") + file_path = Path(path) + + if file_path.is_dir(): + for sub_file in file_path.iterdir(): + if sub_file.is_dir(): + raise ValueError("Nested directory found") + partition, sub_file_path = _find_partition_for_file( + sub_file, partitions + ) + new_uri = SimDBUrl.build( + scheme=partition, path=sub_file_path.as_posix(), host="" + ) + new_file_list.append( + FileData( + type=file.type, + uri=new_uri.encoded_string(), + checksum=_calculate_checksum(sub_file), + datetime=file.datetime, + usage=file.usage, + purpose=file.purpose, + sensitivity=file.sensitivity, + access=file.access, + embargo=file.embargo, + ) + ) + else: + partition, new_file_path = _find_partition_for_file(file_path, partitions) + new_uri = SimDBUrl.build( + scheme=partition, path=new_file_path.as_posix(), host="" + ) + new_file_list.append( + FileData( + type=file.type, + uri=new_uri.encoded_string(), + checksum=_calculate_checksum(file_path), + datetime=file.datetime, + usage=file.usage, + purpose=file.purpose, + sensitivity=file.sensitivity, + access=file.access, + embargo=file.embargo, + ) + ) + return new_file_list + + class RemoteAPI: """ Class to represent connection to remote API. @@ -281,7 +383,7 @@ def _load_cookies( headers = {"User-Agent": "it_script_basic"} cookies_file = f"{remote}-cookies.pkl" cookies_path = Path(appdirs.user_config_dir("simdb")) / cookies_file - parsed_url = urlparse(self._url) + parsed_url: ParseResult = urlparse(self._url) base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" cookies = None @@ -755,6 +857,72 @@ def _send_chunk( ] self.post("files", data={}, files=files) + @try_request + def push_local_simulation(self, simulation: Simulation): + sim_data = simulation.to_model(recurse=True) + + partitions = cast(dict[str, str], self._config.get_section("partition")) + sim_data.inputs.root = _expand_directories(sim_data.inputs.root, partitions) + sim_data.outputs.root = _expand_directories(sim_data.outputs.root, partitions) + + for file in sim_data.inputs.root: + file_uri = SimDBUrl(file.uri) + if file_uri.path is None: + raise ValueError("File has no associated path") + file_path = Path(file_uri.path) + + partition = Path( + self._config.get_string_option(f"partition.{file_uri.scheme}") + ) + if _check_file_is_imas(partition / file_path): + file.type = "IMAS" + + for file in sim_data.outputs.root: + file_uri = SimDBUrl(url=file.uri) + if file_uri.path is None: + raise ValueError("File has no associated path") + file_path = Path(file_uri.path) + + partition = Path( + self._config.get_string_option(f"partition.{file_uri.scheme}") + ) + if _check_file_is_imas(partition / file_path): + file.type = "IMAS" + + uploaded_by = str(simulation.meta_dict().get("uploaded_by", None)) + + headers = {"Content-type": "application/json", "User-Agent": "it_script_basic"} + post_data = SimulationPostData( + simulation=sim_data, add_watcher=False, uploaded_by=uploaded_by + ).model_dump_json() + res = requests.post( + f"{self._url}/v1.3/simulations", + data=post_data, + headers=headers, + auth=self._get_auth(), + cookies=self._cookies, + ) + check_return(res) + + @try_request + def get_ingestion_status(self, sim_id: str) -> str: + headers = {"User-Agent": "it_script_basic"} + if self._server_auth != "None": + res = requests.get( + f"{self._url}/v1.3/simulation/status/{sim_id}", + headers=headers, + auth=self._get_auth(), + cookies=self._cookies, + ) + else: + res = requests.get( + f"{self._url}/v1.3/simulation/status/{sim_id}", + headers=headers, + cookies=self._cookies, + ) + check_return(res) + return res.json()["status"] + @try_request def push_simulation( self, @@ -777,7 +945,7 @@ def push_simulation( sim_data = simulation.data(recurse=True) try: - sim_json = json.dumps( + sim_json: bytes = json.dumps( sim_data, cls=CustomEncoder, separators=(",", ":") ).encode("utf-8") sim_json_size = len(sim_json) @@ -1020,7 +1188,7 @@ def pull_simulation( rel_path = directory / path.relative_to(common_root) self._pull_file(file.uuid, 0, checksum, path, rel_path, out_stream) file.uri = SimDBUrl.build( - scheme="file", path=rel_path.absolute().as_posix() + scheme="file", host="", path=rel_path.absolute().as_posix() ) elif file.type == DataType.IMAS: for index, (path, checksum) in enumerate(info): @@ -1035,7 +1203,7 @@ def pull_simulation( ).absolute() backend = qs.get("backend") file.uri = SimDBUrl.build( - scheme="imas", path=backend, query=f"path={to_path}" + scheme="imas", host="", path=backend, query=f"path={to_path}" ) return simulation diff --git a/src/simdb/imas/utils.py b/src/simdb/imas/utils.py index 7b9ff234..6596495a 100644 --- a/src/simdb/imas/utils.py +++ b/src/simdb/imas/utils.py @@ -213,9 +213,6 @@ def open_imas(uri: SimDBUrl) -> DBEntry: @return: the IMAS data entry object """ - if not _is_al5(): - return _open_legacy(uri) - if uri.path is None: raise ValueError(f"invalid imas URI: {uri} - no path found in URI") diff --git a/src/simdb/remote/apis/files.py b/src/simdb/remote/apis/files.py index cd7cad5e..a02b2601 100644 --- a/src/simdb/remote/apis/files.py +++ b/src/simdb/remote/apis/files.py @@ -49,7 +49,9 @@ def _verify_file( path = secure_path(Path(sim_file.uri.path), common_root, staging_dir) if not path.exists(): raise ValueError(f"file {path} does not exist") - checksum = sha1_checksum(SimDBUrl.build(scheme="file", path=path.as_posix())) + checksum = sha1_checksum( + SimDBUrl.build(scheme="file", host="", path=path.as_posix()) + ) if sim_file.checksum != checksum: raise ValueError(f"checksum failed for file {sim_file!r}") elif sim_file.type == DataType.IMAS: @@ -66,7 +68,7 @@ def _verify_file( else: path_value = str(staging_dir) new_uri = uri.build( - scheme=uri.scheme, path=uri.path, query=f"path={path_value}" + scheme=uri.scheme, host="", path=uri.path, query=f"path={path_value}" ) checksum = imas_checksum(new_uri, ids_list or []) if sim_file.checksum != checksum: diff --git a/src/simdb/remote/apis/v1/simulations.py b/src/simdb/remote/apis/v1/simulations.py index 3a56c9cb..eef6cb48 100644 --- a/src/simdb/remote/apis/v1/simulations.py +++ b/src/simdb/remote/apis/v1/simulations.py @@ -206,7 +206,7 @@ def post(self, user: User): if not path.exists(): raise ValueError(f"simulation file {sim_file.uuid} not uploaded") if sim_file.uri.scheme.name == "file": - sim_file.uri = SimDBUrl.build(scheme="file", path=path) + sim_file.uri = SimDBUrl.build(scheme="file", path=path, host="") result = { "ingested": simulation.uuid.hex, diff --git a/src/simdb/remote/apis/v1_1/simulations.py b/src/simdb/remote/apis/v1_1/simulations.py index 03c956b9..f0b89b71 100644 --- a/src/simdb/remote/apis/v1_1/simulations.py +++ b/src/simdb/remote/apis/v1_1/simulations.py @@ -235,7 +235,7 @@ def post(self, user: User): if not path.exists(): raise ValueError(f"simulation file {sim_file.uuid} not uploaded") if sim_file.uri.scheme.name == "file": - sim_file.uri = SimDBUrl.build(scheme="file", path=path) + sim_file.uri = SimDBUrl.build(scheme="file", path=path, host="") result = { "ingested": simulation.uuid.hex, diff --git a/src/simdb/remote/apis/v1_2/simulations.py b/src/simdb/remote/apis/v1_2/simulations.py index 96681628..8a7e1f12 100644 --- a/src/simdb/remote/apis/v1_2/simulations.py +++ b/src/simdb/remote/apis/v1_2/simulations.py @@ -272,7 +272,9 @@ def post( raise ResponseException( f"simulation file {sim_file.uuid} not uploaded" ) - sim_file.uri = SimDBUrl.build(scheme="file", path=path.as_posix()) + sim_file.uri = SimDBUrl.build( + scheme="file", host="", path=path.as_posix() + ) elif sim_file.uri.scheme == "imas": qs = dict(sim_file.uri.query_params()) if copy_files: diff --git a/src/simdb/remote/models.py b/src/simdb/remote/models.py index de8bded0..53c34679 100644 --- a/src/simdb/remote/models.py +++ b/src/simdb/remote/models.py @@ -344,6 +344,12 @@ class SimulationPostResponse(BaseModel): """Validation result.""" +class SimulationPostResponse3(BaseModel): + """Response from creating a simulation.""" + + job_id: HexUUID + + class SimulationListItem(BaseModel): """Summary of a simulation for list views.""" diff --git a/src/simdb/validation/file/ids_validator.py b/src/simdb/validation/file/ids_validator.py index 07452a41..6c1e1dea 100644 --- a/src/simdb/validation/file/ids_validator.py +++ b/src/simdb/validation/file/ids_validator.py @@ -87,7 +87,7 @@ def validate_uri(self, uri: SimDBUrl, validate_options): backend = qs.get("backend") path = qs.get("path") validate_uri = SimDBUrl.build( - scheme="imas", path=backend, query=f"path={path}" + scheme="imas", host="", path=backend, query=f"path={path}" ) validate_output = validate( diff --git a/src/simdb/validation/validator.py b/src/simdb/validation/validator.py index f9a3bb2b..d317d90b 100644 --- a/src/simdb/validation/validator.py +++ b/src/simdb/validation/validator.py @@ -28,7 +28,7 @@ class ValidationError(Exception): class CustomValidator(ValidatorBase): types_mapping = cast(Any, cerberus.Validator).types_mapping.copy() - types_mapping["numpy"] = cerberus.TypeDefinition("numpy", (np.ndarray,), ()) + types_mapping["numpy"] = cerberus.TypeDefinition("numpy", (np.ndarray, dict), ()) def _validate_exists(self, check_exists, field, value): """The rule's arguments are validated against this schema: @@ -42,26 +42,42 @@ def _validate_min_value(self, min_value, field, value): {'type': 'float'} """ - if not isinstance(value, np.ndarray): - value = value[~np.isnan(value)] + if isinstance(value, dict) and "min" in value and "max" in value: + if min_value is not None and value["min"] < min_value: + self._error(field, f"Minimum {value['min']} less than {min_value}") + elif isinstance(value, np.ndarray): + try: + if np.issubdtype(value.dtype, np.number): + value = value[~np.isnan(value)] + except TypeError: + pass if value.size == 0: self._error(field, "Values in numpy array are NaN or empty") + elif min_value is not None and value.min() < min_value: + self._error(field, f"Minimum {value.min()} less than {min_value}") + else: self._error(field, "Value is not a numpy array") - if min_value is not None and value.min() < min_value: - self._error(field, f"Minimum {value.min()} less than {min_value}") def _validate_max_value(self, max_value, field, value): """The rule's arguments are validated against this schema: {'type': 'float'} """ - if not isinstance(value, np.ndarray): - value = value[~np.isnan(value)] + if isinstance(value, dict) and "min" in value and "max" in value: + if max_value is not None and value["max"] > max_value: + self._error(field, f"Maximum {value['max']} greater than {max_value}") + elif isinstance(value, np.ndarray): + try: + if np.issubdtype(value.dtype, np.number): + value = value[~np.isnan(value)] + except TypeError: + pass if value.size == 0: self._error(field, "Values in numpy array are NaN or empty") + elif max_value is not None and value.max() > max_value: + self._error(field, f"Maximum {value.max()} greater than {max_value}") + else: self._error(field, "Value is not a numpy array") - if max_value is not None and value.max() > max_value: - self._error(field, f"Maximum {value.max()} greater than {max_value}") def _compare(self, comparison, field, value, comparator: str, message: str): if comparison is None: @@ -71,13 +87,25 @@ def _compare(self, comparison, field, value, comparator: str, message: str): value = value[~np.isnan(value)] if value.size == 0: self._error(field, "Values in numpy array are NaN or empty") + return if not getattr(value, comparator)(comparison).all(): self._error(field, f"Values are not {message} {comparison}") - elif isinstance(value, float): + elif isinstance(value, (float, int)): if not getattr(value, comparator)(comparison): self._error(field, f"Value is not {message} {comparison}") + elif isinstance(value, dict) and "min" in value and "max" in value: + if comparator in ("__gt__", "__ge__"): + val_to_compare = value["min"] + elif comparator in ("__lt__", "__le__"): + val_to_compare = value["max"] + else: + self._error(field, f"Unsupported comparison for range: {comparator}") + return + + if not getattr(val_to_compare, comparator)(comparison): + self._error(field, f"Value is not {message} {comparison}") else: - self._error(field, "Value is not a numpy array or a float") + self._error(field, "Value is not a numpy array, range, or a float") def _validate_gt(self, comparison, field, value): """The rule's arguments are validated against this schema: @@ -113,7 +141,7 @@ def _normalize_coerce_float(cls, value): @classmethod def _normalize_coerce_numpy(cls, value): - if isinstance(value, np.ndarray): + if isinstance(value, (np.ndarray, dict)): return value elif isinstance(value, dict) and "min" in value and "max" in value: return np.array([value["min"], value["max"]], dtype=float) diff --git a/src/simdb/workers/tasks.py b/src/simdb/workers/tasks.py index 1a65af4a..d08d9cf5 100644 --- a/src/simdb/workers/tasks.py +++ b/src/simdb/workers/tasks.py @@ -44,7 +44,7 @@ def _imas_path_to_uri(imas_path: Path) -> SimDBUrl: if imas_path.suffix == ".nc": return SimDBUrl.build(scheme="file", path=imas_path.as_posix()) - children = set(imas_path.iterdir()) + children = list(imas_path.iterdir()) if any(child.suffix == ".ids" for child in children): u = SimDBUrl.build( @@ -60,7 +60,7 @@ def _imas_path_to_uri(imas_path: Path) -> SimDBUrl: ) return u - if {p.name for p in children} >= { + if {p.name for p in children} == { "ids_001.tree", "ids_001.characteristics", "ids_001.datafile", @@ -73,7 +73,7 @@ def _imas_path_to_uri(imas_path: Path) -> SimDBUrl: raise ValueError("IMAS backend could not be identified.") -def _resolve_uri_to_path(uri: AnyUrl, config: Config) -> Path: +def _resolve_uri_to_path(uri: SimDBUrl, config: Config) -> Path: partition = uri.scheme if not partition: raise ValueError("Partition not given") diff --git a/tests/validation/test_validator.py b/tests/validation/test_validator.py new file mode 100644 index 00000000..bcef6a40 --- /dev/null +++ b/tests/validation/test_validator.py @@ -0,0 +1,56 @@ +import numpy as np + +from simdb.validation.validator import CustomValidator + + +def test_custom_validator_min_value_max_value(): + schema = { + "field1": { + "type": "numpy", + "coerce": "numpy", + "min_value": 0.0, + "max_value": 10.0, + } + } + validator = CustomValidator(schema) + + # Test valid numpy array + assert validator.validate({"field1": np.array([1.0, 5.0, 9.0])}) + + # Test valid dictionary representing a range + assert validator.validate({"field1": {"min": 1.0, "max": 9.0}}) + + # Test numpy array out of bounds (too low) + assert not validator.validate({"field1": np.array([-1.0, 5.0, 9.0])}) + + # Test numpy array out of bounds (too high) + assert not validator.validate({"field1": np.array([1.0, 5.0, 11.0])}) + + # Test dictionary range out of bounds (min too low) + assert not validator.validate({"field1": {"min": -1.0, "max": 9.0}}) + + # Test dictionary range out of bounds (max too high) + assert not validator.validate({"field1": {"min": 1.0, "max": 11.0}}) + + +def test_custom_validator_comparisons(): + schema = { + "field_ge": {"type": "numpy", "coerce": "numpy", "ge": 0.0}, + "field_le": {"type": "numpy", "coerce": "numpy", "le": 10.0}, + } + validator = CustomValidator(schema) + + # Test valid dictionary representing a range + assert validator.validate( + {"field_ge": {"min": 0.0, "max": 5.0}, "field_le": {"min": 1.0, "max": 10.0}} + ) + + # Test invalid range for ge (min is -1, which is not >= 0) + assert not validator.validate( + {"field_ge": {"min": -1.0, "max": 5.0}, "field_le": {"min": 1.0, "max": 10.0}} + ) + + # Test invalid range for le (max is 11, which is not <= 10) + assert not validator.validate( + {"field_ge": {"min": 0.0, "max": 5.0}, "field_le": {"min": 1.0, "max": 11.0}} + ) diff --git a/tests/workers/test_tasks.py b/tests/workers/test_tasks.py index 69afb4ce..629210e2 100644 --- a/tests/workers/test_tasks.py +++ b/tests/workers/test_tasks.py @@ -46,8 +46,8 @@ def test_get_imas_identifier_path_returns_parent_for_directory(tmp_path): @pytest.mark.parametrize( "files,expected_backend", [ - (["summary.ids"], "ascii"), - (["master.h5", "summary.h5"], "hdf5"), + (["child.ids"], "ascii"), + (["master.h5", "file2.h5"], "hdf5"), (["ids_001.tree", "ids_001.characteristics", "ids_001.datafile"], "mdsplus"), ], ) diff --git a/uv.lock b/uv.lock index b80dab5a..891ac4c6 100644 --- a/uv.lock +++ b/uv.lock @@ -2941,9 +2941,7 @@ build-docs = [ { name = "sphinx-autodoc-typehints", version = "3.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, { name = "sphinx-autodoc-typehints", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "sphinx-autodoc-typehints", version = "3.12.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "sphinx-immaterial", version = "0.11.14", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "sphinx-immaterial", version = "0.12.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, - { name = "sphinx-immaterial", version = "0.13.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "sphinx-rtd-theme" }, ] celery = [ { name = "celery", version = "5.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, @@ -3014,6 +3012,7 @@ requires-dist = [ { name = "myst-parser", marker = "extra == 'build-docs'", specifier = ">=0.18.0" }, { name = "nbsphinx", marker = "extra == 'build-docs'", specifier = ">=0.8.0" }, { name = "netcdf4", specifier = ">=1.5" }, + { name = "netcdf4", specifier = ">=1.7.2" }, { name = "numpy", specifier = ">=1.14" }, { name = "psycopg2-binary", marker = "extra == 'postgres'", specifier = ">=2.8.0" }, { name = "pydantic", specifier = ">=2.10.6" }, @@ -3031,7 +3030,7 @@ requires-dist = [ { name = "simplejson", marker = "extra == 'server'", specifier = "~=3.0" }, { name = "sphinx", marker = "extra == 'build-docs'", specifier = ">=4.5" }, { name = "sphinx-autodoc-typehints", marker = "extra == 'build-docs'", specifier = ">=1.12.0" }, - { name = "sphinx-immaterial", marker = "extra == 'build-docs'", specifier = ">=0.11.14" }, + { name = "sphinx-rtd-theme", marker = "extra == 'build-docs'", specifier = ">=1.0.0" }, { name = "sqlalchemy", specifier = ">=1.2.12,<2.0" }, ] provides-extras = ["server", "auth-ad", "auth-keycloak", "auth-ldap", "auth", "imas-validator", "build-docs", "postgres", "celery", "all"] @@ -5096,48 +5095,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, ] -[[package]] -name = "pydantic-extra-types" -version = "2.10.6" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9' and platform_python_implementation != 'PyPy'", - "python_full_version < '3.9' and platform_python_implementation == 'PyPy'", -] -dependencies = [ - { name = "pydantic", version = "2.10.6", source = { registry = "https://pypi.org/simple" } }, - { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" } }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3a/10/fb64987804cde41bcc39d9cd757cd5f2bb5d97b389d81aa70238b14b8a7e/pydantic_extra_types-2.10.6.tar.gz", hash = "sha256:c63d70bf684366e6bbe1f4ee3957952ebe6973d41e7802aea0b770d06b116aeb", size = 141858, upload-time = "2025-10-08T13:47:49.483Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/93/04/5c918669096da8d1c9ec7bb716bd72e755526103a61bc5e76a3e4fb23b53/pydantic_extra_types-2.10.6-py3-none-any.whl", hash = "sha256:6106c448316d30abf721b5b9fecc65e983ef2614399a24142d689c7546cc246a", size = 40949, upload-time = "2025-10-08T13:47:48.268Z" }, -] - -[[package]] -name = "pydantic-extra-types" -version = "2.11.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.13' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "python_full_version == '3.12.*' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "python_full_version == '3.10.*' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "(python_full_version >= '3.13' and platform_machine != 'ARM64') or (python_full_version >= '3.13' and sys_platform != 'win32')", - "(python_full_version == '3.12.*' and platform_machine != 'ARM64') or (python_full_version == '3.12.*' and sys_platform != 'win32')", - "(python_full_version == '3.11.*' and platform_machine != 'ARM64') or (python_full_version == '3.11.*' and sys_platform != 'win32')", - "(python_full_version == '3.10.*' and platform_machine != 'ARM64') or (python_full_version == '3.10.*' and sys_platform != 'win32')", - "python_full_version > '3.9' and python_full_version < '3.10'", - "python_full_version == '3.9'", -] -dependencies = [ - { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" } }, - { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" } }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/71/dba38ee2651f84f7842206adbd2233d8bbdb59fb85e9fa14232486a8c471/pydantic_extra_types-2.11.1.tar.gz", hash = "sha256:46792d2307383859e923d8fcefa82108b1a141f8a9c0198982b3832ab5ef1049", size = 172002, upload-time = "2026-03-16T08:08:03.92Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl", hash = "sha256:1722ea2bddae5628ace25f2aa685b69978ef533123e5638cfbddb999e0100ec1", size = 79526, upload-time = "2026-03-16T08:08:02.533Z" }, -] - [[package]] name = "pygments" version = "2.19.2" @@ -7185,76 +7142,23 @@ wheels = [ ] [[package]] -name = "sphinx-immaterial" -version = "0.11.14" +name = "sphinx-rtd-theme" +version = "3.1.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9' and platform_python_implementation != 'PyPy'", - "python_full_version < '3.9' and platform_python_implementation == 'PyPy'", -] dependencies = [ - { name = "appdirs" }, - { name = "markupsafe", version = "2.1.5", source = { registry = "https://pypi.org/simple" } }, - { name = "pydantic", version = "2.10.6", source = { registry = "https://pypi.org/simple" } }, - { name = "pydantic-extra-types", version = "2.10.6", source = { registry = "https://pypi.org/simple" } }, - { name = "requests", version = "2.32.4", source = { registry = "https://pypi.org/simple" } }, - { name = "sphinx", version = "7.1.2", source = { registry = "https://pypi.org/simple" } }, - { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" } }, -] -sdist = { url = "https://files.pythonhosted.org/packages/41/1f/5403cb6bd08f2f13c86d9b5367e56078dbe17d11c0bd91becdf34d454976/sphinx_immaterial-0.11.14.tar.gz", hash = "sha256:e1e8ba93c78a3e007743fede01a3be43f5ae97c5cc19b8e2a4d2aa058abead61", size = 8330984, upload-time = "2024-07-03T20:09:35.091Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/fa/db6916f719970ebdd433c5606bea98bbf899d71f255efced93992f944f1a/sphinx_immaterial-0.11.14-py3-none-any.whl", hash = "sha256:dd1a30614c8ecaa931155189e7d54f211232e31cf3e5c6d28ba9f04a4817f0a3", size = 10872122, upload-time = "2024-07-03T20:09:31.945Z" }, -] - -[[package]] -name = "sphinx-immaterial" -version = "0.12.5" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version > '3.9' and python_full_version < '3.10'", - "python_full_version == '3.9'", -] -dependencies = [ - { name = "appdirs" }, - { name = "markupsafe", version = "3.0.3", source = { registry = "https://pypi.org/simple" } }, - { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" } }, - { name = "pydantic-extra-types", version = "2.11.1", source = { registry = "https://pypi.org/simple" } }, - { name = "requests", version = "2.32.5", source = { registry = "https://pypi.org/simple" } }, - { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" } }, - { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" } }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5e/e8/c0ac85c8864b4aada1aa71c0c7a326cce1d8581689c18cb05348ce30bf24/sphinx_immaterial-0.12.5.tar.gz", hash = "sha256:a7c0c4be3dcb4960eb7b299dfee07cdf8a02bf56821f5d0d62e5d31b7b7b5ec5", size = 8349000, upload-time = "2025-01-30T22:51:51.667Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/99/90471644a1dfa18fb801544c9eb3663893801cec049defe077e0e6026c1e/sphinx_immaterial-0.12.5-py3-none-any.whl", hash = "sha256:4173b22ad343fd9c75b51baf305851d89b98b94603c474b428e30e8c8476673b", size = 10885262, upload-time = "2025-01-30T22:51:47.207Z" }, -] - -[[package]] -name = "sphinx-immaterial" -version = "0.13.9" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.13' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "python_full_version == '3.12.*' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "python_full_version == '3.10.*' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "(python_full_version >= '3.13' and platform_machine != 'ARM64') or (python_full_version >= '3.13' and sys_platform != 'win32')", - "(python_full_version == '3.12.*' and platform_machine != 'ARM64') or (python_full_version == '3.12.*' and sys_platform != 'win32')", - "(python_full_version == '3.11.*' and platform_machine != 'ARM64') or (python_full_version == '3.11.*' and sys_platform != 'win32')", - "(python_full_version == '3.10.*' and platform_machine != 'ARM64') or (python_full_version == '3.10.*' and sys_platform != 'win32')", -] -dependencies = [ - { name = "appdirs" }, - { name = "markupsafe", version = "3.0.3", source = { registry = "https://pypi.org/simple" } }, - { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" } }, - { name = "pydantic-extra-types", version = "2.11.1", source = { registry = "https://pypi.org/simple" } }, - { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" } }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "docutils", version = "0.20.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, + { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "sphinx", version = "7.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" } }, + { name = "sphinxcontrib-jquery" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/84/68/a1bfbf38c0f7bccc9b10bbf76b94606f64acb1552ae394f0b8285bfaea25/sphinx_rtd_theme-3.1.0.tar.gz", hash = "sha256:b44276f2c276e909239a4f6c955aa667aaafeb78597923b1c60babc76db78e4c", size = 7620915, upload-time = "2026-01-12T16:03:31.17Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/42/6e958fc5d80ccd18c87d1b7d7c0e17fed04c0ed8a72933dd41c8643622d4/sphinx_immaterial-0.13.9-py3-none-any.whl", hash = "sha256:5ea92d2ddc6befcd0fedbd3e6766ea4746e94d9a8a5cc0ab092a946e1fde4254", size = 13742592, upload-time = "2026-02-06T16:53:11.262Z" }, + { url = "https://files.pythonhosted.org/packages/87/c7/b5c8015d823bfda1a346adb2c634a2101d50bb75d421eb6dcb31acd25ebc/sphinx_rtd_theme-3.1.0-py2.py3-none-any.whl", hash = "sha256:1785824ae8e6632060490f67cf3a72d404a85d2d9fc26bce3619944de5682b89", size = 7655617, upload-time = "2026-01-12T16:03:28.101Z" }, ] [[package]] @@ -7359,6 +7263,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705, upload-time = "2024-07-29T01:09:36.407Z" }, ] +[[package]] +name = "sphinxcontrib-jquery" +version = "4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx", version = "7.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/f3/aa67467e051df70a6330fe7770894b3e4f09436dea6881ae0b4f3d87cad8/sphinxcontrib-jquery-4.1.tar.gz", hash = "sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a", size = 122331, upload-time = "2023-03-14T15:01:01.944Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/85/749bd22d1a68db7291c89e2ebca53f4306c3f205853cf31e9de279034c3c/sphinxcontrib_jquery-4.1-py2.py3-none-any.whl", hash = "sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae", size = 121104, upload-time = "2023-03-14T15:01:00.356Z" }, +] + [[package]] name = "sphinxcontrib-jsmath" version = "1.0.1" From 65624de22b978662f5027dbf05532c96fb78220a Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 11:31:59 +0200 Subject: [PATCH 03/12] docs: add local_push feature guide and regenerate CLI documentation --- docs/cli.md | 90 +++++++++++++++++++++++++++++----------------- docs/cli.md.in | 4 --- docs/user_guide.md | 21 +++++++++++ 3 files changed, 79 insertions(+), 36 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 37875bdf..afe46fd3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -14,7 +14,6 @@ Options: Commands: alias Query remote and local aliases. config Query/update application configuration. - database Manage local simulation database. manifest Create/check manifest file. provenance Create the PROVENANCE_FILE from the current system. remote Interact with the remote SimDB service. @@ -142,7 +141,6 @@ Options: --help Show this message and exit. ``` - ## Manifest @@ -421,30 +419,38 @@ Usage: simdb remote [NAME] query [OPTIONS] [CONSTRAINTS]... NAME=[mod]VALUE Where `[mod]` is an optional query modifier. Available query modifiers are: - eq: - This checks for equality (this is the same behaviour as not providing any modifier). + eq: - This checks for equality (this is the same behaviour as not providing + any modifier). in: - This searches inside the value instead of looking for exact matches. gt: - This checks for values greater than the given quantity. agt: - This checks for any array elements are greater than the given quantity. ge: - This checks for values greater than or equal to the given quantity. - age: - This checks for any array elements are greater than or equal to the given quantity. + age: - This checks for any array elements are greater than or equal to the given + quantity. lt: - This checks for values less than the given quantity. - alt: - This checks for any array elements are less than the given quantity. + alt: - This checks for any array elements are less than the given quantity. le: - This checks for values less than or equal to the given quantity. - ale: - This checks for any array elements are less than or equal to the given quantity. + ale: - This checks for any array elements are less than or equal to the given + quantity. Modifier examples: alias=eq:foo performs exact match - summary.code.name=in:foo matches all names containing foo - summary.heating_current_drive.power_additional.value=agt:0 matches all simulations where any array element - of summary.heating_current_drive.power_additional.value is greater than 0 + summary.code.name=in:foo matches all names + containing foo + summary.heating_current_drive.power_additional.value=agt:0 matches all + simulations where any array element of + summary.heating_current_drive.power_additional.value is greater than 0 - Any string comparisons are done in a case-insensitive manner. If multiple constraints are provided then simulations - are returned that match all given constraints. + Any string comparisons are done in a case-insensitive manner. If multiple + constraints are provided then simulations are returned that match all given + constraints. Examples: - sim remote query workflow.name=in:test finds all simulations where workflow.name contains test - (case-insensitive) - sim remote query pulse=gt:1000 run=0 finds all simulations where pulse is > 1000 and run = 0 + sim remote query workflow.name=in:test finds all simulations where + workflow.name contains test + (case-insensitive) + sim remote query pulse=gt:1000 run=0 finds all simulations where pulse + is > 1000 and run = 0 Options: -m, --meta-data TEXT Additional meta-data field to print. @@ -603,20 +609,21 @@ Options: --help Show this message and exit. Commands: - delete Delete the ingested simulation with given SIM_ID (UUID or... - info Print information on the simulation with given SIM_ID (UUID... - ingest Ingest a MANIFEST_FILE. - list List ingested simulations. - modify Modify the ingested simulation. - pull Pull the simulation with the given SIM_ID (UUID or alias)... - push Push the simulation with the given SIM_ID (UUID or alias) to... - query Perform a metadata query to find matching local simulations. - validate Validate the ingested simulation with given SIM_ID (UUID or... + delete Delete the ingested simulation with given SIM_ID (UUID or... + info Print information on the simulation with given SIM_ID (UUID... + ingest Ingest a MANIFEST_FILE. + list List ingested simulations. + modify Modify the ingested simulation. + pull Pull the simulation with the given SIM_ID (UUID or alias)... + push Push the simulation with the given SIM_ID (UUID or alias)... + push_local Push the simulation with the given SIM_ID (UUID or alias)... + query Perform a metadata query to find matching local simulations. + validate Validate the ingested simulation with given SIM_ID (UUID or... ``` ```text -Usage: simdb simulation delete [OPTIONS] SIM_ID +Usage: simdb simulation delete [OPTIONS] [SIM_ID] Delete the ingested simulation with given SIM_ID (UUID or alias). @@ -703,6 +710,20 @@ Options: ``` +```text +Usage: simdb simulation push_local [OPTIONS] [REMOTE] SIM_ID + + Push the simulation with the given SIM_ID (UUID or alias) to the REMOTE. + +Options: + --username TEXT Username used to authenticate with the remote. + --password TEXT Password used to authenticate with the remote. + --replaces TEXT SIM_ID of simulation to deprecate and replace. + --add-watcher Add the current user as a watcher of the simulation. + --help Show this message and exit. +``` + + ```text Usage: simdb simulation query [OPTIONS] [CONSTRAINTS]... @@ -712,7 +733,8 @@ Usage: simdb simulation query [OPTIONS] [CONSTRAINTS]... NAME=[mod]VALUE Where `[mod]` is an optional query modifier. Available query modifiers are: - eq: - This checks for equality (this is the same behaviour as not providing any modifier). + eq: - This checks for equality (this is the same behaviour as not providing any + modifier). ne: - This checks for value that do not equal. in: - This searches inside the value instead of looking for exact matches. ni: - This searches inside the value for elements that do not match. @@ -722,22 +744,26 @@ Usage: simdb simulation query [OPTIONS] [CONSTRAINTS]... le: - This checks for values less than or equal to the given quantity. For the following modifiers, VALUE should not be provided. exist: - This - returns simulations where metadata with NAME exists, regardless of the - value. + returns simulations where metadata with NAME exists, regardless + of the value. Modifier examples: responsible_name=foo performs exact match responsible_name=in:foo matches all names containing foo pulse=gt:1000 matches all pulses > 1000 - sequence=exist: matches all simulations that have "sequence" metadata values + sequence=exist: matches all simulations that have "sequence" + metadata values - Any string comparisons are done in a case-insensitive manner. If multiple constraints are provided then simulations - are returned that match all given constraints. + Any string comparisons are done in a case-insensitive manner. If multiple + constraints are provided then simulations are returned that match all given + constraints. Examples: - sim simulation query workflow.name=in:test finds all simulations where workflow.name contains test + sim simulation query workflow.name=in:test finds all simulations where + workflow.name contains test (case-insensitive) - sim simulation query pulse=gt:1000 run=0 finds all simulations where pulse is > 1000 and run = 0 + sim simulation query pulse=gt:1000 run=0 finds all simulations where + pulse is > 1000 and run = 0 Options: -m, --meta-data TEXT Additional meta-data field to print. diff --git a/docs/cli.md.in b/docs/cli.md.in index c1b8371b..adf14fa3 100644 --- a/docs/cli.md.in +++ b/docs/cli.md.in @@ -10,10 +10,6 @@ {{ config }} -## Database - -{{ database }} - ## Manifest {{ manifest }} diff --git a/docs/user_guide.md b/docs/user_guide.md index 6b4a8ebd..01c72608 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -259,6 +259,27 @@ simdb simulation push This will upload all the metadata associated with your simulation to the remote server as well as taking copies of all input and output data specified. For non-IMAS data the `file` URIs will be used to locate the files to transfer, whereas for `imas` URIs SimDB will discover which files need to be transferred based on the IMAS backend specified in the URI. The files are copied to the server using an HTTP data transfer. +### Pushing Local Simulations (Optimized for Shared File Systems) + +If your local environment and the remote SimDB server share a common local file system (for example, on the ITER network where both you and the server can access same physical file paths directly), uploading large datasets over HTTP can be slow and redundant. + +In this scenario, you should use the `push_local` command: + +```bash +simdb simulation push_local +``` + +Unlike the standard `push` command, `push_local` only sends the simulation metadata and the storage file paths to the server. The remote server then: +1. Validates the simulation metadata against active schemas. +2. Queues the file copy operation in the background using an asynchronous Celery task queue. +3. Automatically completes the ingestion once background file copying finishes. + +The CLI command will block and print real-time updates while waiting for the background ingestion to complete: +```text +Waiting for ingestion to complete... queued -> copy_files -> completed +Successfully pushed simulation +``` + ## Pulling simulations from a remote The mirror to pushing simulations is the `pull` command. This command will pull the simulation metadata from the SimDB remote to your local SimDB database and download the simulation data into a directory of your choosing. Once you have pulled a simulation it will appear in any local SimDB queries you perform. The command looks as follows: From 992daf1fb978e40f81eaa17fa9a0cd9c8fed784f Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 11:40:53 +0200 Subject: [PATCH 04/12] docs: add partition configuration guide for push_local --- docs/user_guide.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/user_guide.md b/docs/user_guide.md index 01c72608..0a49d708 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -280,6 +280,29 @@ Waiting for ingestion to complete... queued -> copy_files -> completed Successfully pushed simulation ``` +#### Configuring Partitions (Shared Storage Mappings) + +To enable `push_local` to map and resolve files correctly between your local environment and the remote server, you must configure **partitions** in your `simdb.cfg` configuration file. + +Partitions define mappings between short, logical names (such as `data` or `work`) and their absolute paths on your local file system. + +##### 1. Defining Partitions on the Client +Add a `[partition]` section to your `~/.config/simdb/simdb.cfg` file, specifying the directory paths for each partition: + +```ini +[partition] +data = /home/user/my_simdb_data +work = /work/imas/shared +``` + +##### 2. How Partitions are Resolved +When you run `push_local`: +* SimDB scans the manifest's input/output files and checks if any path falls under one of your defined local partitions. +* If a match is found (e.g., `/home/user/my_simdb_data/scenarios/run1.txt` is inside `/home/user/my_simdb_data`), SimDB converts the file URI into a partition-relative scheme: `data:///scenarios/run1.txt`. +* The remote server receives this logical URI. As long as the server also has the `data` partition configured (even if mounted at a different absolute path like `/mnt/shared/partition`), it resolves the URI to `/mnt/shared/partition/scenarios/run1.txt` and completes ingestion. + +This mapping mechanism allows clients and servers to share data over a network or cluster filesystem even if they mount it at different absolute paths. + ## Pulling simulations from a remote The mirror to pushing simulations is the `pull` command. This command will pull the simulation metadata from the SimDB remote to your local SimDB database and download the simulation data into a directory of your choosing. Once you have pulled a simulation it will appear in any local SimDB queries you perform. The command looks as follows: From 54695078b081db32541fb09d19af820205a0eff4 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 11:42:58 +0200 Subject: [PATCH 05/12] docs: add sdcc root partition mapping example --- docs/user_guide.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/user_guide.md b/docs/user_guide.md index 0a49d708..1c082df8 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -293,8 +293,12 @@ Add a `[partition]` section to your `~/.config/simdb/simdb.cfg` file, specifying [partition] data = /home/user/my_simdb_data work = /work/imas/shared +sdcc = / ``` +*Note on `sdcc` partition mapping:* +In environments like the ITER network, files are often located under absolute paths like `/sdcc/projects/...`. Mapping the `sdcc` partition to the system root `/` ensures that any path beginning with `/sdcc` is correctly matched and converted to a partition-relative URI (e.g., `/sdcc/projects/my_run` becomes `sdcc:///sdcc/projects/my_run`). + ##### 2. How Partitions are Resolved When you run `push_local`: * SimDB scans the manifest's input/output files and checks if any path falls under one of your defined local partitions. From 30d7ec1a3357fe7829a7e9d59d44f7a65c478bbf Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Wed, 1 Jul 2026 09:56:37 +0200 Subject: [PATCH 06/12] Cleanup --- src/simdb/checksum.py | 19 +++- src/simdb/cli/commands/simulation.py | 29 ++++- src/simdb/cli/manifest.py | 67 ++++------- src/simdb/cli/remote_api.py | 161 ++++++++++----------------- src/simdb/imas/utils.py | 31 ++++++ src/simdb/workers/tasks.py | 46 ++------ 6 files changed, 162 insertions(+), 191 deletions(-) diff --git a/src/simdb/checksum.py b/src/simdb/checksum.py index 99aef230..f85aea1f 100644 --- a/src/simdb/checksum.py +++ b/src/simdb/checksum.py @@ -4,6 +4,19 @@ from simdb.imas.utils import SimDBUrl +def calculate_checksum(path: Path) -> str: + """Generate a SHA1 checksum from the file at the given path. + + :param path: the path of the file to checksum + :return: a string containing the hex representation of the computed SHA1 checksum + """ + sha1 = hashlib.sha1() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(4096), b""): + sha1.update(chunk) + return sha1.hexdigest() + + def sha1_checksum(uri: SimDBUrl) -> str: """Generate a SHA1 checksum from the given file. @@ -21,8 +34,4 @@ def sha1_checksum(uri: SimDBUrl) -> str: if not path.is_file(): raise ValueError("File appears to be a directory") - sha1 = hashlib.sha1() - with path.open("rb") as file: - for chunk in iter(lambda: file.read(4096), b""): - sha1.update(chunk) - return sha1.hexdigest() + return calculate_checksum(path) diff --git a/src/simdb/cli/commands/simulation.py b/src/simdb/cli/commands/simulation.py index 1fe7a5f4..8f1fb37d 100644 --- a/src/simdb/cli/commands/simulation.py +++ b/src/simdb/cli/commands/simulation.py @@ -15,6 +15,7 @@ from simdb.config.config import Config from simdb.database import DatabaseError, get_local_db from simdb.database.models import Simulation +from simdb.enums import IngestionStatus from simdb.query import QueryType, parse_query_arg from simdb.validation import ValidationError, Validator @@ -216,6 +217,13 @@ def parse_args(self, ctx, args): is_flag=True, help="Add the current user as a watcher of the simulation.", ) +@click.option( + "--timeout", + type=float, + default=600.0, + show_default=True, + help="Maximum number of seconds to wait for ingestion to complete.", +) def simulation_push_local( config: Config, remote: Optional[str], @@ -224,6 +232,7 @@ def simulation_push_local( password: Optional[str], replaces: Optional[str], add_watcher: bool, + timeout: float, ): """Push the simulation with the given SIM_ID (UUID or alias) to the REMOTE.""" @@ -244,10 +253,17 @@ def simulation_push_local( except ValidationError as err: raise click.ClickException(f"Simulation does not validate: {err}") from err - api.push_local_simulation(simulation) + api.push_local_simulation(simulation, add_watcher=add_watcher) + + terminal_statuses = { + IngestionStatus.COMPLETED.value, + IngestionStatus.COPY_FAILED.value, + IngestionStatus.VALIDATION_FAILED.value, + } click.echo("Waiting for ingestion to complete...", nl=False) last_status = None + deadline = time.monotonic() + timeout while True: try: status = api.get_ingestion_status(simulation.uuid.hex) @@ -264,13 +280,20 @@ def simulation_push_local( click.echo(f" {status}", nl=False) last_status = status - if status in ("completed", "copy_failed", "validation_failed"): + if status in terminal_statuses: break + if time.monotonic() >= deadline: + click.echo() + raise click.ClickException( + f"Timed out after {timeout:g}s waiting for ingestion to complete " + f"(last status: {status})" + ) + time.sleep(1) click.echo() - if status == "completed": + if status == IngestionStatus.COMPLETED.value: click.echo(f"Successfully pushed simulation {simulation.uuid}") else: raise click.ClickException(f"Simulation ingestion failed with status: {status}") diff --git a/src/simdb/cli/manifest.py b/src/simdb/cli/manifest.py index f7a41a83..9efdeb3a 100644 --- a/src/simdb/cli/manifest.py +++ b/src/simdb/cli/manifest.py @@ -252,53 +252,30 @@ def resolve_metadata(self, info) -> "Manifest": def resolve_inputs_and_outputs(self, info) -> "Manifest": context = info.context or {} skip_glob_check = context.get("skip_glob_check", False) - base_path = context.get("base_path") - if not base_path: - context["base_path"] = ( - self._path.absolute().parent if self._path != Path() else Path.cwd() - ) - inputs = [] - for i in self.inputs_raw: - if i.type == DataType.FILE: - if i.uri.path: - source_path = Path(i.uri.path) - if not skip_glob_check: - names = [ - p.as_posix() - for p in source_path.parent.glob(source_path.name) - ] - if not names: - raise ValueError( - f"No files found matching path {source_path}" - ) + def _resolve(items, factory_cls): + resolved = [] + for item in items: + if item.type != DataType.FILE or not item.uri.path: + resolved.append(item) + continue + + item_path = Path(item.uri.path) + names = [p.as_posix() for p in item_path.parent.glob(item_path.name)] + if not names: + if skip_glob_check: + names = [item_path.as_posix()] else: - names = [source_path.as_posix()] - for name in names: - inputs.append( - Source(uri=SimDBUrl.build(scheme="file", path=name)) - ) - else: - inputs.append(i) - self._inputs = inputs - - outputs = [] - for i in self.outputs_raw: - if i.type == DataType.FILE: - if i.uri.path: - sink_path = Path(i.uri.path) - names = [ - p.as_posix() for p in sink_path.parent.glob(sink_path.name) - ] - if not names and skip_glob_check: - names = [sink_path.as_posix()] - for name in names: - outputs.append( - Sink(uri=SimDBUrl.build(scheme="file", path=name)) - ) - else: - outputs.append(i) - self._outputs = outputs + raise ValueError(f"No files found matching path {item_path}") + + for name in names: + resolved.append( + factory_cls(uri=SimDBUrl.build(scheme="file", path=name)) + ) + return resolved + + self._inputs = _resolve(self.inputs_raw, Source) + self._outputs = _resolve(self.outputs_raw, Sink) return self diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 845ac17f..e1183975 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -34,13 +34,13 @@ from requests.auth import AuthBase from semantic_version import Version +from simdb.checksum import calculate_checksum from simdb.config import Config from simdb.database.models import Simulation -from simdb.imas.utils import SimDBUrl, imas_files +from simdb.imas.utils import SimDBUrl, imas_backend_for_directory, imas_files from simdb.json import CustomDecoder, CustomEncoder from simdb.remote import CLIENT_API_VERSIONS, APIConstants from simdb.remote.models import FileData, SimulationPostData -from simdb.workers.tasks import _calculate_checksum from .manifest import DataType @@ -170,43 +170,49 @@ def _get_paths(file: "File") -> Iterable[Path]: def _check_file_is_imas(file: Path) -> bool: - # Check NetCDF + # NetCDF is identified by the IMAS "Conventions" attribute if file.suffix == ".nc": - with Dataset(file, "r") as ds: - if getattr(ds, "Conventions", None) == "IMAS": - return True - - children = set(file.parent.iterdir()) - - # ASCII heuristic - if any(child.suffix == ".ids" for child in children): - return True - - # HDF5 heuristic - if any(child.suffix == ".h5" for child in children) and any( - child.name == "master.h5" for child in children - ): - return True - - # MDSplus heuristic - if {p.name for p in children} >= { # noqa: SIM103 - "ids_001.tree", - "ids_001.characteristics", - "ids_001.datafile", - }: - return True + try: + with Dataset(file, "r") as ds: + if getattr(ds, "Conventions", None) == "IMAS": + return True + except OSError: + # Not a readable NetCDF file; fall back to the directory heuristics + pass - # No IMAS data detected - return False + return imas_backend_for_directory(file.parent) is not None -def _find_partition_for_file(file: Path, partitions: dict[str, str]): +def _find_partition_for_file( + file: Path, partitions: dict[str, str] +) -> Tuple[str, Path]: for partition, path in partitions.items(): try: return partition, file.relative_to(Path(path)) except ValueError: pass - return "file", file + raise click.ClickException( + f"File {file} is not located under any configured partition " + f"(configured partitions: {', '.join(partitions) or 'none'})" + ) + + +def _file_data_for_partition( + file: FileData, source: Path, partitions: dict[str, str] +) -> FileData: + partition, partition_path = _find_partition_for_file(source, partitions) + new_uri = SimDBUrl.build(scheme=partition, path=partition_path.as_posix(), host="") + return FileData( + type=file.type, + uri=new_uri.encoded_string(), + checksum=calculate_checksum(source), + datetime=file.datetime, + usage=file.usage, + purpose=file.purpose, + sensitivity=file.sensitivity, + access=file.access, + embargo=file.embargo, + ) def _expand_directories(files: Iterable[FileData], partitions: dict[str, str]): @@ -227,43 +233,11 @@ def _expand_directories(files: Iterable[FileData], partitions: dict[str, str]): for sub_file in file_path.iterdir(): if sub_file.is_dir(): raise ValueError("Nested directory found") - partition, sub_file_path = _find_partition_for_file( - sub_file, partitions - ) - new_uri = SimDBUrl.build( - scheme=partition, path=sub_file_path.as_posix(), host="" - ) new_file_list.append( - FileData( - type=file.type, - uri=new_uri.encoded_string(), - checksum=_calculate_checksum(sub_file), - datetime=file.datetime, - usage=file.usage, - purpose=file.purpose, - sensitivity=file.sensitivity, - access=file.access, - embargo=file.embargo, - ) + _file_data_for_partition(file, sub_file, partitions) ) else: - partition, new_file_path = _find_partition_for_file(file_path, partitions) - new_uri = SimDBUrl.build( - scheme=partition, path=new_file_path.as_posix(), host="" - ) - new_file_list.append( - FileData( - type=file.type, - uri=new_uri.encoded_string(), - checksum=_calculate_checksum(file_path), - datetime=file.datetime, - usage=file.usage, - purpose=file.purpose, - sensitivity=file.sensitivity, - access=file.access, - embargo=file.embargo, - ) - ) + new_file_list.append(_file_data_for_partition(file, file_path, partitions)) return new_file_list @@ -857,43 +831,36 @@ def _send_chunk( ] self.post("files", data={}, files=files) - @try_request - def push_local_simulation(self, simulation: Simulation): - sim_data = simulation.to_model(recurse=True) - - partitions = cast(dict[str, str], self._config.get_section("partition")) - sim_data.inputs.root = _expand_directories(sim_data.inputs.root, partitions) - sim_data.outputs.root = _expand_directories(sim_data.outputs.root, partitions) - - for file in sim_data.inputs.root: + def _mark_imas_files(self, files: Iterable[FileData]) -> None: + for file in files: file_uri = SimDBUrl(file.uri) if file_uri.path is None: raise ValueError("File has no associated path") - file_path = Path(file_uri.path) partition = Path( self._config.get_string_option(f"partition.{file_uri.scheme}") ) - if _check_file_is_imas(partition / file_path): + if _check_file_is_imas(partition / Path(file_uri.path)): file.type = "IMAS" - for file in sim_data.outputs.root: - file_uri = SimDBUrl(url=file.uri) - if file_uri.path is None: - raise ValueError("File has no associated path") - file_path = Path(file_uri.path) + @try_request + def push_local_simulation(self, simulation: Simulation, add_watcher: bool = False): + sim_data = simulation.to_model(recurse=True) - partition = Path( - self._config.get_string_option(f"partition.{file_uri.scheme}") - ) - if _check_file_is_imas(partition / file_path): - file.type = "IMAS" + partitions = cast(dict[str, str], self._config.get_section("partition")) + sim_data.inputs.root = _expand_directories(sim_data.inputs.root, partitions) + sim_data.outputs.root = _expand_directories(sim_data.outputs.root, partitions) + + self._mark_imas_files(sim_data.inputs.root) + self._mark_imas_files(sim_data.outputs.root) - uploaded_by = str(simulation.meta_dict().get("uploaded_by", None)) + uploaded_by = simulation.meta_dict().get("uploaded_by") headers = {"Content-type": "application/json", "User-Agent": "it_script_basic"} post_data = SimulationPostData( - simulation=sim_data, add_watcher=False, uploaded_by=uploaded_by + simulation=sim_data, + add_watcher=add_watcher, + uploaded_by=str(uploaded_by) if uploaded_by is not None else None, ).model_dump_json() res = requests.post( f"{self._url}/v1.3/simulations", @@ -907,19 +874,13 @@ def push_local_simulation(self, simulation: Simulation): @try_request def get_ingestion_status(self, sim_id: str) -> str: headers = {"User-Agent": "it_script_basic"} - if self._server_auth != "None": - res = requests.get( - f"{self._url}/v1.3/simulation/status/{sim_id}", - headers=headers, - auth=self._get_auth(), - cookies=self._cookies, - ) - else: - res = requests.get( - f"{self._url}/v1.3/simulation/status/{sim_id}", - headers=headers, - cookies=self._cookies, - ) + auth = self._get_auth() if self._server_auth != "None" else None + res = requests.get( + f"{self._url}/v1.3/simulation/status/{sim_id}", + headers=headers, + auth=auth, + cookies=self._cookies, + ) check_return(res) return res.json()["status"] diff --git a/src/simdb/imas/utils.py b/src/simdb/imas/utils.py index 6596495a..a48430fd 100644 --- a/src/simdb/imas/utils.py +++ b/src/simdb/imas/utils.py @@ -308,6 +308,37 @@ def _get_path(uri: SimDBUrl) -> Path: return path +def imas_backend_for_directory(directory: Path) -> Optional[str]: + """ + Identify the IMAS backend of a directory by inspecting its contents. + + @param directory: a directory that may contain an IMAS dataset + @return: the backend name ("ascii", "hdf5" or "mdsplus"), or None if no IMAS + dataset is detected + """ + children = list(directory.iterdir()) + + # ASCII heuristic + if any(child.suffix == ".ids" for child in children): + return "ascii" + + # HDF5 heuristic + if any(child.suffix == ".h5" for child in children) and any( + child.name == "master.h5" for child in children + ): + return "hdf5" + + # MDSplus heuristic + if {p.name for p in children} >= { + "ids_001.tree", + "ids_001.characteristics", + "ids_001.datafile", + }: + return "mdsplus" + + return None + + def imas_files(uri: SimDBUrl) -> List[Path]: """ Return all the files associated with the given IMAS URI. diff --git a/src/simdb/workers/tasks.py b/src/simdb/workers/tasks.py index d08d9cf5..ce0608d9 100644 --- a/src/simdb/workers/tasks.py +++ b/src/simdb/workers/tasks.py @@ -1,4 +1,3 @@ -import hashlib import itertools import logging import os @@ -7,14 +6,13 @@ from typing import Iterable, List from uuid import UUID -from pydantic import AnyUrl - +from simdb.checksum import calculate_checksum as _calculate_checksum from simdb.config import Config from simdb.database.database import get_db from simdb.database.models import File from simdb.email.server import EmailServer from simdb.enums import IngestionStatus -from simdb.imas.utils import SimDBUrl +from simdb.imas.utils import SimDBUrl, imas_backend_for_directory from simdb.remote.models import FileData, FileDataList from simdb.workers.celery import celery_app @@ -44,33 +42,13 @@ def _imas_path_to_uri(imas_path: Path) -> SimDBUrl: if imas_path.suffix == ".nc": return SimDBUrl.build(scheme="file", path=imas_path.as_posix()) - children = list(imas_path.iterdir()) - - if any(child.suffix == ".ids" for child in children): - u = SimDBUrl.build( - scheme="imas", path="ascii", query=f"path={imas_path.as_posix()}" - ) - return u + backend = imas_backend_for_directory(imas_path) + if backend is None: + raise ValueError("IMAS backend could not be identified.") - if any(child.suffix == ".h5" for child in children) and any( - child.name == "master.h5" for child in children - ): - u = SimDBUrl.build( - scheme="imas", path="hdf5", query=f"path={imas_path.as_posix()}" - ) - return u - - if {p.name for p in children} == { - "ids_001.tree", - "ids_001.characteristics", - "ids_001.datafile", - }: - u = SimDBUrl.build( - scheme="imas", path="mdsplus", query=f"path={imas_path.as_posix()}" - ) - return u - - raise ValueError("IMAS backend could not be identified.") + return SimDBUrl.build( + scheme="imas", path=backend, query=f"path={imas_path.as_posix()}" + ) def _resolve_uri_to_path(uri: SimDBUrl, config: Config) -> Path: @@ -115,14 +93,6 @@ def _copy_files( shutil.copy2(source, destination) -def _calculate_checksum(path: Path) -> str: - sha1 = hashlib.sha1() - with path.open("rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - sha1.update(chunk) - return sha1.hexdigest() - - def _get_imas_identifier_path(path: Path) -> Path: if path.suffix == ".nc": return path From 239eb32dd552e4c8290cb699cab99c5182bd77e7 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Wed, 15 Jul 2026 09:48:21 +0200 Subject: [PATCH 07/12] Fix typing issue --- src/simdb/cli/manifest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/simdb/cli/manifest.py b/src/simdb/cli/manifest.py index 9efdeb3a..505cf62c 100644 --- a/src/simdb/cli/manifest.py +++ b/src/simdb/cli/manifest.py @@ -55,6 +55,8 @@ def _get_data_object_type(uri: SimDBUrl) -> "DataType": return DataType.IMAS return DataType.FILE + raise ValueError(f"URI scheme ({uri.scheme}:) not recognized") + class DataObject(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) From 52ff35f3f407131fc65262549ca0525ff203eeaa Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 10:01:12 +0200 Subject: [PATCH 08/12] Fix minor issues --- docs/cli.md | 24 +++++++++++++ docs/user_guide.md | 4 +-- src/simdb/cli/commands/simulation.py | 24 ++++++++++--- src/simdb/cli/remote_api.py | 53 ++++++++++++++++++---------- src/simdb/imas/utils.py | 6 ++-- src/simdb/remote/models.py | 6 ---- 6 files changed, 84 insertions(+), 33 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index afe46fd3..e7d312a3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -609,6 +609,7 @@ Options: --help Show this message and exit. Commands: + data Fetch IDS field data for simulation SIM_ID (UUID or alias)... delete Delete the ingested simulation with given SIM_ID (UUID or... info Print information on the simulation with given SIM_ID (UUID... ingest Ingest a MANIFEST_FILE. @@ -622,6 +623,27 @@ Commands: ``` +```text +Usage: simdb simulation data [OPTIONS] [REMOTE] SIM_ID IDS_PATH + + Fetch IDS field data for simulation SIM_ID (UUID or alias) from REMOTE. + + IDS_PATH format: + ids_name[:]/path/to/field + + Examples: + simdb sim data iter 4dd781b... profiles_1d[0]/grid/rho_tor_norm + simdb sim data 4dd781b... equilibrium:0/time_slice[0]/profiles_1d/psi + +Options: + --username TEXT Username used to authenticate with the remote. + --password TEXT Password used to authenticate with the remote. + --dd-version TEXT Convert IDS data to the requested Data Dictionary + version, e.g. 4.1.1. + --help Show this message and exit. +``` + + ```text Usage: simdb simulation delete [OPTIONS] [SIM_ID] @@ -720,6 +742,8 @@ Options: --password TEXT Password used to authenticate with the remote. --replaces TEXT SIM_ID of simulation to deprecate and replace. --add-watcher Add the current user as a watcher of the simulation. + --timeout FLOAT Maximum number of seconds to wait for ingestion to + complete. [default: 600.0] --help Show this message and exit. ``` diff --git a/docs/user_guide.md b/docs/user_guide.md index c5d2f235..1b438ed1 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -335,12 +335,12 @@ sdcc = / ``` *Note on `sdcc` partition mapping:* -In environments like the ITER network, files are often located under absolute paths like `/sdcc/projects/...`. Mapping the `sdcc` partition to the system root `/` ensures that any path beginning with `/sdcc` is correctly matched and converted to a partition-relative URI (e.g., `/sdcc/projects/my_run` becomes `sdcc:///sdcc/projects/my_run`). +In environments like the ITER network, files are often located under absolute paths like `/sdcc/projects/...`. Mapping the `sdcc` partition to the system root `/` ensures that any path beginning with `/sdcc` is correctly matched and converted to a partition-relative URI (e.g., `/sdcc/projects/my_run` becomes `sdcc:sdcc/projects/my_run`). When several partitions contain a file, the partition with the most specific (deepest) path wins, so such a catch-all mapping never shadows the other partitions. ##### 2. How Partitions are Resolved When you run `push_local`: * SimDB scans the manifest's input/output files and checks if any path falls under one of your defined local partitions. -* If a match is found (e.g., `/home/user/my_simdb_data/scenarios/run1.txt` is inside `/home/user/my_simdb_data`), SimDB converts the file URI into a partition-relative scheme: `data:///scenarios/run1.txt`. +* If a match is found (e.g., `/home/user/my_simdb_data/scenarios/run1.txt` is inside `/home/user/my_simdb_data`), SimDB converts the file URI into a partition-relative scheme: `data:scenarios/run1.txt`. * The remote server receives this logical URI. As long as the server also has the `data` partition configured (even if mounted at a different absolute path like `/mnt/shared/partition`), it resolves the URI to `/mnt/shared/partition/scenarios/run1.txt` and completes ingestion. This mapping mechanism allows clients and servers to share data over a network or cluster filesystem even if they mount it at different absolute paths. diff --git a/src/simdb/cli/commands/simulation.py b/src/simdb/cli/commands/simulation.py index 6a5b9937..11f57358 100644 --- a/src/simdb/cli/commands/simulation.py +++ b/src/simdb/cli/commands/simulation.py @@ -270,17 +270,33 @@ def simulation_push_local( IngestionStatus.VALIDATION_FAILED.value, } + max_consecutive_failures = 5 + click.echo("Waiting for ingestion to complete...", nl=False) last_status = None + consecutive_failures = 0 deadline = time.monotonic() + timeout while True: try: status = api.get_ingestion_status(simulation.uuid.hex) except Exception as err: - click.echo() - raise click.ClickException( - f"Failed to check ingestion status: {err}" - ) from err + # Tolerate transient errors: the ingestion continues server-side + consecutive_failures += 1 + if consecutive_failures >= max_consecutive_failures: + click.echo() + raise click.ClickException( + f"Failed to check ingestion status " + f"{consecutive_failures} times in a row: {err}" + ) from err + if time.monotonic() >= deadline: + click.echo() + raise click.ClickException( + f"Timed out after {timeout:g}s waiting for ingestion to " + f"complete (last status: {last_status})" + ) from err + time.sleep(1) + continue + consecutive_failures = 0 if status != last_status: if last_status is not None: diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index b9b59b05..391ef566 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -26,7 +26,7 @@ Union, cast, ) -from urllib.parse import ParseResult, urlparse +from urllib.parse import urlparse import appdirs import click @@ -250,25 +250,40 @@ def _check_file_is_imas(file: Path) -> bool: # Not a readable NetCDF file; fall back to the directory heuristics pass - return imas_backend_for_directory(file.parent) is not None + try: + imas_backend_for_directory(file.parent) + except ValueError: + return False + return True def _find_partition_for_file( - file: Path, partitions: dict[str, str] + file: Path, partitions: Dict[str, str] ) -> Tuple[str, Path]: + # Match the partition with the longest root so that a catch-all mapping + # (e.g. "/") does not shadow more specific partitions. + best: Optional[Tuple[str, Path]] = None + best_depth = -1 for partition, path in partitions.items(): + root = Path(path) try: - return partition, file.relative_to(Path(path)) + relative = file.relative_to(root) except ValueError: - pass - raise click.ClickException( - f"File {file} is not located under any configured partition " - f"(configured partitions: {', '.join(partitions) or 'none'})" - ) + continue + depth = len(root.parts) + if depth > best_depth: + best = (partition, relative) + best_depth = depth + if best is None: + raise APIError( + f"File {file} is not located under any configured partition " + f"(configured partitions: {', '.join(partitions) or 'none'})" + ) + return best def _file_data_for_partition( - file: FileData, source: Path, partitions: dict[str, str] + file: FileData, source: Path, partitions: Dict[str, str] ) -> FileData: partition, partition_path = _find_partition_for_file(source, partitions) new_uri = SimDBUrl.build(scheme=partition, path=partition_path.as_posix()) @@ -285,24 +300,26 @@ def _file_data_for_partition( ) -def _expand_directories(files: Iterable[FileData], partitions: dict[str, str]): +def _expand_directories(files: Iterable[FileData], partitions: Dict[str, str]): new_file_list = [] for file in files: file_uri = SimDBUrl(file.uri) if file_uri.path is None: - raise ValueError("File has no associated path") + raise APIError(f"File URI has no path: {file.uri}") file_path = Path(file_uri.path) if file_uri.scheme == "imas": qs = dict(file_uri.query_params()) path = qs.get("path") if path is None: - raise ValueError("IMAS uri has not path set") + raise APIError(f"IMAS URI has no path set: {file.uri}") file_path = Path(path) if file_path.is_dir(): for sub_file in file_path.iterdir(): if sub_file.is_dir(): - raise ValueError("Nested directory found") + raise APIError( + f"Nested directory found in {file_path}: {sub_file.name}" + ) new_file_list.append( _file_data_for_partition(file, sub_file, partitions) ) @@ -428,7 +445,7 @@ def _load_cookies( headers = {"User-Agent": "it_script_basic"} cookies_file = f"{remote}-cookies.pkl" cookies_path = Path(appdirs.user_config_dir("simdb")) / cookies_file - parsed_url: ParseResult = urlparse(self._url) + parsed_url = urlparse(self._url) base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" cookies = None @@ -936,7 +953,7 @@ def _mark_imas_files(self, files: Iterable[FileData]) -> None: for file in files: file_uri = SimDBUrl(file.uri) if file_uri.path is None: - raise ValueError("File has no associated path") + raise APIError(f"File URI has no path: {file.uri}") partition = Path( self._config.get_string_option(f"partition.{file_uri.scheme}") @@ -949,7 +966,7 @@ def _mark_imas_files(self, files: Iterable[FileData]) -> None: def push_local_simulation(self, simulation: Simulation, add_watcher: bool = False): sim_data = simulation.to_model(recurse=True) - partitions = cast(dict[str, str], self._config.get_section("partition")) + partitions = cast(Dict[str, str], self._config.get_section("partition")) sim_data.inputs.root = _expand_directories(sim_data.inputs.root, partitions) sim_data.outputs.root = _expand_directories(sim_data.outputs.root, partitions) @@ -1010,7 +1027,7 @@ def push_simulation( sim_data = simulation.data(recurse=True) try: - sim_json: bytes = json.dumps( + sim_json = json.dumps( sim_data, cls=CustomEncoder, separators=(",", ":") ).encode("utf-8") sim_json_size = len(sim_json) diff --git a/src/simdb/imas/utils.py b/src/simdb/imas/utils.py index c4c868bd..016bc400 100644 --- a/src/simdb/imas/utils.py +++ b/src/simdb/imas/utils.py @@ -311,13 +311,13 @@ def _get_path(uri: SimDBUrl) -> Path: return path -def imas_backend_for_directory(directory: Path) -> Optional[str]: +def imas_backend_for_directory(directory: Path) -> str: """ Identify the IMAS backend of a directory by inspecting its contents. @param directory: a directory that may contain an IMAS dataset - @return: the backend name ("ascii", "hdf5" or "mdsplus"), or None if no IMAS - dataset is detected + @return: the backend name ("ascii", "hdf5" or "mdsplus") + @raise ValueError: if no IMAS dataset is detected """ children = list(directory.iterdir()) diff --git a/src/simdb/remote/models.py b/src/simdb/remote/models.py index 7ea4d09e..ae9565aa 100644 --- a/src/simdb/remote/models.py +++ b/src/simdb/remote/models.py @@ -386,12 +386,6 @@ class SimulationPostResponse(BaseModel): """Validation result.""" -class SimulationPostResponse3(BaseModel): - """Response from creating a simulation.""" - - job_id: HexUUID - - class SimulationListItem(BaseModel): """Summary of a simulation for list views.""" From ada8876dda559974757c2d7ae1c8546ed4d85108 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 13:32:38 +0200 Subject: [PATCH 09/12] Remove duplicate netcdf4 dependency --- pyproject.toml | 3 +-- uv.lock | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 82af9bf9..44a29a20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dependencies = [ "distro>=1.8.0", "email-validator>=1.1", "imas-python>=2.0.1", - "netCDF4>=1.5", + "netCDF4>=1.7.2", "numpy>=1.14", "pydantic>=2.10.6", "python-dateutil>=2.6", @@ -52,7 +52,6 @@ dependencies = [ "sqlalchemy>=1.2.12,<2.0", "alembic~=1.13", "rich>=14.3.3", - "netcdf4>=1.7.2", ] [project.optional-dependencies] diff --git a/uv.lock b/uv.lock index c371ef67..532738e0 100644 --- a/uv.lock +++ b/uv.lock @@ -3146,7 +3146,6 @@ requires-dist = [ { name = "imas-validator", marker = "extra == 'imas-validator'", specifier = ">=1.0.0" }, { name = "myst-parser", marker = "extra == 'build-docs'", specifier = ">=0.18.0" }, { name = "nbsphinx", marker = "extra == 'build-docs'", specifier = ">=0.8.0" }, - { name = "netcdf4", specifier = ">=1.5" }, { name = "netcdf4", specifier = ">=1.7.2" }, { name = "numpy", specifier = ">=1.14" }, { name = "plotext", specifier = "==5.3.2" }, From e5ae35995f39a55fde4c3892ddcb620f783eac08 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 13:32:55 +0200 Subject: [PATCH 10/12] Fix revision docstring in ingestion status migration --- alembic/versions/b2c52ee8ff12_add_ingestion_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alembic/versions/b2c52ee8ff12_add_ingestion_status.py b/alembic/versions/b2c52ee8ff12_add_ingestion_status.py index 05ebcd98..b9861c90 100644 --- a/alembic/versions/b2c52ee8ff12_add_ingestion_status.py +++ b/alembic/versions/b2c52ee8ff12_add_ingestion_status.py @@ -1,7 +1,7 @@ """Add ingestion status Revision ID: b2c52ee8ff12 -Revises: 9e9a4a7cd639 +Revises: 28bee3aa2429 Create Date: 2026-05-11 16:16:03.768893 """ From ea20477d13f3af3d0eb8901ceac16f1ca5a89741 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 13:33:36 +0200 Subject: [PATCH 11/12] Remove redundant _mark_imas_files pass from push_local File types are already assigned at manifest time and preserved by _file_data_for_partition, so this pass was a no-op for correctly ingested simulations. Worse, the directory heuristic could promote a plain FILE entry living next to IMAS data to IMAS, after which the server rewrites its URI. It also opened every .nc file and scanned every parent directory on a shared filesystem. --- src/simdb/cli/remote_api.py | 36 +----------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 391ef566..cb12fb29 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -31,14 +31,13 @@ import appdirs import click import requests -from netCDF4 import Dataset from requests.auth import AuthBase from semantic_version import Version from simdb.checksum import calculate_checksum from simdb.config import Config from simdb.database.models import Simulation -from simdb.imas.utils import SimDBUrl, imas_backend_for_directory, imas_files +from simdb.imas.utils import SimDBUrl, imas_files from simdb.json import CustomDecoder, CustomEncoder from simdb.remote import CLIENT_API_VERSIONS, APIConstants from simdb.remote.models import FileData, SimulationPostData @@ -239,24 +238,6 @@ def _get_paths(file: "File") -> Iterable[Path]: return imas_files(file.uri) -def _check_file_is_imas(file: Path) -> bool: - # NetCDF is identified by the IMAS "Conventions" attribute - if file.suffix == ".nc": - try: - with Dataset(file, "r") as ds: - if getattr(ds, "Conventions", None) == "IMAS": - return True - except OSError: - # Not a readable NetCDF file; fall back to the directory heuristics - pass - - try: - imas_backend_for_directory(file.parent) - except ValueError: - return False - return True - - def _find_partition_for_file( file: Path, partitions: Dict[str, str] ) -> Tuple[str, Path]: @@ -949,18 +930,6 @@ def _send_chunk( ] self.post("files", data={}, files=files) - def _mark_imas_files(self, files: Iterable[FileData]) -> None: - for file in files: - file_uri = SimDBUrl(file.uri) - if file_uri.path is None: - raise APIError(f"File URI has no path: {file.uri}") - - partition = Path( - self._config.get_string_option(f"partition.{file_uri.scheme}") - ) - if _check_file_is_imas(partition / Path(file_uri.path)): - file.type = "IMAS" - @versioned_method("v1.3") @try_request def push_local_simulation(self, simulation: Simulation, add_watcher: bool = False): @@ -970,9 +939,6 @@ def push_local_simulation(self, simulation: Simulation, add_watcher: bool = Fals sim_data.inputs.root = _expand_directories(sim_data.inputs.root, partitions) sim_data.outputs.root = _expand_directories(sim_data.outputs.root, partitions) - self._mark_imas_files(sim_data.inputs.root) - self._mark_imas_files(sim_data.outputs.root) - uploaded_by = simulation.meta_dict().get("uploaded_by") headers = {"Content-type": "application/json", "User-Agent": "it_script_basic"} From cd097d5fe7528e989435c5b8d53623b1eeb4502d Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 13:34:46 +0200 Subject: [PATCH 12/12] Use request helpers in push_local_simulation and get_ingestion_status Hand-rolled requests.post/requests.get bypassed the auth gating on self._server_auth, the gzip compression for large simulations payloads, and the negotiated self._api_url. --- src/simdb/cli/remote_api.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index cb12fb29..7baaea4f 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -941,33 +941,17 @@ def push_local_simulation(self, simulation: Simulation, add_watcher: bool = Fals uploaded_by = simulation.meta_dict().get("uploaded_by") - headers = {"Content-type": "application/json", "User-Agent": "it_script_basic"} post_data = SimulationPostData( simulation=sim_data, add_watcher=add_watcher, uploaded_by=str(uploaded_by) if uploaded_by is not None else None, - ).model_dump_json() - res = requests.post( - f"{self._url}/v1.3/simulations", - data=post_data, - headers=headers, - auth=self._get_auth(), - cookies=self._cookies, ) - check_return(res) + self.post("simulations", data=post_data.model_dump(mode="json")) @versioned_method("v1.3") @try_request def get_ingestion_status(self, sim_id: str) -> str: - headers = {"User-Agent": "it_script_basic"} - auth = self._get_auth() if self._server_auth != "None" else None - res = requests.get( - f"{self._url}/v1.3/simulation/status/{sim_id}", - headers=headers, - auth=auth, - cookies=self._cookies, - ) - check_return(res) + res = self.get(f"simulation/status/{sim_id}") return res.json()["status"] @versioned_method("v1.2", "v1.3")