From 178c7dea02a10fa1aec7a08522317bb686261a49 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 14 Jul 2026 14:13:03 +0200 Subject: [PATCH 01/31] Add version negotiation --- src/simdb/cli/remote_api.py | 30 +++++++++++++++++++++------- src/simdb/remote/__init__.py | 8 +++----- tests/cli/test_remote_api_version.py | 18 +++++++++++++++++ 3 files changed, 44 insertions(+), 12 deletions(-) create mode 100644 tests/cli/test_remote_api_version.py diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index b8312060..0e0092ec 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -36,7 +36,7 @@ from simdb.database.models import Simulation from simdb.imas.utils import SimDBUrl, imas_files from simdb.json import CustomDecoder, CustomEncoder -from simdb.remote import APIConstants +from simdb.remote import CLIENT_API_VERSIONS, APIConstants from .manifest import DataType @@ -131,6 +131,19 @@ def _read_bytes_in_chunks( yield data +def select_api_version( + server_versions: Iterable[str], + client_versions: Iterable[str] = CLIENT_API_VERSIONS, +) -> Optional[str]: + """ + Select the highest API version supported by both the server and this client. + """ + common_versions = set(server_versions) & set(client_versions) + if not common_versions: + return None + return max(common_versions, key=lambda v: Version.coerce(v.lstrip("v"))) + + def check_return(res: "requests.Response") -> None: if res.status_code != 200: try: @@ -198,7 +211,6 @@ def __init__( f"Remote '{remote}' not found. Use `simdb remote config add` to add it." ) from None - self._api_url: str = f"{self._url}/v{config.api_version}/" self._firewall: Optional[str] = config.get_string_option( f"remote.{remote}.firewall", default=None ) @@ -247,14 +259,18 @@ def __init__( endpoints = self.get_endpoints() endpoint_versions = [endpoint.split("/")[-1] for endpoint in endpoints] - if not endpoint_versions: - raise RemoteError("No compatible API version found on remote") + selected_version = select_api_version(endpoint_versions) + if selected_version is None: + raise RemoteError( + "No compatible API version found on remote: the server provides " + f"{', '.join(endpoint_versions) or 'none'} and this client supports " + f"{', '.join(CLIENT_API_VERSIONS)}." + ) - latest_version = max(endpoint_versions) if config.verbose: - print(f"Selected latest endpoint version {latest_version}") + print(f"Selected API version {selected_version}") - self._api_url += f"{latest_version}/" + self._api_url += f"{selected_version}/" self.version = Version.coerce(self.get_api_version()) self.server_version = Version.coerce(self.get_server_version()) diff --git a/src/simdb/remote/__init__.py b/src/simdb/remote/__init__.py index 02eb4593..c16fa7e9 100644 --- a/src/simdb/remote/__init__.py +++ b/src/simdb/remote/__init__.py @@ -4,11 +4,9 @@ endpoint to which simulations can be sent for staging and signing-off. """ -from semantic_version import SimpleSpec - -# Compatibility scheme for the latest API version, i.e. anything with the same major and -# minor version -COMPATIBILITY_SPEC = SimpleSpec("~=1.2.0") +# API versions supported by this client, as they appear in the server endpoint URLs. +# Update this when a new API version is added to simdb.remote.apis. +CLIENT_API_VERSIONS = ("v1", "v1.1", "v1.2") # API constants diff --git a/tests/cli/test_remote_api_version.py b/tests/cli/test_remote_api_version.py new file mode 100644 index 00000000..08b0b44b --- /dev/null +++ b/tests/cli/test_remote_api_version.py @@ -0,0 +1,18 @@ +from simdb.cli.remote_api import select_api_version + + +def test_selects_highest_common_version(): + assert ( + select_api_version(["v1", "v1.1", "v1.2", "v1.3"], ("v1", "v1.1", "v1.2")) + == "v1.2" + ) + assert select_api_version(["v1", "v1.1"], ("v1", "v1.1", "v1.2")) == "v1.1" + + +def test_no_common_version_returns_none(): + assert select_api_version(["v2"], ("v1", "v1.1", "v1.2")) is None + assert select_api_version([], ("v1", "v1.1", "v1.2")) is None + + +def test_versions_compare_semantically_not_lexicographically(): + assert select_api_version(["v1.2", "v1.10"], ("v1.2", "v1.10")) == "v1.10" From 08157736a841c5a68858dcd29ca5235461b7aef3 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 10:55:40 +0200 Subject: [PATCH 02/31] feat: local simulation push CLI command, netcdf support, and validation tests --- .github/dependabot.yml | 5 - .../b2c52ee8ff12_add_ingestion_status.py | 2 +- docs/Makefile | 8 +- docs/sphinx/conf.py | 42 +---- pyproject.toml | 3 +- src/simdb/cli/commands/simulation.py | 79 +++++++- src/simdb/cli/manifest.py | 93 ++++----- src/simdb/cli/remote_api.py | 178 +++++++++++++++++- src/simdb/imas/utils.py | 3 - src/simdb/remote/apis/files.py | 6 +- src/simdb/remote/apis/v1/simulations.py | 2 +- src/simdb/remote/apis/v1_1/simulations.py | 2 +- src/simdb/remote/apis/v1_2/simulations.py | 4 +- src/simdb/remote/models.py | 6 + src/simdb/validation/file/ids_validator.py | 2 +- src/simdb/validation/validator.py | 52 +++-- src/simdb/workers/tasks.py | 6 +- tests/validation/test_validator.py | 56 ++++++ tests/workers/test_tasks.py | 4 +- uv.lock | 140 +++----------- 20 files changed, 453 insertions(+), 240 deletions(-) create mode 100644 tests/validation/test_validator.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 001b22a5..d52cfe9a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,9 +7,4 @@ updates: # Check for updates once a week schedule: interval: "weekly" - # Group actions version bumps into a single PR - groups: - actions-deps: - patterns: - - "*" diff --git a/alembic/versions/b2c52ee8ff12_add_ingestion_status.py b/alembic/versions/b2c52ee8ff12_add_ingestion_status.py index b9861c90..05ebcd98 100644 --- a/alembic/versions/b2c52ee8ff12_add_ingestion_status.py +++ b/alembic/versions/b2c52ee8ff12_add_ingestion_status.py @@ -1,7 +1,7 @@ """Add ingestion status Revision ID: b2c52ee8ff12 -Revises: 28bee3aa2429 +Revises: 9e9a4a7cd639 Create Date: 2026-05-11 16:16:03.768893 """ diff --git a/docs/Makefile b/docs/Makefile index 44d7ed10..6d45923c 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -11,13 +11,7 @@ BUILDDIR = _build help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -.PHONY: help html Makefile - -# Copy source files from docs/ into sphinx/ before building (mirrors .readthedocs.yml) -html: Makefile - cp ../*.md $(SOURCEDIR)/ 2>/dev/null || true - cp ../*.svg $(SOURCEDIR)/ 2>/dev/null || true - @$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) +.PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py index bb2b19f8..f1cc47d4 100644 --- a/docs/sphinx/conf.py +++ b/docs/sphinx/conf.py @@ -48,8 +48,8 @@ "sphinx.ext.mathjax", "sphinx.ext.viewcode", "myst_parser", - "sphinx_immaterial", # Sphinx immaterial theme ] + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -73,7 +73,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = 'en' +language = None # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -90,46 +90,16 @@ # a list of builtin themes. # # html_theme = 'sphinx_rtd_theme' -html_theme = "sphinx_immaterial" +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # +# html_theme_options = {} html_theme_options = { - "palette": [ - { - "media": "(prefers-color-scheme: light)", - "scheme": "default", - "primary": "blue", - "accent": "light-blue", - "toggle": { - "icon": "material/lightbulb-outline", - "name": "Switch to dark mode", - }, - }, - { - "media": "(prefers-color-scheme: dark)", - "scheme": "slate", - "primary": "blue", - "accent": "light-blue", - "toggle": { - "icon": "material/lightbulb", - "name": "Switch to light mode", - }, - }, - ], - "features": [ - "navigation.expand", - "navigation.tabs", - "navigation.sections", - "navigation.top", - "search.share", - "toc.follow", - "toc.sticky", - ], - "repo_url": "https://github.com/iterorganization/SimDB", - "repo_name": "SimDB", + "page_width": "auto", + "body_max_width": "auto", } # Add any paths that contain custom static files (such as style sheets) here, diff --git a/pyproject.toml b/pyproject.toml index 5f070698..c1dbebd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dependencies = [ "sqlalchemy>=1.2.12,<2.0", "alembic~=1.13", "rich>=14.3.3", + "netcdf4>=1.7.2", ] [project.optional-dependencies] @@ -81,7 +82,7 @@ imas-validator = [ ] build-docs = [ "sphinx>=4.5", - "sphinx-immaterial>=0.11.14", + "sphinx-rtd-theme>=1.0.0", "sphinx-autodoc-typehints>=1.12.0", "myst-parser>=0.18.0", "nbsphinx>=0.8.0", diff --git a/src/simdb/cli/commands/simulation.py b/src/simdb/cli/commands/simulation.py index 5b8045a9..1fe7a5f4 100644 --- a/src/simdb/cli/commands/simulation.py +++ b/src/simdb/cli/commands/simulation.py @@ -1,5 +1,6 @@ import contextlib import sys +import time import urllib.parse from itertools import chain from pathlib import Path @@ -172,11 +173,7 @@ def simulation_info(config: Config, sim_id: str): def simulation_ingest(config: Config, manifest_file: str, alias: str): """Ingest a MANIFEST_FILE.""" - overrides = {} - if alias: - overrides["alias"] = alias - - manifest = Manifest.load_from_file(Path(manifest_file), overrides=overrides) + manifest = Manifest.load_from_file(Path(manifest_file)) simulation = Simulation(manifest, config) if alias: @@ -207,6 +204,78 @@ def parse_args(self, ctx, args): return NRequiredArgs +@simulation.command("push_local", cls=n_required_args_adaptor(1)) +@pass_config +@click.argument("remote", required=False) +@click.argument("sim_id") +@click.option("--username", help="Username used to authenticate with the remote.") +@click.option("--password", help="Password used to authenticate with the remote.") +@click.option("--replaces", help="SIM_ID of simulation to deprecate and replace.") +@click.option( + "--add-watcher", + is_flag=True, + help="Add the current user as a watcher of the simulation.", +) +def simulation_push_local( + config: Config, + remote: Optional[str], + sim_id: str, + username: Optional[str], + password: Optional[str], + replaces: Optional[str], + add_watcher: bool, +): + """Push the simulation with the given SIM_ID (UUID or alias) to the REMOTE.""" + + api = RemoteAPI(remote, username, password, config) + db = get_local_db(config) + + simulation = db.get_simulation(sim_id) + if simulation is None: + raise click.ClickException(f"Failed to find simulation: {sim_id}") + + if replaces: + simulation.set_meta("replaces", replaces) + + schemas = api.get_validation_schemas() + try: + for schema in schemas: + Validator(schema).validate(simulation) + except ValidationError as err: + raise click.ClickException(f"Simulation does not validate: {err}") from err + + api.push_local_simulation(simulation) + + click.echo("Waiting for ingestion to complete...", nl=False) + last_status = None + while True: + try: + status = api.get_ingestion_status(simulation.uuid.hex) + except Exception as err: + click.echo() + raise click.ClickException( + f"Failed to check ingestion status: {err}" + ) from err + + if status != last_status: + if last_status is not None: + click.echo(f" -> {status}", nl=False) + else: + click.echo(f" {status}", nl=False) + last_status = status + + if status in ("completed", "copy_failed", "validation_failed"): + break + + time.sleep(1) + + click.echo() + if status == "completed": + click.echo(f"Successfully pushed simulation {simulation.uuid}") + else: + raise click.ClickException(f"Simulation ingestion failed with status: {status}") + + @simulation.command("push", cls=n_required_args_adaptor(1)) @pass_config @click.argument("remote", required=False) diff --git a/src/simdb/cli/manifest.py b/src/simdb/cli/manifest.py index a64c03e2..f7a41a83 100644 --- a/src/simdb/cli/manifest.py +++ b/src/simdb/cli/manifest.py @@ -55,8 +55,6 @@ def _get_data_object_type(uri: SimDBUrl) -> "DataType": return DataType.IMAS return DataType.FILE - raise ValueError(f"URI scheme ({uri.scheme}:) not recognized") - class DataObject(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) @@ -97,6 +95,7 @@ def validate_uri(cls, v: ManifestUrl, info): elif v.scheme == "file": v = v.build( scheme="file", + host="", path=_expand_path(Path(v.path), base_path).as_posix(), ) @@ -165,7 +164,7 @@ def to_v2_data(self) -> Dict[str, Any]: class Manifest(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) - manifest_version: Literal[2] = Field(default=2) + manifest_version: Literal[2] = Field(default=2, alias="version") alias: Optional[str] = None responsible_name: Optional[str] = None inputs_raw: List[Source] = Field(default_factory=list, alias="inputs") @@ -249,46 +248,57 @@ def resolve_metadata(self, info) -> "Manifest": self._metadata.update(metadata_item) return self - def _resolve_manifest_items(self, items, factory_cls, skip_glob_check): - resolved = [] - for item in items: - if item.type == DataType.FILE and item.uri.path: - path_obj = Path(item.uri.path) - - matches = list(path_obj.parent.glob(path_obj.name)) - - if not matches and skip_glob_check: - matches = [path_obj] - - if not matches: - raise ValueError(f"No files found matching path {path_obj}") - - for p in matches: - resolved.append( - factory_cls( - uri=SimDBUrl.build(scheme="file", path=p.as_posix()) - ) - ) - else: - resolved.append(item) - return resolved - @model_validator(mode="after") def resolve_inputs_and_outputs(self, info) -> "Manifest": context = info.context or {} skip_glob_check = context.get("skip_glob_check", False) + base_path = context.get("base_path") + if not base_path: + context["base_path"] = ( + self._path.absolute().parent if self._path != Path() else Path.cwd() + ) - context.setdefault( - "base_path", - self._path.absolute().parent if self._path != Path() else Path.cwd(), - ) - - self._inputs = self._resolve_manifest_items( - self.inputs_raw, Source, skip_glob_check - ) - self._outputs = self._resolve_manifest_items( - self.outputs_raw, Sink, skip_glob_check - ) + inputs = [] + for i in self.inputs_raw: + if i.type == DataType.FILE: + if i.uri.path: + source_path = Path(i.uri.path) + if not skip_glob_check: + names = [ + p.as_posix() + for p in source_path.parent.glob(source_path.name) + ] + if not names: + raise ValueError( + f"No files found matching path {source_path}" + ) + else: + names = [source_path.as_posix()] + for name in names: + inputs.append( + Source(uri=SimDBUrl.build(scheme="file", path=name)) + ) + else: + inputs.append(i) + self._inputs = inputs + + outputs = [] + for i in self.outputs_raw: + if i.type == DataType.FILE: + if i.uri.path: + sink_path = Path(i.uri.path) + names = [ + p.as_posix() for p in sink_path.parent.glob(sink_path.name) + ] + if not names and skip_glob_check: + names = [sink_path.as_posix()] + for name in names: + outputs.append( + Sink(uri=SimDBUrl.build(scheme="file", path=name)) + ) + else: + outputs.append(i) + self._outputs = outputs return self @@ -318,18 +328,13 @@ def from_template(cls) -> "Manifest": return model @classmethod - def load_from_file( - cls, file_path: Path, overrides: Optional[dict] = None - ) -> "Manifest": + def load_from_file(cls, file_path: Path) -> "Manifest": with file_path.open() as file: try: raw_data = yaml.load(file, Loader=cls._get_loader()) except yaml.YAMLError as err: raise ValueError("badly formatted manifest") from err - if overrides: - raw_data.update(overrides) - model = cls.model_validate( raw_data, context={"base_path": file_path.absolute().parent} ) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 0e0092ec..845ac17f 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -23,12 +23,14 @@ Optional, Tuple, Union, + cast, ) -from urllib.parse import urlparse +from urllib.parse import ParseResult, urlparse import appdirs import click import requests +from netCDF4 import Dataset from requests.auth import AuthBase from semantic_version import Version @@ -37,6 +39,8 @@ from simdb.imas.utils import SimDBUrl, imas_files from simdb.json import CustomDecoder, CustomEncoder from simdb.remote import CLIENT_API_VERSIONS, APIConstants +from simdb.remote.models import FileData, SimulationPostData +from simdb.workers.tasks import _calculate_checksum from .manifest import DataType @@ -165,6 +169,104 @@ def _get_paths(file: "File") -> Iterable[Path]: return imas_files(file.uri) +def _check_file_is_imas(file: Path) -> bool: + # Check NetCDF + if file.suffix == ".nc": + with Dataset(file, "r") as ds: + if getattr(ds, "Conventions", None) == "IMAS": + return True + + children = set(file.parent.iterdir()) + + # ASCII heuristic + if any(child.suffix == ".ids" for child in children): + return True + + # HDF5 heuristic + if any(child.suffix == ".h5" for child in children) and any( + child.name == "master.h5" for child in children + ): + return True + + # MDSplus heuristic + if {p.name for p in children} >= { # noqa: SIM103 + "ids_001.tree", + "ids_001.characteristics", + "ids_001.datafile", + }: + return True + + # No IMAS data detected + return False + + +def _find_partition_for_file(file: Path, partitions: dict[str, str]): + for partition, path in partitions.items(): + try: + return partition, file.relative_to(Path(path)) + except ValueError: + pass + return "file", file + + +def _expand_directories(files: Iterable[FileData], partitions: dict[str, str]): + new_file_list = [] + for file in files: + file_uri = SimDBUrl(file.uri) + if file_uri.path is None: + raise ValueError("File has no associated path") + file_path = Path(file_uri.path) + if file_uri.scheme == "imas": + qs = dict(file_uri.query_params()) + path = qs.get("path") + if path is None: + raise ValueError("IMAS uri has not path set") + file_path = Path(path) + + if file_path.is_dir(): + for sub_file in file_path.iterdir(): + if sub_file.is_dir(): + raise ValueError("Nested directory found") + partition, sub_file_path = _find_partition_for_file( + sub_file, partitions + ) + new_uri = SimDBUrl.build( + scheme=partition, path=sub_file_path.as_posix(), host="" + ) + new_file_list.append( + FileData( + type=file.type, + uri=new_uri.encoded_string(), + checksum=_calculate_checksum(sub_file), + datetime=file.datetime, + usage=file.usage, + purpose=file.purpose, + sensitivity=file.sensitivity, + access=file.access, + embargo=file.embargo, + ) + ) + else: + partition, new_file_path = _find_partition_for_file(file_path, partitions) + new_uri = SimDBUrl.build( + scheme=partition, path=new_file_path.as_posix(), host="" + ) + new_file_list.append( + FileData( + type=file.type, + uri=new_uri.encoded_string(), + checksum=_calculate_checksum(file_path), + datetime=file.datetime, + usage=file.usage, + purpose=file.purpose, + sensitivity=file.sensitivity, + access=file.access, + embargo=file.embargo, + ) + ) + return new_file_list + + class RemoteAPI: """ Class to represent connection to remote API. @@ -281,7 +383,7 @@ def _load_cookies( headers = {"User-Agent": "it_script_basic"} cookies_file = f"{remote}-cookies.pkl" cookies_path = Path(appdirs.user_config_dir("simdb")) / cookies_file - parsed_url = urlparse(self._url) + parsed_url: ParseResult = urlparse(self._url) base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" cookies = None @@ -755,6 +857,72 @@ def _send_chunk( ] self.post("files", data={}, files=files) + @try_request + def push_local_simulation(self, simulation: Simulation): + sim_data = simulation.to_model(recurse=True) + + partitions = cast(dict[str, str], self._config.get_section("partition")) + sim_data.inputs.root = _expand_directories(sim_data.inputs.root, partitions) + sim_data.outputs.root = _expand_directories(sim_data.outputs.root, partitions) + + for file in sim_data.inputs.root: + file_uri = SimDBUrl(file.uri) + if file_uri.path is None: + raise ValueError("File has no associated path") + file_path = Path(file_uri.path) + + partition = Path( + self._config.get_string_option(f"partition.{file_uri.scheme}") + ) + if _check_file_is_imas(partition / file_path): + file.type = "IMAS" + + for file in sim_data.outputs.root: + file_uri = SimDBUrl(url=file.uri) + if file_uri.path is None: + raise ValueError("File has no associated path") + file_path = Path(file_uri.path) + + partition = Path( + self._config.get_string_option(f"partition.{file_uri.scheme}") + ) + if _check_file_is_imas(partition / file_path): + file.type = "IMAS" + + uploaded_by = str(simulation.meta_dict().get("uploaded_by", None)) + + headers = {"Content-type": "application/json", "User-Agent": "it_script_basic"} + post_data = SimulationPostData( + simulation=sim_data, add_watcher=False, uploaded_by=uploaded_by + ).model_dump_json() + res = requests.post( + f"{self._url}/v1.3/simulations", + data=post_data, + headers=headers, + auth=self._get_auth(), + cookies=self._cookies, + ) + check_return(res) + + @try_request + def get_ingestion_status(self, sim_id: str) -> str: + headers = {"User-Agent": "it_script_basic"} + if self._server_auth != "None": + res = requests.get( + f"{self._url}/v1.3/simulation/status/{sim_id}", + headers=headers, + auth=self._get_auth(), + cookies=self._cookies, + ) + else: + res = requests.get( + f"{self._url}/v1.3/simulation/status/{sim_id}", + headers=headers, + cookies=self._cookies, + ) + check_return(res) + return res.json()["status"] + @try_request def push_simulation( self, @@ -777,7 +945,7 @@ def push_simulation( sim_data = simulation.data(recurse=True) try: - sim_json = json.dumps( + sim_json: bytes = json.dumps( sim_data, cls=CustomEncoder, separators=(",", ":") ).encode("utf-8") sim_json_size = len(sim_json) @@ -1020,7 +1188,7 @@ def pull_simulation( rel_path = directory / path.relative_to(common_root) self._pull_file(file.uuid, 0, checksum, path, rel_path, out_stream) file.uri = SimDBUrl.build( - scheme="file", path=rel_path.absolute().as_posix() + scheme="file", host="", path=rel_path.absolute().as_posix() ) elif file.type == DataType.IMAS: for index, (path, checksum) in enumerate(info): @@ -1035,7 +1203,7 @@ def pull_simulation( ).absolute() backend = qs.get("backend") file.uri = SimDBUrl.build( - scheme="imas", path=backend, query=f"path={to_path}" + scheme="imas", host="", path=backend, query=f"path={to_path}" ) return simulation diff --git a/src/simdb/imas/utils.py b/src/simdb/imas/utils.py index 7b9ff234..6596495a 100644 --- a/src/simdb/imas/utils.py +++ b/src/simdb/imas/utils.py @@ -213,9 +213,6 @@ def open_imas(uri: SimDBUrl) -> DBEntry: @return: the IMAS data entry object """ - if not _is_al5(): - return _open_legacy(uri) - if uri.path is None: raise ValueError(f"invalid imas URI: {uri} - no path found in URI") diff --git a/src/simdb/remote/apis/files.py b/src/simdb/remote/apis/files.py index cd7cad5e..a02b2601 100644 --- a/src/simdb/remote/apis/files.py +++ b/src/simdb/remote/apis/files.py @@ -49,7 +49,9 @@ def _verify_file( path = secure_path(Path(sim_file.uri.path), common_root, staging_dir) if not path.exists(): raise ValueError(f"file {path} does not exist") - checksum = sha1_checksum(SimDBUrl.build(scheme="file", path=path.as_posix())) + checksum = sha1_checksum( + SimDBUrl.build(scheme="file", host="", path=path.as_posix()) + ) if sim_file.checksum != checksum: raise ValueError(f"checksum failed for file {sim_file!r}") elif sim_file.type == DataType.IMAS: @@ -66,7 +68,7 @@ def _verify_file( else: path_value = str(staging_dir) new_uri = uri.build( - scheme=uri.scheme, path=uri.path, query=f"path={path_value}" + scheme=uri.scheme, host="", path=uri.path, query=f"path={path_value}" ) checksum = imas_checksum(new_uri, ids_list or []) if sim_file.checksum != checksum: diff --git a/src/simdb/remote/apis/v1/simulations.py b/src/simdb/remote/apis/v1/simulations.py index 3a56c9cb..eef6cb48 100644 --- a/src/simdb/remote/apis/v1/simulations.py +++ b/src/simdb/remote/apis/v1/simulations.py @@ -206,7 +206,7 @@ def post(self, user: User): if not path.exists(): raise ValueError(f"simulation file {sim_file.uuid} not uploaded") if sim_file.uri.scheme.name == "file": - sim_file.uri = SimDBUrl.build(scheme="file", path=path) + sim_file.uri = SimDBUrl.build(scheme="file", path=path, host="") result = { "ingested": simulation.uuid.hex, diff --git a/src/simdb/remote/apis/v1_1/simulations.py b/src/simdb/remote/apis/v1_1/simulations.py index 03c956b9..f0b89b71 100644 --- a/src/simdb/remote/apis/v1_1/simulations.py +++ b/src/simdb/remote/apis/v1_1/simulations.py @@ -235,7 +235,7 @@ def post(self, user: User): if not path.exists(): raise ValueError(f"simulation file {sim_file.uuid} not uploaded") if sim_file.uri.scheme.name == "file": - sim_file.uri = SimDBUrl.build(scheme="file", path=path) + sim_file.uri = SimDBUrl.build(scheme="file", path=path, host="") result = { "ingested": simulation.uuid.hex, diff --git a/src/simdb/remote/apis/v1_2/simulations.py b/src/simdb/remote/apis/v1_2/simulations.py index 96681628..8a7e1f12 100644 --- a/src/simdb/remote/apis/v1_2/simulations.py +++ b/src/simdb/remote/apis/v1_2/simulations.py @@ -272,7 +272,9 @@ def post( raise ResponseException( f"simulation file {sim_file.uuid} not uploaded" ) - sim_file.uri = SimDBUrl.build(scheme="file", path=path.as_posix()) + sim_file.uri = SimDBUrl.build( + scheme="file", host="", path=path.as_posix() + ) elif sim_file.uri.scheme == "imas": qs = dict(sim_file.uri.query_params()) if copy_files: diff --git a/src/simdb/remote/models.py b/src/simdb/remote/models.py index de8bded0..53c34679 100644 --- a/src/simdb/remote/models.py +++ b/src/simdb/remote/models.py @@ -344,6 +344,12 @@ class SimulationPostResponse(BaseModel): """Validation result.""" +class SimulationPostResponse3(BaseModel): + """Response from creating a simulation.""" + + job_id: HexUUID + + class SimulationListItem(BaseModel): """Summary of a simulation for list views.""" diff --git a/src/simdb/validation/file/ids_validator.py b/src/simdb/validation/file/ids_validator.py index 07452a41..6c1e1dea 100644 --- a/src/simdb/validation/file/ids_validator.py +++ b/src/simdb/validation/file/ids_validator.py @@ -87,7 +87,7 @@ def validate_uri(self, uri: SimDBUrl, validate_options): backend = qs.get("backend") path = qs.get("path") validate_uri = SimDBUrl.build( - scheme="imas", path=backend, query=f"path={path}" + scheme="imas", host="", path=backend, query=f"path={path}" ) validate_output = validate( diff --git a/src/simdb/validation/validator.py b/src/simdb/validation/validator.py index f9a3bb2b..d317d90b 100644 --- a/src/simdb/validation/validator.py +++ b/src/simdb/validation/validator.py @@ -28,7 +28,7 @@ class ValidationError(Exception): class CustomValidator(ValidatorBase): types_mapping = cast(Any, cerberus.Validator).types_mapping.copy() - types_mapping["numpy"] = cerberus.TypeDefinition("numpy", (np.ndarray,), ()) + types_mapping["numpy"] = cerberus.TypeDefinition("numpy", (np.ndarray, dict), ()) def _validate_exists(self, check_exists, field, value): """The rule's arguments are validated against this schema: @@ -42,26 +42,42 @@ def _validate_min_value(self, min_value, field, value): {'type': 'float'} """ - if not isinstance(value, np.ndarray): - value = value[~np.isnan(value)] + if isinstance(value, dict) and "min" in value and "max" in value: + if min_value is not None and value["min"] < min_value: + self._error(field, f"Minimum {value['min']} less than {min_value}") + elif isinstance(value, np.ndarray): + try: + if np.issubdtype(value.dtype, np.number): + value = value[~np.isnan(value)] + except TypeError: + pass if value.size == 0: self._error(field, "Values in numpy array are NaN or empty") + elif min_value is not None and value.min() < min_value: + self._error(field, f"Minimum {value.min()} less than {min_value}") + else: self._error(field, "Value is not a numpy array") - if min_value is not None and value.min() < min_value: - self._error(field, f"Minimum {value.min()} less than {min_value}") def _validate_max_value(self, max_value, field, value): """The rule's arguments are validated against this schema: {'type': 'float'} """ - if not isinstance(value, np.ndarray): - value = value[~np.isnan(value)] + if isinstance(value, dict) and "min" in value and "max" in value: + if max_value is not None and value["max"] > max_value: + self._error(field, f"Maximum {value['max']} greater than {max_value}") + elif isinstance(value, np.ndarray): + try: + if np.issubdtype(value.dtype, np.number): + value = value[~np.isnan(value)] + except TypeError: + pass if value.size == 0: self._error(field, "Values in numpy array are NaN or empty") + elif max_value is not None and value.max() > max_value: + self._error(field, f"Maximum {value.max()} greater than {max_value}") + else: self._error(field, "Value is not a numpy array") - if max_value is not None and value.max() > max_value: - self._error(field, f"Maximum {value.max()} greater than {max_value}") def _compare(self, comparison, field, value, comparator: str, message: str): if comparison is None: @@ -71,13 +87,25 @@ def _compare(self, comparison, field, value, comparator: str, message: str): value = value[~np.isnan(value)] if value.size == 0: self._error(field, "Values in numpy array are NaN or empty") + return if not getattr(value, comparator)(comparison).all(): self._error(field, f"Values are not {message} {comparison}") - elif isinstance(value, float): + elif isinstance(value, (float, int)): if not getattr(value, comparator)(comparison): self._error(field, f"Value is not {message} {comparison}") + elif isinstance(value, dict) and "min" in value and "max" in value: + if comparator in ("__gt__", "__ge__"): + val_to_compare = value["min"] + elif comparator in ("__lt__", "__le__"): + val_to_compare = value["max"] + else: + self._error(field, f"Unsupported comparison for range: {comparator}") + return + + if not getattr(val_to_compare, comparator)(comparison): + self._error(field, f"Value is not {message} {comparison}") else: - self._error(field, "Value is not a numpy array or a float") + self._error(field, "Value is not a numpy array, range, or a float") def _validate_gt(self, comparison, field, value): """The rule's arguments are validated against this schema: @@ -113,7 +141,7 @@ def _normalize_coerce_float(cls, value): @classmethod def _normalize_coerce_numpy(cls, value): - if isinstance(value, np.ndarray): + if isinstance(value, (np.ndarray, dict)): return value elif isinstance(value, dict) and "min" in value and "max" in value: return np.array([value["min"], value["max"]], dtype=float) diff --git a/src/simdb/workers/tasks.py b/src/simdb/workers/tasks.py index 1a65af4a..d08d9cf5 100644 --- a/src/simdb/workers/tasks.py +++ b/src/simdb/workers/tasks.py @@ -44,7 +44,7 @@ def _imas_path_to_uri(imas_path: Path) -> SimDBUrl: if imas_path.suffix == ".nc": return SimDBUrl.build(scheme="file", path=imas_path.as_posix()) - children = set(imas_path.iterdir()) + children = list(imas_path.iterdir()) if any(child.suffix == ".ids" for child in children): u = SimDBUrl.build( @@ -60,7 +60,7 @@ def _imas_path_to_uri(imas_path: Path) -> SimDBUrl: ) return u - if {p.name for p in children} >= { + if {p.name for p in children} == { "ids_001.tree", "ids_001.characteristics", "ids_001.datafile", @@ -73,7 +73,7 @@ def _imas_path_to_uri(imas_path: Path) -> SimDBUrl: raise ValueError("IMAS backend could not be identified.") -def _resolve_uri_to_path(uri: AnyUrl, config: Config) -> Path: +def _resolve_uri_to_path(uri: SimDBUrl, config: Config) -> Path: partition = uri.scheme if not partition: raise ValueError("Partition not given") diff --git a/tests/validation/test_validator.py b/tests/validation/test_validator.py new file mode 100644 index 00000000..bcef6a40 --- /dev/null +++ b/tests/validation/test_validator.py @@ -0,0 +1,56 @@ +import numpy as np + +from simdb.validation.validator import CustomValidator + + +def test_custom_validator_min_value_max_value(): + schema = { + "field1": { + "type": "numpy", + "coerce": "numpy", + "min_value": 0.0, + "max_value": 10.0, + } + } + validator = CustomValidator(schema) + + # Test valid numpy array + assert validator.validate({"field1": np.array([1.0, 5.0, 9.0])}) + + # Test valid dictionary representing a range + assert validator.validate({"field1": {"min": 1.0, "max": 9.0}}) + + # Test numpy array out of bounds (too low) + assert not validator.validate({"field1": np.array([-1.0, 5.0, 9.0])}) + + # Test numpy array out of bounds (too high) + assert not validator.validate({"field1": np.array([1.0, 5.0, 11.0])}) + + # Test dictionary range out of bounds (min too low) + assert not validator.validate({"field1": {"min": -1.0, "max": 9.0}}) + + # Test dictionary range out of bounds (max too high) + assert not validator.validate({"field1": {"min": 1.0, "max": 11.0}}) + + +def test_custom_validator_comparisons(): + schema = { + "field_ge": {"type": "numpy", "coerce": "numpy", "ge": 0.0}, + "field_le": {"type": "numpy", "coerce": "numpy", "le": 10.0}, + } + validator = CustomValidator(schema) + + # Test valid dictionary representing a range + assert validator.validate( + {"field_ge": {"min": 0.0, "max": 5.0}, "field_le": {"min": 1.0, "max": 10.0}} + ) + + # Test invalid range for ge (min is -1, which is not >= 0) + assert not validator.validate( + {"field_ge": {"min": -1.0, "max": 5.0}, "field_le": {"min": 1.0, "max": 10.0}} + ) + + # Test invalid range for le (max is 11, which is not <= 10) + assert not validator.validate( + {"field_ge": {"min": 0.0, "max": 5.0}, "field_le": {"min": 1.0, "max": 11.0}} + ) diff --git a/tests/workers/test_tasks.py b/tests/workers/test_tasks.py index 69afb4ce..629210e2 100644 --- a/tests/workers/test_tasks.py +++ b/tests/workers/test_tasks.py @@ -46,8 +46,8 @@ def test_get_imas_identifier_path_returns_parent_for_directory(tmp_path): @pytest.mark.parametrize( "files,expected_backend", [ - (["summary.ids"], "ascii"), - (["master.h5", "summary.h5"], "hdf5"), + (["child.ids"], "ascii"), + (["master.h5", "file2.h5"], "hdf5"), (["ids_001.tree", "ids_001.characteristics", "ids_001.datafile"], "mdsplus"), ], ) diff --git a/uv.lock b/uv.lock index b80dab5a..891ac4c6 100644 --- a/uv.lock +++ b/uv.lock @@ -2941,9 +2941,7 @@ build-docs = [ { name = "sphinx-autodoc-typehints", version = "3.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, { name = "sphinx-autodoc-typehints", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "sphinx-autodoc-typehints", version = "3.12.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "sphinx-immaterial", version = "0.11.14", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "sphinx-immaterial", version = "0.12.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, - { name = "sphinx-immaterial", version = "0.13.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "sphinx-rtd-theme" }, ] celery = [ { name = "celery", version = "5.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, @@ -3014,6 +3012,7 @@ requires-dist = [ { name = "myst-parser", marker = "extra == 'build-docs'", specifier = ">=0.18.0" }, { name = "nbsphinx", marker = "extra == 'build-docs'", specifier = ">=0.8.0" }, { name = "netcdf4", specifier = ">=1.5" }, + { name = "netcdf4", specifier = ">=1.7.2" }, { name = "numpy", specifier = ">=1.14" }, { name = "psycopg2-binary", marker = "extra == 'postgres'", specifier = ">=2.8.0" }, { name = "pydantic", specifier = ">=2.10.6" }, @@ -3031,7 +3030,7 @@ requires-dist = [ { name = "simplejson", marker = "extra == 'server'", specifier = "~=3.0" }, { name = "sphinx", marker = "extra == 'build-docs'", specifier = ">=4.5" }, { name = "sphinx-autodoc-typehints", marker = "extra == 'build-docs'", specifier = ">=1.12.0" }, - { name = "sphinx-immaterial", marker = "extra == 'build-docs'", specifier = ">=0.11.14" }, + { name = "sphinx-rtd-theme", marker = "extra == 'build-docs'", specifier = ">=1.0.0" }, { name = "sqlalchemy", specifier = ">=1.2.12,<2.0" }, ] provides-extras = ["server", "auth-ad", "auth-keycloak", "auth-ldap", "auth", "imas-validator", "build-docs", "postgres", "celery", "all"] @@ -5096,48 +5095,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, ] -[[package]] -name = "pydantic-extra-types" -version = "2.10.6" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9' and platform_python_implementation != 'PyPy'", - "python_full_version < '3.9' and platform_python_implementation == 'PyPy'", -] -dependencies = [ - { name = "pydantic", version = "2.10.6", source = { registry = "https://pypi.org/simple" } }, - { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" } }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3a/10/fb64987804cde41bcc39d9cd757cd5f2bb5d97b389d81aa70238b14b8a7e/pydantic_extra_types-2.10.6.tar.gz", hash = "sha256:c63d70bf684366e6bbe1f4ee3957952ebe6973d41e7802aea0b770d06b116aeb", size = 141858, upload-time = "2025-10-08T13:47:49.483Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/93/04/5c918669096da8d1c9ec7bb716bd72e755526103a61bc5e76a3e4fb23b53/pydantic_extra_types-2.10.6-py3-none-any.whl", hash = "sha256:6106c448316d30abf721b5b9fecc65e983ef2614399a24142d689c7546cc246a", size = 40949, upload-time = "2025-10-08T13:47:48.268Z" }, -] - -[[package]] -name = "pydantic-extra-types" -version = "2.11.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.13' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "python_full_version == '3.12.*' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "python_full_version == '3.10.*' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "(python_full_version >= '3.13' and platform_machine != 'ARM64') or (python_full_version >= '3.13' and sys_platform != 'win32')", - "(python_full_version == '3.12.*' and platform_machine != 'ARM64') or (python_full_version == '3.12.*' and sys_platform != 'win32')", - "(python_full_version == '3.11.*' and platform_machine != 'ARM64') or (python_full_version == '3.11.*' and sys_platform != 'win32')", - "(python_full_version == '3.10.*' and platform_machine != 'ARM64') or (python_full_version == '3.10.*' and sys_platform != 'win32')", - "python_full_version > '3.9' and python_full_version < '3.10'", - "python_full_version == '3.9'", -] -dependencies = [ - { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" } }, - { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" } }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/71/dba38ee2651f84f7842206adbd2233d8bbdb59fb85e9fa14232486a8c471/pydantic_extra_types-2.11.1.tar.gz", hash = "sha256:46792d2307383859e923d8fcefa82108b1a141f8a9c0198982b3832ab5ef1049", size = 172002, upload-time = "2026-03-16T08:08:03.92Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl", hash = "sha256:1722ea2bddae5628ace25f2aa685b69978ef533123e5638cfbddb999e0100ec1", size = 79526, upload-time = "2026-03-16T08:08:02.533Z" }, -] - [[package]] name = "pygments" version = "2.19.2" @@ -7185,76 +7142,23 @@ wheels = [ ] [[package]] -name = "sphinx-immaterial" -version = "0.11.14" +name = "sphinx-rtd-theme" +version = "3.1.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9' and platform_python_implementation != 'PyPy'", - "python_full_version < '3.9' and platform_python_implementation == 'PyPy'", -] dependencies = [ - { name = "appdirs" }, - { name = "markupsafe", version = "2.1.5", source = { registry = "https://pypi.org/simple" } }, - { name = "pydantic", version = "2.10.6", source = { registry = "https://pypi.org/simple" } }, - { name = "pydantic-extra-types", version = "2.10.6", source = { registry = "https://pypi.org/simple" } }, - { name = "requests", version = "2.32.4", source = { registry = "https://pypi.org/simple" } }, - { name = "sphinx", version = "7.1.2", source = { registry = "https://pypi.org/simple" } }, - { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" } }, -] -sdist = { url = "https://files.pythonhosted.org/packages/41/1f/5403cb6bd08f2f13c86d9b5367e56078dbe17d11c0bd91becdf34d454976/sphinx_immaterial-0.11.14.tar.gz", hash = "sha256:e1e8ba93c78a3e007743fede01a3be43f5ae97c5cc19b8e2a4d2aa058abead61", size = 8330984, upload-time = "2024-07-03T20:09:35.091Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/fa/db6916f719970ebdd433c5606bea98bbf899d71f255efced93992f944f1a/sphinx_immaterial-0.11.14-py3-none-any.whl", hash = "sha256:dd1a30614c8ecaa931155189e7d54f211232e31cf3e5c6d28ba9f04a4817f0a3", size = 10872122, upload-time = "2024-07-03T20:09:31.945Z" }, -] - -[[package]] -name = "sphinx-immaterial" -version = "0.12.5" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version > '3.9' and python_full_version < '3.10'", - "python_full_version == '3.9'", -] -dependencies = [ - { name = "appdirs" }, - { name = "markupsafe", version = "3.0.3", source = { registry = "https://pypi.org/simple" } }, - { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" } }, - { name = "pydantic-extra-types", version = "2.11.1", source = { registry = "https://pypi.org/simple" } }, - { name = "requests", version = "2.32.5", source = { registry = "https://pypi.org/simple" } }, - { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" } }, - { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" } }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5e/e8/c0ac85c8864b4aada1aa71c0c7a326cce1d8581689c18cb05348ce30bf24/sphinx_immaterial-0.12.5.tar.gz", hash = "sha256:a7c0c4be3dcb4960eb7b299dfee07cdf8a02bf56821f5d0d62e5d31b7b7b5ec5", size = 8349000, upload-time = "2025-01-30T22:51:51.667Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/99/90471644a1dfa18fb801544c9eb3663893801cec049defe077e0e6026c1e/sphinx_immaterial-0.12.5-py3-none-any.whl", hash = "sha256:4173b22ad343fd9c75b51baf305851d89b98b94603c474b428e30e8c8476673b", size = 10885262, upload-time = "2025-01-30T22:51:47.207Z" }, -] - -[[package]] -name = "sphinx-immaterial" -version = "0.13.9" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.13' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "python_full_version == '3.12.*' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "python_full_version == '3.10.*' and platform_machine == 'ARM64' and sys_platform == 'win32'", - "(python_full_version >= '3.13' and platform_machine != 'ARM64') or (python_full_version >= '3.13' and sys_platform != 'win32')", - "(python_full_version == '3.12.*' and platform_machine != 'ARM64') or (python_full_version == '3.12.*' and sys_platform != 'win32')", - "(python_full_version == '3.11.*' and platform_machine != 'ARM64') or (python_full_version == '3.11.*' and sys_platform != 'win32')", - "(python_full_version == '3.10.*' and platform_machine != 'ARM64') or (python_full_version == '3.10.*' and sys_platform != 'win32')", -] -dependencies = [ - { name = "appdirs" }, - { name = "markupsafe", version = "3.0.3", source = { registry = "https://pypi.org/simple" } }, - { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" } }, - { name = "pydantic-extra-types", version = "2.11.1", source = { registry = "https://pypi.org/simple" } }, - { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" } }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "docutils", version = "0.20.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, + { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "sphinx", version = "7.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" } }, + { name = "sphinxcontrib-jquery" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/84/68/a1bfbf38c0f7bccc9b10bbf76b94606f64acb1552ae394f0b8285bfaea25/sphinx_rtd_theme-3.1.0.tar.gz", hash = "sha256:b44276f2c276e909239a4f6c955aa667aaafeb78597923b1c60babc76db78e4c", size = 7620915, upload-time = "2026-01-12T16:03:31.17Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/42/6e958fc5d80ccd18c87d1b7d7c0e17fed04c0ed8a72933dd41c8643622d4/sphinx_immaterial-0.13.9-py3-none-any.whl", hash = "sha256:5ea92d2ddc6befcd0fedbd3e6766ea4746e94d9a8a5cc0ab092a946e1fde4254", size = 13742592, upload-time = "2026-02-06T16:53:11.262Z" }, + { url = "https://files.pythonhosted.org/packages/87/c7/b5c8015d823bfda1a346adb2c634a2101d50bb75d421eb6dcb31acd25ebc/sphinx_rtd_theme-3.1.0-py2.py3-none-any.whl", hash = "sha256:1785824ae8e6632060490f67cf3a72d404a85d2d9fc26bce3619944de5682b89", size = 7655617, upload-time = "2026-01-12T16:03:28.101Z" }, ] [[package]] @@ -7359,6 +7263,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705, upload-time = "2024-07-29T01:09:36.407Z" }, ] +[[package]] +name = "sphinxcontrib-jquery" +version = "4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx", version = "7.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/f3/aa67467e051df70a6330fe7770894b3e4f09436dea6881ae0b4f3d87cad8/sphinxcontrib-jquery-4.1.tar.gz", hash = "sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a", size = 122331, upload-time = "2023-03-14T15:01:01.944Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/85/749bd22d1a68db7291c89e2ebca53f4306c3f205853cf31e9de279034c3c/sphinxcontrib_jquery-4.1-py2.py3-none-any.whl", hash = "sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae", size = 121104, upload-time = "2023-03-14T15:01:00.356Z" }, +] + [[package]] name = "sphinxcontrib-jsmath" version = "1.0.1" From 65624de22b978662f5027dbf05532c96fb78220a Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 11:31:59 +0200 Subject: [PATCH 03/31] docs: add local_push feature guide and regenerate CLI documentation --- docs/cli.md | 90 +++++++++++++++++++++++++++++----------------- docs/cli.md.in | 4 --- docs/user_guide.md | 21 +++++++++++ 3 files changed, 79 insertions(+), 36 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 37875bdf..afe46fd3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -14,7 +14,6 @@ Options: Commands: alias Query remote and local aliases. config Query/update application configuration. - database Manage local simulation database. manifest Create/check manifest file. provenance Create the PROVENANCE_FILE from the current system. remote Interact with the remote SimDB service. @@ -142,7 +141,6 @@ Options: --help Show this message and exit. ``` - ## Manifest @@ -421,30 +419,38 @@ Usage: simdb remote [NAME] query [OPTIONS] [CONSTRAINTS]... NAME=[mod]VALUE Where `[mod]` is an optional query modifier. Available query modifiers are: - eq: - This checks for equality (this is the same behaviour as not providing any modifier). + eq: - This checks for equality (this is the same behaviour as not providing + any modifier). in: - This searches inside the value instead of looking for exact matches. gt: - This checks for values greater than the given quantity. agt: - This checks for any array elements are greater than the given quantity. ge: - This checks for values greater than or equal to the given quantity. - age: - This checks for any array elements are greater than or equal to the given quantity. + age: - This checks for any array elements are greater than or equal to the given + quantity. lt: - This checks for values less than the given quantity. - alt: - This checks for any array elements are less than the given quantity. + alt: - This checks for any array elements are less than the given quantity. le: - This checks for values less than or equal to the given quantity. - ale: - This checks for any array elements are less than or equal to the given quantity. + ale: - This checks for any array elements are less than or equal to the given + quantity. Modifier examples: alias=eq:foo performs exact match - summary.code.name=in:foo matches all names containing foo - summary.heating_current_drive.power_additional.value=agt:0 matches all simulations where any array element - of summary.heating_current_drive.power_additional.value is greater than 0 + summary.code.name=in:foo matches all names + containing foo + summary.heating_current_drive.power_additional.value=agt:0 matches all + simulations where any array element of + summary.heating_current_drive.power_additional.value is greater than 0 - Any string comparisons are done in a case-insensitive manner. If multiple constraints are provided then simulations - are returned that match all given constraints. + Any string comparisons are done in a case-insensitive manner. If multiple + constraints are provided then simulations are returned that match all given + constraints. Examples: - sim remote query workflow.name=in:test finds all simulations where workflow.name contains test - (case-insensitive) - sim remote query pulse=gt:1000 run=0 finds all simulations where pulse is > 1000 and run = 0 + sim remote query workflow.name=in:test finds all simulations where + workflow.name contains test + (case-insensitive) + sim remote query pulse=gt:1000 run=0 finds all simulations where pulse + is > 1000 and run = 0 Options: -m, --meta-data TEXT Additional meta-data field to print. @@ -603,20 +609,21 @@ Options: --help Show this message and exit. Commands: - delete Delete the ingested simulation with given SIM_ID (UUID or... - info Print information on the simulation with given SIM_ID (UUID... - ingest Ingest a MANIFEST_FILE. - list List ingested simulations. - modify Modify the ingested simulation. - pull Pull the simulation with the given SIM_ID (UUID or alias)... - push Push the simulation with the given SIM_ID (UUID or alias) to... - query Perform a metadata query to find matching local simulations. - validate Validate the ingested simulation with given SIM_ID (UUID or... + delete Delete the ingested simulation with given SIM_ID (UUID or... + info Print information on the simulation with given SIM_ID (UUID... + ingest Ingest a MANIFEST_FILE. + list List ingested simulations. + modify Modify the ingested simulation. + pull Pull the simulation with the given SIM_ID (UUID or alias)... + push Push the simulation with the given SIM_ID (UUID or alias)... + push_local Push the simulation with the given SIM_ID (UUID or alias)... + query Perform a metadata query to find matching local simulations. + validate Validate the ingested simulation with given SIM_ID (UUID or... ``` ```text -Usage: simdb simulation delete [OPTIONS] SIM_ID +Usage: simdb simulation delete [OPTIONS] [SIM_ID] Delete the ingested simulation with given SIM_ID (UUID or alias). @@ -703,6 +710,20 @@ Options: ``` +```text +Usage: simdb simulation push_local [OPTIONS] [REMOTE] SIM_ID + + Push the simulation with the given SIM_ID (UUID or alias) to the REMOTE. + +Options: + --username TEXT Username used to authenticate with the remote. + --password TEXT Password used to authenticate with the remote. + --replaces TEXT SIM_ID of simulation to deprecate and replace. + --add-watcher Add the current user as a watcher of the simulation. + --help Show this message and exit. +``` + + ```text Usage: simdb simulation query [OPTIONS] [CONSTRAINTS]... @@ -712,7 +733,8 @@ Usage: simdb simulation query [OPTIONS] [CONSTRAINTS]... NAME=[mod]VALUE Where `[mod]` is an optional query modifier. Available query modifiers are: - eq: - This checks for equality (this is the same behaviour as not providing any modifier). + eq: - This checks for equality (this is the same behaviour as not providing any + modifier). ne: - This checks for value that do not equal. in: - This searches inside the value instead of looking for exact matches. ni: - This searches inside the value for elements that do not match. @@ -722,22 +744,26 @@ Usage: simdb simulation query [OPTIONS] [CONSTRAINTS]... le: - This checks for values less than or equal to the given quantity. For the following modifiers, VALUE should not be provided. exist: - This - returns simulations where metadata with NAME exists, regardless of the - value. + returns simulations where metadata with NAME exists, regardless + of the value. Modifier examples: responsible_name=foo performs exact match responsible_name=in:foo matches all names containing foo pulse=gt:1000 matches all pulses > 1000 - sequence=exist: matches all simulations that have "sequence" metadata values + sequence=exist: matches all simulations that have "sequence" + metadata values - Any string comparisons are done in a case-insensitive manner. If multiple constraints are provided then simulations - are returned that match all given constraints. + Any string comparisons are done in a case-insensitive manner. If multiple + constraints are provided then simulations are returned that match all given + constraints. Examples: - sim simulation query workflow.name=in:test finds all simulations where workflow.name contains test + sim simulation query workflow.name=in:test finds all simulations where + workflow.name contains test (case-insensitive) - sim simulation query pulse=gt:1000 run=0 finds all simulations where pulse is > 1000 and run = 0 + sim simulation query pulse=gt:1000 run=0 finds all simulations where + pulse is > 1000 and run = 0 Options: -m, --meta-data TEXT Additional meta-data field to print. diff --git a/docs/cli.md.in b/docs/cli.md.in index c1b8371b..adf14fa3 100644 --- a/docs/cli.md.in +++ b/docs/cli.md.in @@ -10,10 +10,6 @@ {{ config }} -## Database - -{{ database }} - ## Manifest {{ manifest }} diff --git a/docs/user_guide.md b/docs/user_guide.md index 6b4a8ebd..01c72608 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -259,6 +259,27 @@ simdb simulation push This will upload all the metadata associated with your simulation to the remote server as well as taking copies of all input and output data specified. For non-IMAS data the `file` URIs will be used to locate the files to transfer, whereas for `imas` URIs SimDB will discover which files need to be transferred based on the IMAS backend specified in the URI. The files are copied to the server using an HTTP data transfer. +### Pushing Local Simulations (Optimized for Shared File Systems) + +If your local environment and the remote SimDB server share a common local file system (for example, on the ITER network where both you and the server can access same physical file paths directly), uploading large datasets over HTTP can be slow and redundant. + +In this scenario, you should use the `push_local` command: + +```bash +simdb simulation push_local +``` + +Unlike the standard `push` command, `push_local` only sends the simulation metadata and the storage file paths to the server. The remote server then: +1. Validates the simulation metadata against active schemas. +2. Queues the file copy operation in the background using an asynchronous Celery task queue. +3. Automatically completes the ingestion once background file copying finishes. + +The CLI command will block and print real-time updates while waiting for the background ingestion to complete: +```text +Waiting for ingestion to complete... queued -> copy_files -> completed +Successfully pushed simulation +``` + ## Pulling simulations from a remote The mirror to pushing simulations is the `pull` command. This command will pull the simulation metadata from the SimDB remote to your local SimDB database and download the simulation data into a directory of your choosing. Once you have pulled a simulation it will appear in any local SimDB queries you perform. The command looks as follows: From 992daf1fb978e40f81eaa17fa9a0cd9c8fed784f Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 11:40:53 +0200 Subject: [PATCH 04/31] docs: add partition configuration guide for push_local --- docs/user_guide.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/user_guide.md b/docs/user_guide.md index 01c72608..0a49d708 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -280,6 +280,29 @@ Waiting for ingestion to complete... queued -> copy_files -> completed Successfully pushed simulation ``` +#### Configuring Partitions (Shared Storage Mappings) + +To enable `push_local` to map and resolve files correctly between your local environment and the remote server, you must configure **partitions** in your `simdb.cfg` configuration file. + +Partitions define mappings between short, logical names (such as `data` or `work`) and their absolute paths on your local file system. + +##### 1. Defining Partitions on the Client +Add a `[partition]` section to your `~/.config/simdb/simdb.cfg` file, specifying the directory paths for each partition: + +```ini +[partition] +data = /home/user/my_simdb_data +work = /work/imas/shared +``` + +##### 2. How Partitions are Resolved +When you run `push_local`: +* SimDB scans the manifest's input/output files and checks if any path falls under one of your defined local partitions. +* If a match is found (e.g., `/home/user/my_simdb_data/scenarios/run1.txt` is inside `/home/user/my_simdb_data`), SimDB converts the file URI into a partition-relative scheme: `data:///scenarios/run1.txt`. +* The remote server receives this logical URI. As long as the server also has the `data` partition configured (even if mounted at a different absolute path like `/mnt/shared/partition`), it resolves the URI to `/mnt/shared/partition/scenarios/run1.txt` and completes ingestion. + +This mapping mechanism allows clients and servers to share data over a network or cluster filesystem even if they mount it at different absolute paths. + ## Pulling simulations from a remote The mirror to pushing simulations is the `pull` command. This command will pull the simulation metadata from the SimDB remote to your local SimDB database and download the simulation data into a directory of your choosing. Once you have pulled a simulation it will appear in any local SimDB queries you perform. The command looks as follows: From 54695078b081db32541fb09d19af820205a0eff4 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 11:42:58 +0200 Subject: [PATCH 05/31] docs: add sdcc root partition mapping example --- docs/user_guide.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/user_guide.md b/docs/user_guide.md index 0a49d708..1c082df8 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -293,8 +293,12 @@ Add a `[partition]` section to your `~/.config/simdb/simdb.cfg` file, specifying [partition] data = /home/user/my_simdb_data work = /work/imas/shared +sdcc = / ``` +*Note on `sdcc` partition mapping:* +In environments like the ITER network, files are often located under absolute paths like `/sdcc/projects/...`. Mapping the `sdcc` partition to the system root `/` ensures that any path beginning with `/sdcc` is correctly matched and converted to a partition-relative URI (e.g., `/sdcc/projects/my_run` becomes `sdcc:///sdcc/projects/my_run`). + ##### 2. How Partitions are Resolved When you run `push_local`: * SimDB scans the manifest's input/output files and checks if any path falls under one of your defined local partitions. From 30d7ec1a3357fe7829a7e9d59d44f7a65c478bbf Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Wed, 1 Jul 2026 09:56:37 +0200 Subject: [PATCH 06/31] Cleanup --- src/simdb/checksum.py | 19 +++- src/simdb/cli/commands/simulation.py | 29 ++++- src/simdb/cli/manifest.py | 67 ++++------- src/simdb/cli/remote_api.py | 161 ++++++++++----------------- src/simdb/imas/utils.py | 31 ++++++ src/simdb/workers/tasks.py | 46 ++------ 6 files changed, 162 insertions(+), 191 deletions(-) diff --git a/src/simdb/checksum.py b/src/simdb/checksum.py index 99aef230..f85aea1f 100644 --- a/src/simdb/checksum.py +++ b/src/simdb/checksum.py @@ -4,6 +4,19 @@ from simdb.imas.utils import SimDBUrl +def calculate_checksum(path: Path) -> str: + """Generate a SHA1 checksum from the file at the given path. + + :param path: the path of the file to checksum + :return: a string containing the hex representation of the computed SHA1 checksum + """ + sha1 = hashlib.sha1() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(4096), b""): + sha1.update(chunk) + return sha1.hexdigest() + + def sha1_checksum(uri: SimDBUrl) -> str: """Generate a SHA1 checksum from the given file. @@ -21,8 +34,4 @@ def sha1_checksum(uri: SimDBUrl) -> str: if not path.is_file(): raise ValueError("File appears to be a directory") - sha1 = hashlib.sha1() - with path.open("rb") as file: - for chunk in iter(lambda: file.read(4096), b""): - sha1.update(chunk) - return sha1.hexdigest() + return calculate_checksum(path) diff --git a/src/simdb/cli/commands/simulation.py b/src/simdb/cli/commands/simulation.py index 1fe7a5f4..8f1fb37d 100644 --- a/src/simdb/cli/commands/simulation.py +++ b/src/simdb/cli/commands/simulation.py @@ -15,6 +15,7 @@ from simdb.config.config import Config from simdb.database import DatabaseError, get_local_db from simdb.database.models import Simulation +from simdb.enums import IngestionStatus from simdb.query import QueryType, parse_query_arg from simdb.validation import ValidationError, Validator @@ -216,6 +217,13 @@ def parse_args(self, ctx, args): is_flag=True, help="Add the current user as a watcher of the simulation.", ) +@click.option( + "--timeout", + type=float, + default=600.0, + show_default=True, + help="Maximum number of seconds to wait for ingestion to complete.", +) def simulation_push_local( config: Config, remote: Optional[str], @@ -224,6 +232,7 @@ def simulation_push_local( password: Optional[str], replaces: Optional[str], add_watcher: bool, + timeout: float, ): """Push the simulation with the given SIM_ID (UUID or alias) to the REMOTE.""" @@ -244,10 +253,17 @@ def simulation_push_local( except ValidationError as err: raise click.ClickException(f"Simulation does not validate: {err}") from err - api.push_local_simulation(simulation) + api.push_local_simulation(simulation, add_watcher=add_watcher) + + terminal_statuses = { + IngestionStatus.COMPLETED.value, + IngestionStatus.COPY_FAILED.value, + IngestionStatus.VALIDATION_FAILED.value, + } click.echo("Waiting for ingestion to complete...", nl=False) last_status = None + deadline = time.monotonic() + timeout while True: try: status = api.get_ingestion_status(simulation.uuid.hex) @@ -264,13 +280,20 @@ def simulation_push_local( click.echo(f" {status}", nl=False) last_status = status - if status in ("completed", "copy_failed", "validation_failed"): + if status in terminal_statuses: break + if time.monotonic() >= deadline: + click.echo() + raise click.ClickException( + f"Timed out after {timeout:g}s waiting for ingestion to complete " + f"(last status: {status})" + ) + time.sleep(1) click.echo() - if status == "completed": + if status == IngestionStatus.COMPLETED.value: click.echo(f"Successfully pushed simulation {simulation.uuid}") else: raise click.ClickException(f"Simulation ingestion failed with status: {status}") diff --git a/src/simdb/cli/manifest.py b/src/simdb/cli/manifest.py index f7a41a83..9efdeb3a 100644 --- a/src/simdb/cli/manifest.py +++ b/src/simdb/cli/manifest.py @@ -252,53 +252,30 @@ def resolve_metadata(self, info) -> "Manifest": def resolve_inputs_and_outputs(self, info) -> "Manifest": context = info.context or {} skip_glob_check = context.get("skip_glob_check", False) - base_path = context.get("base_path") - if not base_path: - context["base_path"] = ( - self._path.absolute().parent if self._path != Path() else Path.cwd() - ) - inputs = [] - for i in self.inputs_raw: - if i.type == DataType.FILE: - if i.uri.path: - source_path = Path(i.uri.path) - if not skip_glob_check: - names = [ - p.as_posix() - for p in source_path.parent.glob(source_path.name) - ] - if not names: - raise ValueError( - f"No files found matching path {source_path}" - ) + def _resolve(items, factory_cls): + resolved = [] + for item in items: + if item.type != DataType.FILE or not item.uri.path: + resolved.append(item) + continue + + item_path = Path(item.uri.path) + names = [p.as_posix() for p in item_path.parent.glob(item_path.name)] + if not names: + if skip_glob_check: + names = [item_path.as_posix()] else: - names = [source_path.as_posix()] - for name in names: - inputs.append( - Source(uri=SimDBUrl.build(scheme="file", path=name)) - ) - else: - inputs.append(i) - self._inputs = inputs - - outputs = [] - for i in self.outputs_raw: - if i.type == DataType.FILE: - if i.uri.path: - sink_path = Path(i.uri.path) - names = [ - p.as_posix() for p in sink_path.parent.glob(sink_path.name) - ] - if not names and skip_glob_check: - names = [sink_path.as_posix()] - for name in names: - outputs.append( - Sink(uri=SimDBUrl.build(scheme="file", path=name)) - ) - else: - outputs.append(i) - self._outputs = outputs + raise ValueError(f"No files found matching path {item_path}") + + for name in names: + resolved.append( + factory_cls(uri=SimDBUrl.build(scheme="file", path=name)) + ) + return resolved + + self._inputs = _resolve(self.inputs_raw, Source) + self._outputs = _resolve(self.outputs_raw, Sink) return self diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 845ac17f..e1183975 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -34,13 +34,13 @@ from requests.auth import AuthBase from semantic_version import Version +from simdb.checksum import calculate_checksum from simdb.config import Config from simdb.database.models import Simulation -from simdb.imas.utils import SimDBUrl, imas_files +from simdb.imas.utils import SimDBUrl, imas_backend_for_directory, imas_files from simdb.json import CustomDecoder, CustomEncoder from simdb.remote import CLIENT_API_VERSIONS, APIConstants from simdb.remote.models import FileData, SimulationPostData -from simdb.workers.tasks import _calculate_checksum from .manifest import DataType @@ -170,43 +170,49 @@ def _get_paths(file: "File") -> Iterable[Path]: def _check_file_is_imas(file: Path) -> bool: - # Check NetCDF + # NetCDF is identified by the IMAS "Conventions" attribute if file.suffix == ".nc": - with Dataset(file, "r") as ds: - if getattr(ds, "Conventions", None) == "IMAS": - return True - - children = set(file.parent.iterdir()) - - # ASCII heuristic - if any(child.suffix == ".ids" for child in children): - return True - - # HDF5 heuristic - if any(child.suffix == ".h5" for child in children) and any( - child.name == "master.h5" for child in children - ): - return True - - # MDSplus heuristic - if {p.name for p in children} >= { # noqa: SIM103 - "ids_001.tree", - "ids_001.characteristics", - "ids_001.datafile", - }: - return True + try: + with Dataset(file, "r") as ds: + if getattr(ds, "Conventions", None) == "IMAS": + return True + except OSError: + # Not a readable NetCDF file; fall back to the directory heuristics + pass - # No IMAS data detected - return False + return imas_backend_for_directory(file.parent) is not None -def _find_partition_for_file(file: Path, partitions: dict[str, str]): +def _find_partition_for_file( + file: Path, partitions: dict[str, str] +) -> Tuple[str, Path]: for partition, path in partitions.items(): try: return partition, file.relative_to(Path(path)) except ValueError: pass - return "file", file + raise click.ClickException( + f"File {file} is not located under any configured partition " + f"(configured partitions: {', '.join(partitions) or 'none'})" + ) + + +def _file_data_for_partition( + file: FileData, source: Path, partitions: dict[str, str] +) -> FileData: + partition, partition_path = _find_partition_for_file(source, partitions) + new_uri = SimDBUrl.build(scheme=partition, path=partition_path.as_posix(), host="") + return FileData( + type=file.type, + uri=new_uri.encoded_string(), + checksum=calculate_checksum(source), + datetime=file.datetime, + usage=file.usage, + purpose=file.purpose, + sensitivity=file.sensitivity, + access=file.access, + embargo=file.embargo, + ) def _expand_directories(files: Iterable[FileData], partitions: dict[str, str]): @@ -227,43 +233,11 @@ def _expand_directories(files: Iterable[FileData], partitions: dict[str, str]): for sub_file in file_path.iterdir(): if sub_file.is_dir(): raise ValueError("Nested directory found") - partition, sub_file_path = _find_partition_for_file( - sub_file, partitions - ) - new_uri = SimDBUrl.build( - scheme=partition, path=sub_file_path.as_posix(), host="" - ) new_file_list.append( - FileData( - type=file.type, - uri=new_uri.encoded_string(), - checksum=_calculate_checksum(sub_file), - datetime=file.datetime, - usage=file.usage, - purpose=file.purpose, - sensitivity=file.sensitivity, - access=file.access, - embargo=file.embargo, - ) + _file_data_for_partition(file, sub_file, partitions) ) else: - partition, new_file_path = _find_partition_for_file(file_path, partitions) - new_uri = SimDBUrl.build( - scheme=partition, path=new_file_path.as_posix(), host="" - ) - new_file_list.append( - FileData( - type=file.type, - uri=new_uri.encoded_string(), - checksum=_calculate_checksum(file_path), - datetime=file.datetime, - usage=file.usage, - purpose=file.purpose, - sensitivity=file.sensitivity, - access=file.access, - embargo=file.embargo, - ) - ) + new_file_list.append(_file_data_for_partition(file, file_path, partitions)) return new_file_list @@ -857,43 +831,36 @@ def _send_chunk( ] self.post("files", data={}, files=files) - @try_request - def push_local_simulation(self, simulation: Simulation): - sim_data = simulation.to_model(recurse=True) - - partitions = cast(dict[str, str], self._config.get_section("partition")) - sim_data.inputs.root = _expand_directories(sim_data.inputs.root, partitions) - sim_data.outputs.root = _expand_directories(sim_data.outputs.root, partitions) - - for file in sim_data.inputs.root: + def _mark_imas_files(self, files: Iterable[FileData]) -> None: + for file in files: file_uri = SimDBUrl(file.uri) if file_uri.path is None: raise ValueError("File has no associated path") - file_path = Path(file_uri.path) partition = Path( self._config.get_string_option(f"partition.{file_uri.scheme}") ) - if _check_file_is_imas(partition / file_path): + if _check_file_is_imas(partition / Path(file_uri.path)): file.type = "IMAS" - for file in sim_data.outputs.root: - file_uri = SimDBUrl(url=file.uri) - if file_uri.path is None: - raise ValueError("File has no associated path") - file_path = Path(file_uri.path) + @try_request + def push_local_simulation(self, simulation: Simulation, add_watcher: bool = False): + sim_data = simulation.to_model(recurse=True) - partition = Path( - self._config.get_string_option(f"partition.{file_uri.scheme}") - ) - if _check_file_is_imas(partition / file_path): - file.type = "IMAS" + partitions = cast(dict[str, str], self._config.get_section("partition")) + sim_data.inputs.root = _expand_directories(sim_data.inputs.root, partitions) + sim_data.outputs.root = _expand_directories(sim_data.outputs.root, partitions) + + self._mark_imas_files(sim_data.inputs.root) + self._mark_imas_files(sim_data.outputs.root) - uploaded_by = str(simulation.meta_dict().get("uploaded_by", None)) + uploaded_by = simulation.meta_dict().get("uploaded_by") headers = {"Content-type": "application/json", "User-Agent": "it_script_basic"} post_data = SimulationPostData( - simulation=sim_data, add_watcher=False, uploaded_by=uploaded_by + simulation=sim_data, + add_watcher=add_watcher, + uploaded_by=str(uploaded_by) if uploaded_by is not None else None, ).model_dump_json() res = requests.post( f"{self._url}/v1.3/simulations", @@ -907,19 +874,13 @@ def push_local_simulation(self, simulation: Simulation): @try_request def get_ingestion_status(self, sim_id: str) -> str: headers = {"User-Agent": "it_script_basic"} - if self._server_auth != "None": - res = requests.get( - f"{self._url}/v1.3/simulation/status/{sim_id}", - headers=headers, - auth=self._get_auth(), - cookies=self._cookies, - ) - else: - res = requests.get( - f"{self._url}/v1.3/simulation/status/{sim_id}", - headers=headers, - cookies=self._cookies, - ) + auth = self._get_auth() if self._server_auth != "None" else None + res = requests.get( + f"{self._url}/v1.3/simulation/status/{sim_id}", + headers=headers, + auth=auth, + cookies=self._cookies, + ) check_return(res) return res.json()["status"] diff --git a/src/simdb/imas/utils.py b/src/simdb/imas/utils.py index 6596495a..a48430fd 100644 --- a/src/simdb/imas/utils.py +++ b/src/simdb/imas/utils.py @@ -308,6 +308,37 @@ def _get_path(uri: SimDBUrl) -> Path: return path +def imas_backend_for_directory(directory: Path) -> Optional[str]: + """ + Identify the IMAS backend of a directory by inspecting its contents. + + @param directory: a directory that may contain an IMAS dataset + @return: the backend name ("ascii", "hdf5" or "mdsplus"), or None if no IMAS + dataset is detected + """ + children = list(directory.iterdir()) + + # ASCII heuristic + if any(child.suffix == ".ids" for child in children): + return "ascii" + + # HDF5 heuristic + if any(child.suffix == ".h5" for child in children) and any( + child.name == "master.h5" for child in children + ): + return "hdf5" + + # MDSplus heuristic + if {p.name for p in children} >= { + "ids_001.tree", + "ids_001.characteristics", + "ids_001.datafile", + }: + return "mdsplus" + + return None + + def imas_files(uri: SimDBUrl) -> List[Path]: """ Return all the files associated with the given IMAS URI. diff --git a/src/simdb/workers/tasks.py b/src/simdb/workers/tasks.py index d08d9cf5..ce0608d9 100644 --- a/src/simdb/workers/tasks.py +++ b/src/simdb/workers/tasks.py @@ -1,4 +1,3 @@ -import hashlib import itertools import logging import os @@ -7,14 +6,13 @@ from typing import Iterable, List from uuid import UUID -from pydantic import AnyUrl - +from simdb.checksum import calculate_checksum as _calculate_checksum from simdb.config import Config from simdb.database.database import get_db from simdb.database.models import File from simdb.email.server import EmailServer from simdb.enums import IngestionStatus -from simdb.imas.utils import SimDBUrl +from simdb.imas.utils import SimDBUrl, imas_backend_for_directory from simdb.remote.models import FileData, FileDataList from simdb.workers.celery import celery_app @@ -44,33 +42,13 @@ def _imas_path_to_uri(imas_path: Path) -> SimDBUrl: if imas_path.suffix == ".nc": return SimDBUrl.build(scheme="file", path=imas_path.as_posix()) - children = list(imas_path.iterdir()) - - if any(child.suffix == ".ids" for child in children): - u = SimDBUrl.build( - scheme="imas", path="ascii", query=f"path={imas_path.as_posix()}" - ) - return u + backend = imas_backend_for_directory(imas_path) + if backend is None: + raise ValueError("IMAS backend could not be identified.") - if any(child.suffix == ".h5" for child in children) and any( - child.name == "master.h5" for child in children - ): - u = SimDBUrl.build( - scheme="imas", path="hdf5", query=f"path={imas_path.as_posix()}" - ) - return u - - if {p.name for p in children} == { - "ids_001.tree", - "ids_001.characteristics", - "ids_001.datafile", - }: - u = SimDBUrl.build( - scheme="imas", path="mdsplus", query=f"path={imas_path.as_posix()}" - ) - return u - - raise ValueError("IMAS backend could not be identified.") + return SimDBUrl.build( + scheme="imas", path=backend, query=f"path={imas_path.as_posix()}" + ) def _resolve_uri_to_path(uri: SimDBUrl, config: Config) -> Path: @@ -115,14 +93,6 @@ def _copy_files( shutil.copy2(source, destination) -def _calculate_checksum(path: Path) -> str: - sha1 = hashlib.sha1() - with path.open("rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - sha1.update(chunk) - return sha1.hexdigest() - - def _get_imas_identifier_path(path: Path) -> Path: if path.suffix == ".nc": return path From 239eb32dd552e4c8290cb699cab99c5182bd77e7 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Wed, 15 Jul 2026 09:48:21 +0200 Subject: [PATCH 07/31] Fix typing issue --- src/simdb/cli/manifest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/simdb/cli/manifest.py b/src/simdb/cli/manifest.py index 9efdeb3a..505cf62c 100644 --- a/src/simdb/cli/manifest.py +++ b/src/simdb/cli/manifest.py @@ -55,6 +55,8 @@ def _get_data_object_type(uri: SimDBUrl) -> "DataType": return DataType.IMAS return DataType.FILE + raise ValueError(f"URI scheme ({uri.scheme}:) not recognized") + class DataObject(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) From 52ff35f3f407131fc65262549ca0525ff203eeaa Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 10:01:12 +0200 Subject: [PATCH 08/31] Fix minor issues --- docs/cli.md | 24 +++++++++++++ docs/user_guide.md | 4 +-- src/simdb/cli/commands/simulation.py | 24 ++++++++++--- src/simdb/cli/remote_api.py | 53 ++++++++++++++++++---------- src/simdb/imas/utils.py | 6 ++-- src/simdb/remote/models.py | 6 ---- 6 files changed, 84 insertions(+), 33 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index afe46fd3..e7d312a3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -609,6 +609,7 @@ Options: --help Show this message and exit. Commands: + data Fetch IDS field data for simulation SIM_ID (UUID or alias)... delete Delete the ingested simulation with given SIM_ID (UUID or... info Print information on the simulation with given SIM_ID (UUID... ingest Ingest a MANIFEST_FILE. @@ -622,6 +623,27 @@ Commands: ``` +```text +Usage: simdb simulation data [OPTIONS] [REMOTE] SIM_ID IDS_PATH + + Fetch IDS field data for simulation SIM_ID (UUID or alias) from REMOTE. + + IDS_PATH format: + ids_name[:]/path/to/field + + Examples: + simdb sim data iter 4dd781b... profiles_1d[0]/grid/rho_tor_norm + simdb sim data 4dd781b... equilibrium:0/time_slice[0]/profiles_1d/psi + +Options: + --username TEXT Username used to authenticate with the remote. + --password TEXT Password used to authenticate with the remote. + --dd-version TEXT Convert IDS data to the requested Data Dictionary + version, e.g. 4.1.1. + --help Show this message and exit. +``` + + ```text Usage: simdb simulation delete [OPTIONS] [SIM_ID] @@ -720,6 +742,8 @@ Options: --password TEXT Password used to authenticate with the remote. --replaces TEXT SIM_ID of simulation to deprecate and replace. --add-watcher Add the current user as a watcher of the simulation. + --timeout FLOAT Maximum number of seconds to wait for ingestion to + complete. [default: 600.0] --help Show this message and exit. ``` diff --git a/docs/user_guide.md b/docs/user_guide.md index c5d2f235..1b438ed1 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -335,12 +335,12 @@ sdcc = / ``` *Note on `sdcc` partition mapping:* -In environments like the ITER network, files are often located under absolute paths like `/sdcc/projects/...`. Mapping the `sdcc` partition to the system root `/` ensures that any path beginning with `/sdcc` is correctly matched and converted to a partition-relative URI (e.g., `/sdcc/projects/my_run` becomes `sdcc:///sdcc/projects/my_run`). +In environments like the ITER network, files are often located under absolute paths like `/sdcc/projects/...`. Mapping the `sdcc` partition to the system root `/` ensures that any path beginning with `/sdcc` is correctly matched and converted to a partition-relative URI (e.g., `/sdcc/projects/my_run` becomes `sdcc:sdcc/projects/my_run`). When several partitions contain a file, the partition with the most specific (deepest) path wins, so such a catch-all mapping never shadows the other partitions. ##### 2. How Partitions are Resolved When you run `push_local`: * SimDB scans the manifest's input/output files and checks if any path falls under one of your defined local partitions. -* If a match is found (e.g., `/home/user/my_simdb_data/scenarios/run1.txt` is inside `/home/user/my_simdb_data`), SimDB converts the file URI into a partition-relative scheme: `data:///scenarios/run1.txt`. +* If a match is found (e.g., `/home/user/my_simdb_data/scenarios/run1.txt` is inside `/home/user/my_simdb_data`), SimDB converts the file URI into a partition-relative scheme: `data:scenarios/run1.txt`. * The remote server receives this logical URI. As long as the server also has the `data` partition configured (even if mounted at a different absolute path like `/mnt/shared/partition`), it resolves the URI to `/mnt/shared/partition/scenarios/run1.txt` and completes ingestion. This mapping mechanism allows clients and servers to share data over a network or cluster filesystem even if they mount it at different absolute paths. diff --git a/src/simdb/cli/commands/simulation.py b/src/simdb/cli/commands/simulation.py index 6a5b9937..11f57358 100644 --- a/src/simdb/cli/commands/simulation.py +++ b/src/simdb/cli/commands/simulation.py @@ -270,17 +270,33 @@ def simulation_push_local( IngestionStatus.VALIDATION_FAILED.value, } + max_consecutive_failures = 5 + click.echo("Waiting for ingestion to complete...", nl=False) last_status = None + consecutive_failures = 0 deadline = time.monotonic() + timeout while True: try: status = api.get_ingestion_status(simulation.uuid.hex) except Exception as err: - click.echo() - raise click.ClickException( - f"Failed to check ingestion status: {err}" - ) from err + # Tolerate transient errors: the ingestion continues server-side + consecutive_failures += 1 + if consecutive_failures >= max_consecutive_failures: + click.echo() + raise click.ClickException( + f"Failed to check ingestion status " + f"{consecutive_failures} times in a row: {err}" + ) from err + if time.monotonic() >= deadline: + click.echo() + raise click.ClickException( + f"Timed out after {timeout:g}s waiting for ingestion to " + f"complete (last status: {last_status})" + ) from err + time.sleep(1) + continue + consecutive_failures = 0 if status != last_status: if last_status is not None: diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index b9b59b05..391ef566 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -26,7 +26,7 @@ Union, cast, ) -from urllib.parse import ParseResult, urlparse +from urllib.parse import urlparse import appdirs import click @@ -250,25 +250,40 @@ def _check_file_is_imas(file: Path) -> bool: # Not a readable NetCDF file; fall back to the directory heuristics pass - return imas_backend_for_directory(file.parent) is not None + try: + imas_backend_for_directory(file.parent) + except ValueError: + return False + return True def _find_partition_for_file( - file: Path, partitions: dict[str, str] + file: Path, partitions: Dict[str, str] ) -> Tuple[str, Path]: + # Match the partition with the longest root so that a catch-all mapping + # (e.g. "/") does not shadow more specific partitions. + best: Optional[Tuple[str, Path]] = None + best_depth = -1 for partition, path in partitions.items(): + root = Path(path) try: - return partition, file.relative_to(Path(path)) + relative = file.relative_to(root) except ValueError: - pass - raise click.ClickException( - f"File {file} is not located under any configured partition " - f"(configured partitions: {', '.join(partitions) or 'none'})" - ) + continue + depth = len(root.parts) + if depth > best_depth: + best = (partition, relative) + best_depth = depth + if best is None: + raise APIError( + f"File {file} is not located under any configured partition " + f"(configured partitions: {', '.join(partitions) or 'none'})" + ) + return best def _file_data_for_partition( - file: FileData, source: Path, partitions: dict[str, str] + file: FileData, source: Path, partitions: Dict[str, str] ) -> FileData: partition, partition_path = _find_partition_for_file(source, partitions) new_uri = SimDBUrl.build(scheme=partition, path=partition_path.as_posix()) @@ -285,24 +300,26 @@ def _file_data_for_partition( ) -def _expand_directories(files: Iterable[FileData], partitions: dict[str, str]): +def _expand_directories(files: Iterable[FileData], partitions: Dict[str, str]): new_file_list = [] for file in files: file_uri = SimDBUrl(file.uri) if file_uri.path is None: - raise ValueError("File has no associated path") + raise APIError(f"File URI has no path: {file.uri}") file_path = Path(file_uri.path) if file_uri.scheme == "imas": qs = dict(file_uri.query_params()) path = qs.get("path") if path is None: - raise ValueError("IMAS uri has not path set") + raise APIError(f"IMAS URI has no path set: {file.uri}") file_path = Path(path) if file_path.is_dir(): for sub_file in file_path.iterdir(): if sub_file.is_dir(): - raise ValueError("Nested directory found") + raise APIError( + f"Nested directory found in {file_path}: {sub_file.name}" + ) new_file_list.append( _file_data_for_partition(file, sub_file, partitions) ) @@ -428,7 +445,7 @@ def _load_cookies( headers = {"User-Agent": "it_script_basic"} cookies_file = f"{remote}-cookies.pkl" cookies_path = Path(appdirs.user_config_dir("simdb")) / cookies_file - parsed_url: ParseResult = urlparse(self._url) + parsed_url = urlparse(self._url) base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" cookies = None @@ -936,7 +953,7 @@ def _mark_imas_files(self, files: Iterable[FileData]) -> None: for file in files: file_uri = SimDBUrl(file.uri) if file_uri.path is None: - raise ValueError("File has no associated path") + raise APIError(f"File URI has no path: {file.uri}") partition = Path( self._config.get_string_option(f"partition.{file_uri.scheme}") @@ -949,7 +966,7 @@ def _mark_imas_files(self, files: Iterable[FileData]) -> None: def push_local_simulation(self, simulation: Simulation, add_watcher: bool = False): sim_data = simulation.to_model(recurse=True) - partitions = cast(dict[str, str], self._config.get_section("partition")) + partitions = cast(Dict[str, str], self._config.get_section("partition")) sim_data.inputs.root = _expand_directories(sim_data.inputs.root, partitions) sim_data.outputs.root = _expand_directories(sim_data.outputs.root, partitions) @@ -1010,7 +1027,7 @@ def push_simulation( sim_data = simulation.data(recurse=True) try: - sim_json: bytes = json.dumps( + sim_json = json.dumps( sim_data, cls=CustomEncoder, separators=(",", ":") ).encode("utf-8") sim_json_size = len(sim_json) diff --git a/src/simdb/imas/utils.py b/src/simdb/imas/utils.py index c4c868bd..016bc400 100644 --- a/src/simdb/imas/utils.py +++ b/src/simdb/imas/utils.py @@ -311,13 +311,13 @@ def _get_path(uri: SimDBUrl) -> Path: return path -def imas_backend_for_directory(directory: Path) -> Optional[str]: +def imas_backend_for_directory(directory: Path) -> str: """ Identify the IMAS backend of a directory by inspecting its contents. @param directory: a directory that may contain an IMAS dataset - @return: the backend name ("ascii", "hdf5" or "mdsplus"), or None if no IMAS - dataset is detected + @return: the backend name ("ascii", "hdf5" or "mdsplus") + @raise ValueError: if no IMAS dataset is detected """ children = list(directory.iterdir()) diff --git a/src/simdb/remote/models.py b/src/simdb/remote/models.py index 7ea4d09e..ae9565aa 100644 --- a/src/simdb/remote/models.py +++ b/src/simdb/remote/models.py @@ -386,12 +386,6 @@ class SimulationPostResponse(BaseModel): """Validation result.""" -class SimulationPostResponse3(BaseModel): - """Response from creating a simulation.""" - - job_id: HexUUID - - class SimulationListItem(BaseModel): """Summary of a simulation for list views.""" From ada8876dda559974757c2d7ae1c8546ed4d85108 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 13:32:38 +0200 Subject: [PATCH 09/31] Remove duplicate netcdf4 dependency --- pyproject.toml | 3 +-- uv.lock | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 82af9bf9..44a29a20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dependencies = [ "distro>=1.8.0", "email-validator>=1.1", "imas-python>=2.0.1", - "netCDF4>=1.5", + "netCDF4>=1.7.2", "numpy>=1.14", "pydantic>=2.10.6", "python-dateutil>=2.6", @@ -52,7 +52,6 @@ dependencies = [ "sqlalchemy>=1.2.12,<2.0", "alembic~=1.13", "rich>=14.3.3", - "netcdf4>=1.7.2", ] [project.optional-dependencies] diff --git a/uv.lock b/uv.lock index c371ef67..532738e0 100644 --- a/uv.lock +++ b/uv.lock @@ -3146,7 +3146,6 @@ requires-dist = [ { name = "imas-validator", marker = "extra == 'imas-validator'", specifier = ">=1.0.0" }, { name = "myst-parser", marker = "extra == 'build-docs'", specifier = ">=0.18.0" }, { name = "nbsphinx", marker = "extra == 'build-docs'", specifier = ">=0.8.0" }, - { name = "netcdf4", specifier = ">=1.5" }, { name = "netcdf4", specifier = ">=1.7.2" }, { name = "numpy", specifier = ">=1.14" }, { name = "plotext", specifier = "==5.3.2" }, From e5ae35995f39a55fde4c3892ddcb620f783eac08 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 13:32:55 +0200 Subject: [PATCH 10/31] Fix revision docstring in ingestion status migration --- alembic/versions/b2c52ee8ff12_add_ingestion_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alembic/versions/b2c52ee8ff12_add_ingestion_status.py b/alembic/versions/b2c52ee8ff12_add_ingestion_status.py index 05ebcd98..b9861c90 100644 --- a/alembic/versions/b2c52ee8ff12_add_ingestion_status.py +++ b/alembic/versions/b2c52ee8ff12_add_ingestion_status.py @@ -1,7 +1,7 @@ """Add ingestion status Revision ID: b2c52ee8ff12 -Revises: 9e9a4a7cd639 +Revises: 28bee3aa2429 Create Date: 2026-05-11 16:16:03.768893 """ From ea20477d13f3af3d0eb8901ceac16f1ca5a89741 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 13:33:36 +0200 Subject: [PATCH 11/31] Remove redundant _mark_imas_files pass from push_local File types are already assigned at manifest time and preserved by _file_data_for_partition, so this pass was a no-op for correctly ingested simulations. Worse, the directory heuristic could promote a plain FILE entry living next to IMAS data to IMAS, after which the server rewrites its URI. It also opened every .nc file and scanned every parent directory on a shared filesystem. --- src/simdb/cli/remote_api.py | 36 +----------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 391ef566..cb12fb29 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -31,14 +31,13 @@ import appdirs import click import requests -from netCDF4 import Dataset from requests.auth import AuthBase from semantic_version import Version from simdb.checksum import calculate_checksum from simdb.config import Config from simdb.database.models import Simulation -from simdb.imas.utils import SimDBUrl, imas_backend_for_directory, imas_files +from simdb.imas.utils import SimDBUrl, imas_files from simdb.json import CustomDecoder, CustomEncoder from simdb.remote import CLIENT_API_VERSIONS, APIConstants from simdb.remote.models import FileData, SimulationPostData @@ -239,24 +238,6 @@ def _get_paths(file: "File") -> Iterable[Path]: return imas_files(file.uri) -def _check_file_is_imas(file: Path) -> bool: - # NetCDF is identified by the IMAS "Conventions" attribute - if file.suffix == ".nc": - try: - with Dataset(file, "r") as ds: - if getattr(ds, "Conventions", None) == "IMAS": - return True - except OSError: - # Not a readable NetCDF file; fall back to the directory heuristics - pass - - try: - imas_backend_for_directory(file.parent) - except ValueError: - return False - return True - - def _find_partition_for_file( file: Path, partitions: Dict[str, str] ) -> Tuple[str, Path]: @@ -949,18 +930,6 @@ def _send_chunk( ] self.post("files", data={}, files=files) - def _mark_imas_files(self, files: Iterable[FileData]) -> None: - for file in files: - file_uri = SimDBUrl(file.uri) - if file_uri.path is None: - raise APIError(f"File URI has no path: {file.uri}") - - partition = Path( - self._config.get_string_option(f"partition.{file_uri.scheme}") - ) - if _check_file_is_imas(partition / Path(file_uri.path)): - file.type = "IMAS" - @versioned_method("v1.3") @try_request def push_local_simulation(self, simulation: Simulation, add_watcher: bool = False): @@ -970,9 +939,6 @@ def push_local_simulation(self, simulation: Simulation, add_watcher: bool = Fals sim_data.inputs.root = _expand_directories(sim_data.inputs.root, partitions) sim_data.outputs.root = _expand_directories(sim_data.outputs.root, partitions) - self._mark_imas_files(sim_data.inputs.root) - self._mark_imas_files(sim_data.outputs.root) - uploaded_by = simulation.meta_dict().get("uploaded_by") headers = {"Content-type": "application/json", "User-Agent": "it_script_basic"} From cd097d5fe7528e989435c5b8d53623b1eeb4502d Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 13:34:46 +0200 Subject: [PATCH 12/31] Use request helpers in push_local_simulation and get_ingestion_status Hand-rolled requests.post/requests.get bypassed the auth gating on self._server_auth, the gzip compression for large simulations payloads, and the negotiated self._api_url. --- src/simdb/cli/remote_api.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index cb12fb29..7baaea4f 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -941,33 +941,17 @@ def push_local_simulation(self, simulation: Simulation, add_watcher: bool = Fals uploaded_by = simulation.meta_dict().get("uploaded_by") - headers = {"Content-type": "application/json", "User-Agent": "it_script_basic"} post_data = SimulationPostData( simulation=sim_data, add_watcher=add_watcher, uploaded_by=str(uploaded_by) if uploaded_by is not None else None, - ).model_dump_json() - res = requests.post( - f"{self._url}/v1.3/simulations", - data=post_data, - headers=headers, - auth=self._get_auth(), - cookies=self._cookies, ) - check_return(res) + self.post("simulations", data=post_data.model_dump(mode="json")) @versioned_method("v1.3") @try_request def get_ingestion_status(self, sim_id: str) -> str: - headers = {"User-Agent": "it_script_basic"} - auth = self._get_auth() if self._server_auth != "None" else None - res = requests.get( - f"{self._url}/v1.3/simulation/status/{sim_id}", - headers=headers, - auth=auth, - cookies=self._cookies, - ) - check_return(res) + res = self.get(f"simulation/status/{sim_id}") return res.json()["status"] @versioned_method("v1.2", "v1.3") From 52329029eb2e23b594eda401f78e2df76a652e9f Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 14:50:24 +0200 Subject: [PATCH 13/31] feat: add a vendored resumable HTTP upload client implementing the IETF resumable-upload draft --- src/simdb/cli/resumable_upload.py | 257 ++++++++++++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 src/simdb/cli/resumable_upload.py diff --git a/src/simdb/cli/resumable_upload.py b/src/simdb/cli/resumable_upload.py new file mode 100644 index 00000000..a08513da --- /dev/null +++ b/src/simdb/cli/resumable_upload.py @@ -0,0 +1,257 @@ +"""Client for the IETF "Resumable Uploads for HTTP" protocol. + +This is a small, dependency-free (uses ``requests``, already a dependency) +implementation of draft-ietf-httpbis-resumable-upload-11 (interop version 8) - +the same protocol implemented by https://github.com/Yannicked/pyrufh. + +The single public entry point :func:`resumable_upload` uploads a local file to a +server endpoint that speaks the same protocol. The upload resource is identified +by the request URL itself: an interrupted upload can be resumed by simply +re-invoking :func:`resumable_upload` with the same arguments - the client asks +the server (via ``HEAD``) how many bytes it already has and continues from there. +""" + +import logging +from pathlib import Path +from typing import Callable, Mapping, Optional, Tuple, Union + +import requests +from requests.auth import AuthBase + +logger = logging.getLogger(__name__) + +#: The draft interop version this client implements. +INTEROP_VERSION = "8" +INTEROP_HEADER = "Upload-Draft-Interop-Version" +#: Content type used for the body of append (``PATCH``) requests. +PARTIAL_UPLOAD_CONTENT_TYPE = "application/partial-upload" +#: Default size of a single ``PATCH`` chunk (kept below the 10 MB request cap +#: enforced on the ITER network, see ``RemoteAPI.push_simulation``). +DEFAULT_CHUNK_SIZE = 8 * 1024 * 1024 + +#: Maximum number of consecutive failures (connection errors / offset +#: conflicts) tolerated before giving up. +_MAX_RETRIES = 5 + + +class ResumableUploadError(RuntimeError): + """Raised when a resumable upload cannot be completed.""" + + +def _bool_field(value: bool) -> str: + """Render a boolean as an HTTP structured-field item (``?1``/``?0``).""" + return "?1" if value else "?0" + + +def _parse_bool_field(value: Optional[str]) -> Optional[bool]: + if value is None: + return None + value = value.strip() + if value == "?1": + return True + if value == "?0": + return False + return None + + +def _header_int(resp: "requests.Response", name: str) -> Optional[int]: + raw = resp.headers.get(name) + if raw is None: + return None + try: + return int(raw.strip()) + except ValueError: + return None + + +def _base_headers(extra: Optional[Mapping[str, str]]) -> dict: + headers = {INTEROP_HEADER: INTEROP_VERSION} + if extra: + headers.update(extra) + return headers + + +def _parse_upload_limit(value: Optional[str]) -> dict: + """Parse an ``Upload-Limit`` structured-field dictionary into a dict. + + Only the integer-valued members this client cares about are kept (e.g. + ``max-append-size``). Unparseable members are ignored. + """ + limits: dict = {} + if not value: + return limits + for member in value.split(","): + member = member.strip() + if "=" not in member: + continue + key, _, raw = member.partition("=") + try: + limits[key.strip()] = int(raw.strip()) + except ValueError: + continue + return limits + + +def _clamp_chunk_size(chunk_size: int, limits: dict) -> int: + """Reduce ``chunk_size`` to the server-advertised ``max-append-size``.""" + max_append = limits.get("max-append-size") + if max_append and max_append > 0: + return min(chunk_size, max_append) + return chunk_size + + +def resumable_upload( + url: str, + path: Union[str, Path], + *, + auth: Optional[Union[AuthBase, Tuple[str, str]]] = None, + cookies: Optional[Mapping[str, str]] = None, + headers: Optional[Mapping[str, str]] = None, + chunk_size: int = DEFAULT_CHUNK_SIZE, + progress: Optional[Callable[[int], None]] = None, +) -> None: + """Upload ``path`` to ``url`` using the resumable upload protocol. + + @param url: the upload resource URL. The server is expected to treat this + URL itself as the upload resource (it is both the creation + target and the resource that is appended to / queried). + @param path: the local file to upload. + @param auth: authentication passed through to ``requests``. + @param cookies: cookies passed through to ``requests`` (e.g. firewall). + @param headers: extra headers to send with every request. + @param chunk_size: number of bytes sent per ``PATCH`` request. + @param progress: optional callback invoked with the absolute number of bytes + confirmed by the server, after resuming and after each + chunk. Useful for driving a progress bar. + """ + path = Path(path) + total = path.stat().st_size + + offset, complete, limits = _resume_or_create(url, total, auth, cookies, headers) + if complete: + if progress: + progress(total) + return + + # Reflect any bytes the server already holds (resumed upload). + if progress: + progress(offset) + + # The server advertises its append-size limit via Upload-Limit; never send a + # chunk larger than it will accept. + chunk_size = _clamp_chunk_size(chunk_size, limits) + + with path.open("rb") as f: + _send_chunks( + url, f, offset, total, chunk_size, auth, cookies, headers, progress + ) + + +def _resume_or_create( + url: str, + total: int, + auth, + cookies, + headers, +) -> Tuple[int, bool, dict]: + """Return ``(offset, complete, limits)`` for the upload resource at ``url``. + + Probes the resource with ``HEAD``; if it does not yet exist the resource is + created with an empty body (``Upload-Complete: ?0``). ``limits`` is the + parsed ``Upload-Limit`` dictionary advertised by the server. + """ + resp = requests.head( + url, headers=_base_headers(headers), auth=auth, cookies=cookies + ) + if resp.status_code in (200, 204): + limits = _parse_upload_limit(resp.headers.get("Upload-Limit")) + if _parse_bool_field(resp.headers.get("Upload-Complete")): + return total, True, limits + return _header_int(resp, "Upload-Offset") or 0, False, limits + + create_headers = _base_headers(headers) + create_headers["Upload-Complete"] = _bool_field(False) + create_headers["Upload-Length"] = str(total) + resp = requests.post( + url, data=b"", headers=create_headers, auth=auth, cookies=cookies + ) + if resp.status_code not in (200, 201, 204): + raise ResumableUploadError( + f"Failed to create upload resource ({resp.status_code}): {resp.text}" + ) + limits = _parse_upload_limit(resp.headers.get("Upload-Limit")) + return _header_int(resp, "Upload-Offset") or 0, False, limits + + +def _send_chunks( + url: str, + f, + offset: int, + total: int, + chunk_size: int, + auth, + cookies, + headers, + progress: Optional[Callable[[int], None]] = None, +) -> None: + attempts = 0 + while True: + f.seek(offset) + chunk = f.read(chunk_size) + complete = (offset + len(chunk)) >= total + + patch_headers = _base_headers(headers) + patch_headers["Content-Type"] = PARTIAL_UPLOAD_CONTENT_TYPE + patch_headers["Upload-Offset"] = str(offset) + patch_headers["Upload-Complete"] = _bool_field(complete) + + try: + resp = requests.patch( + url, data=chunk, headers=patch_headers, auth=auth, cookies=cookies + ) + except (requests.ConnectionError, requests.Timeout) as err: + attempts += 1 + if attempts > _MAX_RETRIES: + raise ResumableUploadError( + f"Upload failed after {_MAX_RETRIES} retries: {err}" + ) from err + logger.warning("Upload chunk failed (%s), resuming from server offset", err) + offset = _query_offset(url, auth, cookies, headers) + continue + + if resp.status_code == 409: + # Offset mismatch: resynchronise to the server-reported offset. + server_offset = _header_int(resp, "Upload-Offset") + if server_offset is None: + raise ResumableUploadError( + "Server reported an offset conflict without an Upload-Offset header" + ) + attempts += 1 + if attempts > _MAX_RETRIES: + raise ResumableUploadError("Too many offset conflicts during upload") + offset = server_offset + continue + + if resp.status_code not in (200, 201, 204): + raise ResumableUploadError( + f"Unexpected status {resp.status_code} while appending: {resp.text}" + ) + + attempts = 0 + server_offset = _header_int(resp, "Upload-Offset") + offset = server_offset if server_offset is not None else offset + len(chunk) + + if progress: + progress(offset) + + if complete: + return + + +def _query_offset(url: str, auth, cookies, headers) -> int: + resp = requests.head( + url, headers=_base_headers(headers), auth=auth, cookies=cookies + ) + if resp.status_code in (200, 204): + return _header_int(resp, "Upload-Offset") or 0 + return 0 From 477fac91425f76d49497fbb1d9ad4e4e723e8f6a Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 14:50:31 +0200 Subject: [PATCH 14/31] feat: add a server resumable upload endpoint that stages files into the http partition --- src/simdb/remote/apis/v1_3/__init__.py | 3 +- src/simdb/remote/apis/v1_3/upload.py | 184 +++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 src/simdb/remote/apis/v1_3/upload.py diff --git a/src/simdb/remote/apis/v1_3/__init__.py b/src/simdb/remote/apis/v1_3/__init__.py index a0bcf15d..67a54293 100644 --- a/src/simdb/remote/apis/v1_3/__init__.py +++ b/src/simdb/remote/apis/v1_3/__init__.py @@ -9,6 +9,7 @@ from simdb.remote.core.auth import TokenAuthenticator from .simulation_data import api as data_ns +from .upload import api as upload_ns api = Api( title="SimDB REST API", @@ -28,7 +29,7 @@ doc="/docs", ) -namespaces = [metadata_ns, watcher_ns, file_ns, sim_ns, data_ns] +namespaces = [metadata_ns, watcher_ns, file_ns, sim_ns, data_ns, upload_ns] api.route("/staging_dir", defaults={"sim_hex": None})(StagingDirectory) api.route("/staging_dir/")(StagingDirectory) diff --git a/src/simdb/remote/apis/v1_3/upload.py b/src/simdb/remote/apis/v1_3/upload.py new file mode 100644 index 00000000..702c19d1 --- /dev/null +++ b/src/simdb/remote/apis/v1_3/upload.py @@ -0,0 +1,184 @@ +"""Server side of the IETF "Resumable Uploads for HTTP" protocol. + +Implements draft-ietf-httpbis-resumable-upload-11 (interop version 8), the +counterpart to :mod:`simdb.cli.resumable_upload`. Uploaded bytes are staged into +the ``http`` partition (config ``partition.http``): a client uploading to +``/v1.3/upload//`` results in the file being written to +``//``. The simulation is then pushed +(metadata only) referencing those files with ``http:////`` +URIs, which the existing ingestion pipeline resolves via the ``http`` partition. + +Upload state lives on disk so it survives across worker processes: in-progress +bytes are written to ``.partial`` and atomically renamed to ```` +when the upload completes. The current offset is simply the size of that file. +""" + +import contextlib +from pathlib import Path +from typing import Optional, Tuple + +from flask import Response, request +from flask_restx import Namespace, Resource +from werkzeug.exceptions import Forbidden + +from simdb.remote.core.auth import User, requires_auth +from simdb.remote.core.typing import current_app + +api = Namespace("upload", path="/") + +#: The draft interop version this server implements. +INTEROP_VERSION = "8" +INTEROP_HEADER = "Upload-Draft-Interop-Version" +PARTIAL_SUFFIX = ".partial" +#: Default maximum size of a single append (``PATCH``) body, advertised to +#: clients via the ``Upload-Limit`` header. Overridable with the +#: ``server.max_append_size`` config option. +DEFAULT_MAX_APPEND_SIZE = 8 * 1024 * 1024 + + +def _max_append_size() -> int: + value = current_app.simdb_config.get_option( + "server.max_append_size", default=DEFAULT_MAX_APPEND_SIZE + ) + try: + return int(value) + except (TypeError, ValueError): + return DEFAULT_MAX_APPEND_SIZE + + +def _bool_field(value: bool) -> str: + return "?1" if value else "?0" + + +def _parse_bool_field(value: Optional[str]) -> Optional[bool]: + if value is None: + return None + value = value.strip() + if value == "?1": + return True + if value == "?0": + return False + return None + + +def _partition_base() -> Path: + base = current_app.simdb_config.get_string_option("partition.http", default=None) + if not base: + raise ValueError("Partition 'http' is not configured on the server") + return Path(base).resolve() + + +def _resolve_target(target: str) -> Tuple[Path, Path]: + """Resolve ``target`` to its ``(final, partial)`` paths within ``partition.http``. + + Raises ``ValueError`` if the resolved path would escape the partition. + """ + base = _partition_base() + final = (base / target).resolve() + if not final.is_relative_to(base): + raise Forbidden("Access denied.") + partial = final.parent / (final.name + PARTIAL_SUFFIX) + return final, partial + + +def _state(final: Path, partial: Path) -> Tuple[int, bool, bool]: + """Return ``(offset, complete, exists)`` for the upload resource.""" + if partial.exists(): + return partial.stat().st_size, False, True + if final.exists(): + return final.stat().st_size, True, True + return 0, False, False + + +def _headers(offset: int, complete: bool) -> dict: + return { + INTEROP_HEADER: INTEROP_VERSION, + "Upload-Offset": str(offset), + "Upload-Complete": _bool_field(complete), + # Advertise the server's append-size limit (structured-field dictionary) + # so the client sizes its chunks accordingly. + "Upload-Limit": f"max-append-size={_max_append_size()}", + "Cache-Control": "no-store", + } + + +@api.route("/upload/") +class ResumableUpload(Resource): + """A single resumable upload resource staged into the ``http`` partition.""" + + @requires_auth() + def post(self, target: str, user: User) -> Response: + """Create (or reset) the upload resource and optionally write data.""" + final, partial = _resolve_target(target) + partial.parent.mkdir(parents=True, exist_ok=True) + + data = request.get_data() or b"" + if len(data) > _max_append_size(): + return Response(status=413, headers=_headers(0, False)) + with partial.open("wb") as f: + f.write(data) + offset = len(data) + + complete = _parse_bool_field(request.headers.get("Upload-Complete")) or False + if complete: + partial.replace(final) + + headers = _headers(offset, complete) + headers["Location"] = request.url + return Response(status=201, headers=headers) + + @requires_auth() + def head(self, target: str, user: User) -> Response: + """Report the current offset / completeness of the upload resource.""" + final, partial = _resolve_target(target) + offset, complete, exists = _state(final, partial) + if not exists: + return Response(status=404, headers={INTEROP_HEADER: INTEROP_VERSION}) + return Response(status=204, headers=_headers(offset, complete)) + + @requires_auth() + def patch(self, target: str, user: User) -> Response: + """Append data to the upload resource at the given ``Upload-Offset``.""" + final, partial = _resolve_target(target) + offset, complete, _exists = _state(final, partial) + + # Appending to an already-completed upload is a no-op when the client is + # simply confirming completion at the final offset. + if complete: + return Response(status=200, headers=_headers(offset, True)) + + try: + requested_offset = int(request.headers.get("Upload-Offset", "")) + except ValueError: + return Response(status=400, headers={INTEROP_HEADER: INTEROP_VERSION}) + + if requested_offset != offset: + # Offset mismatch - tell the client our current offset so it resyncs. + return Response(status=409, headers=_headers(offset, False)) + + data = request.get_data() or b"" + if len(data) > _max_append_size(): + return Response(status=413, headers=_headers(offset, False)) + + partial.parent.mkdir(parents=True, exist_ok=True) + with partial.open("ab") as f: + f.write(data) + offset += len(data) + + request_complete = ( + _parse_bool_field(request.headers.get("Upload-Complete")) or False + ) + if request_complete: + partial.replace(final) + return Response(status=200, headers=_headers(offset, True)) + + return Response(status=204, headers=_headers(offset, False)) + + @requires_auth() + def delete(self, target: str, user: User) -> Response: + """Cancel the upload and remove any staged data.""" + final, partial = _resolve_target(target) + for path in (partial, final): + with contextlib.suppress(FileNotFoundError): + path.unlink() + return Response(status=204, headers={INTEROP_HEADER: INTEROP_VERSION}) From f499773b680350821b0c50c88b6c72a21a03a171 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 14:50:37 +0200 Subject: [PATCH 15/31] feat: resolve http-partition URIs during ingestion and remove staged files after copy --- src/simdb/remote/apis/v1_3/simulations.py | 12 ++++++++++- src/simdb/workers/tasks.py | 26 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/simdb/remote/apis/v1_3/simulations.py b/src/simdb/remote/apis/v1_3/simulations.py index 7bb0fcb6..8e7da0de 100644 --- a/src/simdb/remote/apis/v1_3/simulations.py +++ b/src/simdb/remote/apis/v1_3/simulations.py @@ -7,6 +7,7 @@ from simdb.database.models import simulation as models_sim from simdb.database.models import watcher as models_watcher from simdb.enums import IngestionStatus +from simdb.imas.utils import SimDBUrl from simdb.remote.apis.v1_2.simulations import ( Simulation, SimulationMeta, @@ -30,6 +31,7 @@ SimulationStatusResponse, ) from simdb.workers.tasks import ( + cleanup_http_staging_task, complete_ingestion_task, copy_files_task, ) @@ -116,8 +118,16 @@ def post( # The complete job will set simulation.ingestion_status = Completed complete = complete_ingestion_task.si(simulation.uuid) + chain = copy_files | complete + + # Files uploaded over HTTP are staged in the ``http`` partition; once + # copied into the upload folder, remove those staged duplicates. + all_files = [*body.simulation.inputs.root, *body.simulation.outputs.root] + if any(SimDBUrl(f.uri).scheme == "http" for f in all_files): + chain = chain | cleanup_http_staging_task.si(simulation.uuid) + try: - _ = (copy_files | complete).apply_async() + _ = chain.apply_async() except Exception as err: simulation.ingestion_status = IngestionStatus.COPY_FAILED current_app.db.session.commit() diff --git a/src/simdb/workers/tasks.py b/src/simdb/workers/tasks.py index 77ed2144..01adf8fc 100644 --- a/src/simdb/workers/tasks.py +++ b/src/simdb/workers/tasks.py @@ -82,6 +82,10 @@ def _resolve_uri_to_path(uri: SimDBUrl, config: Config) -> Path: raise ValueError("Path not given") path = Path(path) path = path.relative_to(path.anchor) + # Standard schemes (e.g. ``http``) parse the first path segment as the URL + # authority. Fold it back in so the full relative path is reconstructed. + if uri.host: + path = Path(uri.host) / path target = (partition_path / path).resolve() if not target.is_relative_to(partition_path): raise ValueError("Access denied.") @@ -282,3 +286,25 @@ def fail_stale_ingestions_task() -> dict: return {"failed": failed} finally: database.close() + + +@celery_app.task +def cleanup_http_staging_task(simulation_uuid: UUID): + """Remove a simulation's staged files from the ``http`` partition. + + HTTP-uploaded files are staged into the ``http`` partition and then copied + into the simulation's upload folder by :func:`copy_files_task`. Once copied + they are duplicates, so the staging directory is removed here. + """ + config = Config() + config.load() + + partition_path_str = config.get_string_option("partition.http", default=None) + if not partition_path_str: + return + + partition_path = Path(partition_path_str).resolve() + staging_dir = (partition_path / simulation_uuid.hex).resolve() + # Guard against escaping the partition before removing anything. + if staging_dir.is_relative_to(partition_path) and staging_dir != partition_path: + shutil.rmtree(staging_dir, ignore_errors=True) From 3d2910fdd2860779ab2492371987b526ffb0d03b Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 14:50:43 +0200 Subject: [PATCH 16/31] feat: add the 'simdb simulation push_http' command uploading files with overall and per-file progress bars --- src/simdb/cli/commands/simulation.py | 78 +++++++++++++ src/simdb/cli/remote_api.py | 165 ++++++++++++++++++++++++++- 2 files changed, 242 insertions(+), 1 deletion(-) diff --git a/src/simdb/cli/commands/simulation.py b/src/simdb/cli/commands/simulation.py index 11f57358..cb1992f8 100644 --- a/src/simdb/cli/commands/simulation.py +++ b/src/simdb/cli/commands/simulation.py @@ -324,6 +324,84 @@ def simulation_push_local( raise click.ClickException(f"Simulation ingestion failed with status: {status}") +@simulation.command("push_http", cls=n_required_args_adaptor(1)) +@pass_config +@click.argument("remote", required=False) +@click.argument("sim_id") +@click.option("--username", help="Username used to authenticate with the remote.") +@click.option("--password", help="Password used to authenticate with the remote.") +@click.option("--replaces", help="SIM_ID of simulation to deprecate and replace.") +@click.option( + "--add-watcher", + is_flag=True, + help="Add the current user as a watcher of the simulation.", +) +def simulation_push_http( + config: Config, + remote: Optional[str], + sim_id: str, + username: Optional[str], + password: Optional[str], + replaces: Optional[str], + add_watcher: bool, +): + """Push the simulation with the given SIM_ID to the REMOTE over resumable HTTP. + + Unlike push_local, this does not require a filesystem shared with the server: + the file bytes are uploaded over HTTP using a resumable protocol and staged + into the server's 'http' partition. An interrupted push can be resumed by + re-running the command. + """ + + api = RemoteAPI(remote, username, password, config) + db = get_local_db(config) + + simulation = db.get_simulation(sim_id) + if simulation is None: + raise click.ClickException(f"Failed to find simulation: {sim_id}") + + if replaces: + simulation.set_meta("replaces", replaces) + + schemas = api.get_validation_schemas() + try: + for schema in schemas: + Validator(schema).validate(simulation) + except ValidationError as err: + raise click.ClickException(f"Simulation does not validate: {err}") from err + + api.push_http_simulation(simulation) + + click.echo("Waiting for ingestion to complete...", nl=False) + last_status = None + while True: + try: + status = api.get_ingestion_status(simulation.uuid.hex) + except Exception as err: + click.echo() + raise click.ClickException( + f"Failed to check ingestion status: {err}" + ) from err + + if status != last_status: + if last_status is not None: + click.echo(f" -> {status}", nl=False) + else: + click.echo(f" {status}", nl=False) + last_status = status + + if status in ("completed", "copy_failed", "validation_failed"): + break + + time.sleep(1) + + click.echo() + if status == "completed": + click.echo(f"Successfully pushed simulation {simulation.uuid}") + else: + raise click.ClickException(f"Simulation ingestion failed with status: {status}") + + @simulation.command("push", cls=n_required_args_adaptor(1)) @pass_config @click.argument("remote", required=False) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 7baaea4f..8aa1a4f1 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -26,15 +26,24 @@ Union, cast, ) -from urllib.parse import urlparse +from urllib.parse import ParseResult, quote, urlparse import appdirs import click import requests from requests.auth import AuthBase +from rich.progress import ( + BarColumn, + DownloadColumn, + Progress, + TextColumn, + TimeRemainingColumn, + TransferSpeedColumn, +) from semantic_version import Version from simdb.checksum import calculate_checksum +from simdb.cli.resumable_upload import resumable_upload from simdb.config import Config from simdb.database.models import Simulation from simdb.imas.utils import SimDBUrl, imas_files @@ -309,6 +318,76 @@ def _expand_directories(files: Iterable[FileData], partitions: Dict[str, str]): return new_file_list +def _expand_directories_http( + files: Iterable[FileData], sim_uuid: uuid.UUID, partitions: dict[str, str] +) -> List[Tuple[FileData, Path, str]]: + """Expand directories / IMAS data into individual files for HTTP upload. + + Returns ``(file_data, local_source_path, target)`` triples. Each file keeps + the same partition-relative layout that :func:`_expand_directories` produces + for ``push_local`` - so structure handling (IMAS directories stay grouped, + standalone files stay flat) is identical to local push. The layout is then + namespaced under ``//`` and assigned an ``http://`` URI so + the server stages it into the ``http`` partition; the server's existing copy + step strips the common root exactly as it does for local push. + """ + result: List[Tuple[FileData, Path, str]] = [] + for file in files: + file_uri = SimDBUrl(file.uri) + if file_uri.path is None: + raise ValueError("File has no associated path") + file_path = Path(file_uri.path) + if file_uri.scheme == "imas": + qs = dict(file_uri.query_params()) + path = qs.get("path") + if path is None: + raise ValueError("IMAS uri has not path set") + file_path = Path(path) + + if file_path.is_dir(): + for sub_file in file_path.iterdir(): + if sub_file.is_dir(): + raise ValueError("Nested directory found") + result.append(_make_http_entry(file, sub_file, sim_uuid, partitions)) + else: + result.append(_make_http_entry(file, file_path, sim_uuid, partitions)) + return result + + +def _make_http_entry( + template: FileData, + local_path: Path, + sim_uuid: uuid.UUID, + partitions: dict[str, str], +) -> Tuple[FileData, Path, str]: + """Build the HTTP upload entry for a single local file. + + The relative path is taken from :func:`_find_partition_for_file` (the same + mapping ``push_local`` uses) and namespaced under ``//`` so + uploads from different partitions never collide on the server. + """ + scheme, rel = _find_partition_for_file(local_path, partitions) + rel_posix = rel.as_posix().lstrip("/") + target = f"{sim_uuid.hex}/{scheme}/{rel_posix}" + new_uri = SimDBUrl.build(scheme="http", path=target, host="") + file_type = "IMAS" if _check_file_is_imas(local_path) else template.type + return ( + FileData( + type=file_type, + uri=new_uri.encoded_string(), + checksum=calculate_checksum(local_path), + datetime=template.datetime, + usage=template.usage, + purpose=template.purpose, + sensitivity=template.sensitivity, + access=template.access, + embargo=template.embargo, + ), + local_path, + target, + ) + + class RemoteAPI: """ Class to represent connection to remote API. @@ -948,6 +1027,90 @@ def push_local_simulation(self, simulation: Simulation, add_watcher: bool = Fals ) self.post("simulations", data=post_data.model_dump(mode="json")) + def _upload_files( + self, + files: List[Tuple[FileData, Path, str]], + upload_headers: Dict[str, str], + ): + """Upload the expanded files over resumable HTTP, showing two progress + bars: an overall bar across all bytes and a sub-bar for the current file. + """ + total_bytes = sum(local_path.stat().st_size for _, local_path, _ in files) + + with Progress( + TextColumn("[bold blue]{task.description}"), + BarColumn(), + DownloadColumn(), + TransferSpeedColumn(), + TimeRemainingColumn(), + ) as progress: + overall = progress.add_task("Overall", total=total_bytes) + file_task = progress.add_task("", total=0) + uploaded = 0 + for _file_data, local_path, target in files: + size = local_path.stat().st_size + progress.reset( + file_task, total=size, description=f" {local_path.name}" + ) + url = f"{self._url}/v1.3/upload/{quote(target)}" + + def _on_progress(completed: int, _base: int = uploaded) -> None: + progress.update(file_task, completed=completed) + progress.update(overall, completed=_base + completed) + + resumable_upload( + url, + local_path, + auth=self._get_auth(), + cookies=self._cookies, + headers=upload_headers, + progress=_on_progress, + ) + uploaded += size + progress.update(file_task, completed=size) + progress.update(overall, completed=uploaded) + + @try_request + def push_http_simulation(self, simulation: Simulation): + """Push a simulation by uploading its files over resumable HTTP. + + Unlike :meth:`push_local_simulation` (which requires a filesystem shared + with the server), this uploads the file bytes to the server's ``http`` + partition using a resumable protocol, then pushes the metadata. + """ + sim_data = simulation.to_model(recurse=True) + + partitions = cast(dict[str, str], self._config.get_section("partition")) + inputs = _expand_directories_http( + sim_data.inputs.root, simulation.uuid, partitions + ) + outputs = _expand_directories_http( + sim_data.outputs.root, simulation.uuid, partitions + ) + + files = list(itertools.chain(inputs, outputs)) + upload_headers = {"User-Agent": "it_script_basic"} + if files: + self._upload_files(files, upload_headers) + + sim_data.inputs.root = [file_data for file_data, _, _ in inputs] + sim_data.outputs.root = [file_data for file_data, _, _ in outputs] + + uploaded_by = str(simulation.meta_dict().get("uploaded_by", None)) + + headers = {"Content-type": "application/json", "User-Agent": "it_script_basic"} + post_data = SimulationPostData( + simulation=sim_data, add_watcher=False, uploaded_by=uploaded_by + ).model_dump_json() + res = requests.post( + f"{self._url}/v1.3/simulations", + data=post_data, + headers=headers, + auth=self._get_auth(), + cookies=self._cookies, + ) + check_return(res) + @versioned_method("v1.3") @try_request def get_ingestion_status(self, sim_id: str) -> str: From 5998b22de43120b444cfa3270906d0d3463649db Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 14:50:50 +0200 Subject: [PATCH 17/31] chore: configure the http partition and mount its staging directory for the server --- config/simdb.cfg | 1 + docker-compose.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/config/simdb.cfg b/config/simdb.cfg index 4a6816f5..5315fab0 100644 --- a/config/simdb.cfg +++ b/config/simdb.cfg @@ -34,3 +34,4 @@ result_backend = redis://redis:6379/0 [partition] data = /data/simdb/partition +http = /data/simdb/http \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index eff71ff3..f9b57500 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,7 @@ services: - ./validation:/app/validation:ro - ./config:/app/config:ro - ./tmp/partition_data:/data/simdb/partition:ro + - ./tmp/http:/data/simdb/http - ./upload_folder:/data/simdb/simulations depends_on: redis: @@ -38,6 +39,7 @@ services: volumes: - ./config:/app/config:ro - ./tmp/partition_data:/data/simdb/partition:ro + - ./tmp/http:/data/simdb/http - ./upload_folder:/data/simdb/simulations depends_on: redis: From e67a38c896c54a2c8da3a195491c74a9293a4cb3 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 14:50:57 +0200 Subject: [PATCH 18/31] test: cover the resumable upload client, the server endpoint, and http file ingestion --- tests/cli/test_push_http.py | 138 +++++++++ .../remote/api/v1.3/test_resumable_upload.py | 272 ++++++++++++++++++ tests/workers/test_tasks.py | 88 ++++++ 3 files changed, 498 insertions(+) create mode 100644 tests/cli/test_push_http.py create mode 100644 tests/remote/api/v1.3/test_resumable_upload.py diff --git a/tests/cli/test_push_http.py b/tests/cli/test_push_http.py new file mode 100644 index 00000000..c480a6b6 --- /dev/null +++ b/tests/cli/test_push_http.py @@ -0,0 +1,138 @@ +"""Tests for the HTTP push client helpers and CLI command.""" + +import uuid +from datetime import datetime, timezone +from unittest import mock + +from click.testing import CliRunner +from utils import config_test_file + +from simdb.cli.remote_api import _expand_directories_http +from simdb.cli.simdb import cli +from simdb.imas.utils import SimDBUrl +from simdb.remote.models import FileData + + +def _file_data(path) -> FileData: + return FileData( + type="FILE", + uri=SimDBUrl.build(scheme="file", path=str(path), host="").encoded_string(), + checksum="ignored", + datetime=datetime.now(timezone.utc), + ) + + +def test_expand_directories_http_uses_partition_relative_paths(tmp_path): + # Files under a configured partition keep their partition-relative layout, + # namespaced under // (mirrors local push mapping). + partition = tmp_path / "data" + (partition / "subdir").mkdir(parents=True) + f = partition / "subdir" / "file.txt" + f.write_text("hello") + sim_uuid = uuid.uuid4() + partitions = {"data": str(partition)} + + result = _expand_directories_http([_file_data(f)], sim_uuid, partitions) + + assert len(result) == 1 + file_data, local_path, target = result[0] + assert local_path == f + assert target == f"{sim_uuid.hex}/data/subdir/file.txt" + parsed = SimDBUrl(file_data.uri) + assert parsed.scheme == "http" + assert parsed.host == sim_uuid.hex + assert parsed.path == "/data/subdir/file.txt" + assert file_data.type == "FILE" + assert file_data.checksum != "ignored" + + +def test_expand_directories_http_keeps_imas_directory(tmp_path): + # An IMAS (hdf5) directory must stay contained in its own folder. + partition = tmp_path / "data" + imas_dir = partition / "run" / "myids" + imas_dir.mkdir(parents=True) + (imas_dir / "master.h5").write_text("m") + (imas_dir / "0001.h5").write_text("d") + sim_uuid = uuid.uuid4() + partitions = {"data": str(partition)} + + imas_file = FileData( + type="IMAS", + uri=SimDBUrl.build( + scheme="imas", path="hdf5", host="", query=f"path={imas_dir}" + ).encoded_string(), + checksum="ignored", + datetime=datetime.now(timezone.utc), + ) + + result = _expand_directories_http([imas_file], sim_uuid, partitions) + + targets = sorted(t for _, _, t in result) + assert targets == [ + f"{sim_uuid.hex}/data/run/myids/0001.h5", + f"{sim_uuid.hex}/data/run/myids/master.h5", + ] + assert all(file_data.type == "IMAS" for file_data, _, _ in result) + + +def test_expand_directories_http_unpartitioned_file_uses_file_scheme(tmp_path): + # A file outside any partition falls back to the "file" namespace. + f = tmp_path / "loose.txt" + f.write_text("x") + sim_uuid = uuid.uuid4() + + result = _expand_directories_http([_file_data(f)], sim_uuid, {}) + + _, _, target = result[0] + assert target == f"{sim_uuid.hex}/file/{str(f).lstrip('/')}" + + +def test_push_http_command_pushes_and_reports_success(tmp_path): + runner = CliRunner() + config_file = config_test_file() + + fake_api = mock.MagicMock() + fake_api.get_validation_schemas.return_value = [] + fake_api.get_ingestion_status.return_value = "completed" + + sim = mock.MagicMock() + sim.uuid = uuid.uuid4() + fake_db = mock.MagicMock() + fake_db.get_simulation.return_value = sim + + with mock.patch( + "simdb.cli.commands.simulation.RemoteAPI", return_value=fake_api + ), mock.patch("simdb.cli.commands.simulation.get_local_db", return_value=fake_db): + result = runner.invoke( + cli, + [f"--config-file={config_file}", "simulation", "push_http", "iter", "sim1"], + ) + + assert result.exit_code == 0, result.output + fake_api.push_http_simulation.assert_called_once_with(sim) + assert "Successfully pushed simulation" in result.output + + +def test_push_http_command_fails_on_failed_status(tmp_path): + runner = CliRunner() + config_file = config_test_file() + + fake_api = mock.MagicMock() + fake_api.get_validation_schemas.return_value = [] + fake_api.get_ingestion_status.return_value = "copy_failed" + + sim = mock.MagicMock() + sim.uuid = uuid.uuid4() + fake_db = mock.MagicMock() + fake_db.get_simulation.return_value = sim + + with mock.patch( + "simdb.cli.commands.simulation.RemoteAPI", return_value=fake_api + ), mock.patch("simdb.cli.commands.simulation.get_local_db", return_value=fake_db): + result = runner.invoke( + cli, + [f"--config-file={config_file}", "simulation", "push_http", "iter", "sim1"], + ) + + assert result.exit_code != 0 + assert "copy_failed" in result.output diff --git a/tests/remote/api/v1.3/test_resumable_upload.py b/tests/remote/api/v1.3/test_resumable_upload.py new file mode 100644 index 00000000..343320eb --- /dev/null +++ b/tests/remote/api/v1.3/test_resumable_upload.py @@ -0,0 +1,272 @@ +"""Tests for the resumable HTTP upload endpoint (/v1.3/upload/).""" + +import uuid +from pathlib import Path +from urllib.parse import urlsplit + +import pytest +import requests +from conftest import HEADERS + +from simdb.cli import resumable_upload as ru + +INTEROP_HEADER = "Upload-Draft-Interop-Version" + + +@pytest.fixture +def http_partition(client, tmp_path): + """Point the ``http`` partition at a temporary directory for the test.""" + base = tmp_path / "http_staging" + base.mkdir() + client.application.simdb_config.set_option("partition.http", str(base)) + return base + + +def _patch(client, target, offset, data, complete, headers=None): + h = dict(headers or HEADERS) + h["Upload-Offset"] = str(offset) + h["Upload-Complete"] = "?1" if complete else "?0" + h["Content-Type"] = "application/partial-upload" + return client.patch(f"/v1.3/upload/{target}", data=data, headers=h) + + +def test_upload_create_append_complete(client, http_partition): + sim_hex = uuid.uuid4().hex + target = f"{sim_hex}/sub/file.txt" + + # Create the upload resource (empty body). + rv = client.post( + f"/v1.3/upload/{target}", + data=b"", + headers={**HEADERS, "Upload-Complete": "?0", "Upload-Length": "11"}, + ) + assert rv.status_code == 201 + assert rv.headers["Upload-Offset"] == "0" + assert rv.headers[INTEROP_HEADER] == "8" + assert "Location" in rv.headers + + # First chunk. + rv = _patch(client, target, 0, b"hello", complete=False) + assert rv.status_code == 204 + assert rv.headers["Upload-Offset"] == "5" + + # HEAD reports current progress. + rv = client.head(f"/v1.3/upload/{target}", headers=HEADERS) + assert rv.status_code == 204 + assert rv.headers["Upload-Offset"] == "5" + assert rv.headers["Upload-Complete"] == "?0" + + # Final chunk completes the upload. + rv = _patch(client, target, 5, b" world", complete=True) + assert rv.status_code == 200 + assert rv.headers["Upload-Offset"] == "11" + assert rv.headers["Upload-Complete"] == "?1" + + final = http_partition / sim_hex / "sub" / "file.txt" + assert final.read_bytes() == b"hello world" + assert not (final.parent / (final.name + ".partial")).exists() + + +def test_upload_offset_mismatch_returns_409(client, http_partition): + target = f"{uuid.uuid4().hex}/file.txt" + client.post( + f"/v1.3/upload/{target}", data=b"", headers={**HEADERS, "Upload-Complete": "?0"} + ) + _patch(client, target, 0, b"abc", complete=False) + + # Wrong offset -> 409 with the server's actual offset. + rv = _patch(client, target, 0, b"def", complete=False) + assert rv.status_code == 409 + assert rv.headers["Upload-Offset"] == "3" + + +def test_upload_head_missing_returns_404(client, http_partition): + rv = client.head(f"/v1.3/upload/{uuid.uuid4().hex}/missing.txt", headers=HEADERS) + assert rv.status_code == 404 + + +def test_upload_empty_file(client, http_partition): + sim_hex = uuid.uuid4().hex + target = f"{sim_hex}/empty.txt" + client.post( + f"/v1.3/upload/{target}", data=b"", headers={**HEADERS, "Upload-Complete": "?0"} + ) + rv = _patch(client, target, 0, b"", complete=True) + assert rv.status_code == 200 + assert (http_partition / sim_hex / "empty.txt").read_bytes() == b"" + + +def test_upload_delete(client, http_partition): + sim_hex = uuid.uuid4().hex + target = f"{sim_hex}/file.txt" + client.post( + f"/v1.3/upload/{target}", + data=b"data", + headers={**HEADERS, "Upload-Complete": "?1"}, + ) + assert (http_partition / sim_hex / "file.txt").exists() + + rv = client.delete(f"/v1.3/upload/{target}", headers=HEADERS) + assert rv.status_code == 204 + assert not (http_partition / sim_hex / "file.txt").exists() + + +def test_upload_path_traversal_rejected(client, http_partition): + rv = client.post( + "/v1.3/upload/..%2f..%2fescape.txt", + data=b"x", + headers={**HEADERS, "Upload-Complete": "?1"}, + ) + assert rv.status_code in (400, 403, 404) + assert not (http_partition.parent / "escape.txt").exists() + assert not Path("/tmp/escape.txt").exists() + + +class _Resp: + """Adapt a Flask test-client response to the bits resumable_upload uses.""" + + def __init__(self, rv): + self.status_code = rv.status_code + self.headers = rv.headers + self.text = rv.get_data(as_text=True) + + +def _flask_transport(client, fail_once_at=None): + """Route resumable_upload's ``requests`` calls to the Flask test client. + + @param fail_once_at: if set, raise ConnectionError the first time a PATCH is + sent at this offset, to exercise the resume path. + """ + state = {"failed": False} + + def _path(url): + return urlsplit(url).path + + def head(url, headers=None, **kwargs): + return _Resp(client.head(_path(url), headers=dict(headers or {}, **HEADERS))) + + def post(url, data=b"", headers=None, **kwargs): + return _Resp( + client.post(_path(url), data=data, headers=dict(headers or {}, **HEADERS)) + ) + + def patch(url, data=b"", headers=None, **kwargs): + offset = int((headers or {}).get("Upload-Offset", -1)) + if fail_once_at is not None and offset == fail_once_at and not state["failed"]: + state["failed"] = True + raise requests.ConnectionError("simulated network drop") + return _Resp( + client.patch(_path(url), data=data, headers=dict(headers or {}, **HEADERS)) + ) + + return head, post, patch + + +def test_resumable_upload_client_against_server(client, http_partition, monkeypatch): + head, post, patch = _flask_transport(client) + monkeypatch.setattr(ru.requests, "head", head) + monkeypatch.setattr(ru.requests, "post", post) + monkeypatch.setattr(ru.requests, "patch", patch) + + sim_hex = uuid.uuid4().hex + src = http_partition.parent / "source.bin" + payload = b"0123456789" * 100 # 1000 bytes + src.write_bytes(payload) + + ru.resumable_upload( + f"http://localhost/v1.3/upload/{sim_hex}/source.bin", src, chunk_size=256 + ) + + assert (http_partition / sim_hex / "source.bin").read_bytes() == payload + + +def test_resumable_upload_reports_progress(client, http_partition, monkeypatch): + head, post, patch = _flask_transport(client) + monkeypatch.setattr(ru.requests, "head", head) + monkeypatch.setattr(ru.requests, "post", post) + monkeypatch.setattr(ru.requests, "patch", patch) + + sim_hex = uuid.uuid4().hex + src = http_partition.parent / "progress.bin" + payload = b"y" * 1000 + src.write_bytes(payload) + + seen = [] + ru.resumable_upload( + f"http://localhost/v1.3/upload/{sim_hex}/progress.bin", + src, + chunk_size=256, + progress=seen.append, + ) + + # Progress is monotonic non-decreasing and reaches the full file size. + assert seen == sorted(seen) + assert seen[-1] == len(payload) + + +def test_resumable_upload_client_resumes_after_failure( + client, http_partition, monkeypatch +): + # Inject a connection drop at offset 256; the client should HEAD to recover + # the server offset and continue rather than restart. + head, post, patch = _flask_transport(client, fail_once_at=256) + monkeypatch.setattr(ru.requests, "head", head) + monkeypatch.setattr(ru.requests, "post", post) + monkeypatch.setattr(ru.requests, "patch", patch) + + sim_hex = uuid.uuid4().hex + src = http_partition.parent / "resume.bin" + payload = bytes(range(256)) * 4 # 1024 bytes + src.write_bytes(payload) + + ru.resumable_upload( + f"http://localhost/v1.3/upload/{sim_hex}/resume.bin", src, chunk_size=256 + ) + + assert (http_partition / sim_hex / "resume.bin").read_bytes() == payload + + +@pytest.fixture +def small_append_limit(client): + """Advertise (and enforce) a tiny max-append-size for the duration of a test.""" + cfg = client.application.simdb_config + cfg.set_option("server.max_append_size", "256") + yield 256 + cfg.set_option("server.max_append_size", str(8 * 1024 * 1024)) + + +def test_upload_advertises_and_enforces_append_limit( + client, http_partition, small_append_limit +): + target = f"{uuid.uuid4().hex}/file.bin" + rv = client.post( + f"/v1.3/upload/{target}", data=b"", headers={**HEADERS, "Upload-Complete": "?0"} + ) + assert rv.status_code == 201 + assert rv.headers["Upload-Limit"] == "max-append-size=256" + + # A PATCH body larger than the advertised limit is rejected. + rv = _patch(client, target, 0, b"x" * 300, complete=False) + assert rv.status_code == 413 + + +def test_client_respects_server_append_limit( + client, http_partition, small_append_limit, monkeypatch +): + head, post, patch = _flask_transport(client) + monkeypatch.setattr(ru.requests, "head", head) + monkeypatch.setattr(ru.requests, "post", post) + monkeypatch.setattr(ru.requests, "patch", patch) + + sim_hex = uuid.uuid4().hex + src = http_partition.parent / "big.bin" + payload = b"z" * 1000 # larger than the 256-byte append limit + src.write_bytes(payload) + + # Request a chunk size far larger than the server allows; the client must + # clamp to the advertised max-append-size, so the upload still succeeds. + ru.resumable_upload( + f"http://localhost/v1.3/upload/{sim_hex}/big.bin", src, chunk_size=1_000_000 + ) + + assert (http_partition / sim_hex / "big.bin").read_bytes() == payload diff --git a/tests/workers/test_tasks.py b/tests/workers/test_tasks.py index 6d411be4..c52ecb20 100644 --- a/tests/workers/test_tasks.py +++ b/tests/workers/test_tasks.py @@ -18,6 +18,7 @@ _notify_watchers, _resolve_paths, _resolve_uri_to_path, + cleanup_http_staging_task, copy_files_task, ) @@ -226,3 +227,90 @@ def test_notify_watchers_noop_without_watchers(): _notify_watchers(simulation, "subject", "body") delay.assert_not_called() + +def test_copy_files_task_http_keeps_imas_folder_flattens_sibling(task_environment): + """HTTP-staged files are copied like local push: a shared root is stripped, + so an IMAS directory keeps its folder while a sibling file stays flat.""" + env = task_environment + sim_hex = env["simulation_uuid"].hex + + http_partition = env["partition_dir"].parent / "http_staging" + env["config"].set_option("partition.http", str(http_partition)) + + # Stage as the client would: /data/subdir/{test_hdf5/*, test.nc} + staged = http_partition / sim_hex / "data" / "subdir" + (staged / "test_hdf5").mkdir(parents=True) + master = staged / "test_hdf5" / "master.h5" + extra = staged / "test_hdf5" / "0001.h5" + nc = staged / "test.nc" + master.write_text("m") + extra.write_text("d") + nc.write_text("n") + + output_files = [ + _make_file_data( + f"http://{sim_hex}/data/subdir/test_hdf5/master.h5", + checksum=_calculate_checksum(master), + ), + _make_file_data( + f"http://{sim_hex}/data/subdir/test_hdf5/0001.h5", + checksum=_calculate_checksum(extra), + ), + _make_file_data( + f"http://{sim_hex}/data/subdir/test.nc", checksum=_calculate_checksum(nc) + ), + ] + + copy_files_task(env["simulation_uuid"], [], output_files) + + dest = env["upload_dir"] / sim_hex + # The IMAS hdf5 directory keeps its folder... + assert (dest / "test_hdf5" / "master.h5").read_text() == "m" + assert (dest / "test_hdf5" / "0001.h5").read_text() == "d" + # ...while the standalone netcdf file is not given a spurious parent folder. + assert (dest / "test.nc").read_text() == "n" + assert env["simulation"].ingestion_status == IngestionStatus.COPIED + + +def test_resolve_uri_to_path_folds_http_host_into_path(tmp_path): + """http:// URIs put the sim-uuid in the authority; it must be reconstructed.""" + config = Config() + partition_path = tmp_path / "http_staging" + partition_path.mkdir() + config.set_option("partition.http", str(partition_path)) + + uri = SimDBUrl("http://deadbeef/subdir/file.txt") + result = _resolve_uri_to_path(uri, config) + + assert result == partition_path / "deadbeef" / "subdir" / "file.txt" + + +def test_cleanup_http_staging_task_removes_simulation_dir(tmp_path): + partition_path = tmp_path / "http_staging" + sim_uuid = uuid1() + staging = partition_path / sim_uuid.hex + staging.mkdir(parents=True) + (staging / "file.txt").write_text("data") + # A sibling simulation's data must be left untouched. + other = partition_path / "other" + other.mkdir() + (other / "keep.txt").write_text("keep") + + config = Config() + config.set_option("partition.http", str(partition_path)) + config.load = mock.MagicMock() + + with mock.patch("simdb.workers.tasks.Config", return_value=config): + cleanup_http_staging_task(sim_uuid) + + assert not staging.exists() + assert (other / "keep.txt").exists() + + +def test_cleanup_http_staging_task_without_partition_is_noop(tmp_path): + config = Config() + config.load = mock.MagicMock() + + with mock.patch("simdb.workers.tasks.Config", return_value=config): + # Should not raise even though partition.http is unset. + cleanup_http_staging_task(uuid1()) From 44b423e04d13c2bae2a5fc01a352c85935c820f0 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 18 Jun 2026 14:51:05 +0200 Subject: [PATCH 19/31] docs: document push_http and regenerate the CLI reference --- docs/cli.md | 20 ++++++++++++++++++++ docs/user_guide.md | 27 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/docs/cli.md b/docs/cli.md index e7d312a3..62f3c64f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -617,6 +617,7 @@ Commands: modify Modify the ingested simulation. pull Pull the simulation with the given SIM_ID (UUID or alias)... push Push the simulation with the given SIM_ID (UUID or alias)... + push_http Push the simulation with the given SIM_ID to the REMOTE... push_local Push the simulation with the given SIM_ID (UUID or alias)... query Perform a metadata query to find matching local simulations. validate Validate the ingested simulation with given SIM_ID (UUID or... @@ -732,6 +733,25 @@ Options: ``` +```text +Usage: simdb simulation push_http [OPTIONS] [REMOTE] SIM_ID + + Push the simulation with the given SIM_ID to the REMOTE over resumable HTTP. + + Unlike push_local, this does not require a filesystem shared with the + server: the file bytes are uploaded over HTTP using a resumable protocol and + staged into the server's 'http' partition. An interrupted push can be + resumed by re-running the command. + +Options: + --username TEXT Username used to authenticate with the remote. + --password TEXT Password used to authenticate with the remote. + --replaces TEXT SIM_ID of simulation to deprecate and replace. + --add-watcher Add the current user as a watcher of the simulation. + --help Show this message and exit. +``` + + ```text Usage: simdb simulation push_local [OPTIONS] [REMOTE] SIM_ID diff --git a/docs/user_guide.md b/docs/user_guide.md index 1b438ed1..21493ca0 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -345,6 +345,33 @@ When you run `push_local`: This mapping mechanism allows clients and servers to share data over a network or cluster filesystem even if they mount it at different absolute paths. +### Pushing Simulations over Resumable HTTP + +When your local environment and the remote server do **not** share a filesystem, but you still want the background ingestion workflow of `push_local` (rather than the single, non-resumable transfer of `push`), use the `push_http` command: + +```bash +simdb simulation push_http +``` + +`push_http` uploads each file's bytes to the server over HTTP using the IETF "Resumable Uploads for HTTP" protocol (draft-ietf-httpbis-resumable-upload-11). The uploaded files are staged into a server-side partition named `http`, and from then on the flow is identical to `push_local`: SimDB sends the metadata, the server queues a background copy into its upload folder, and the CLI blocks while printing ingestion status updates. + +Because the protocol is resumable, an interrupted upload (lost connection, Ctrl-C) does not have to start over. Re-running `push_http` for the same simulation asks the server how many bytes it already received for each file and continues from that offset. + +The size of each upload chunk is governed by the server: it advertises a maximum append size via the `Upload-Limit` response header, and the client sizes its chunks to stay within that bound. The limit defaults to 8 MiB and can be tuned on the server with the `server.max_append_size` config option (for example, to fit within a reverse proxy's request body limit). + +#### Configuring the `http` Partition on the Server + +Unlike `push_local`, `push_http` needs **no client-side partition configuration** - the file paths are taken directly from the local simulation. The server, however, must define where uploaded bytes are staged by configuring a partition named `http` in its `simdb.cfg`: + +```ini +[partition] +http = /var/lib/simdb/http-staging +``` + +When a file is uploaded to `/`, the server writes it to `//` and references it with an `http:///` URI. The background ingestion task resolves that URI against the `http` partition, copies the file into the simulation's upload folder, and finally removes the staged copy from the `http` partition. + +Subfolder structure is handled exactly as it is for `push_local`: files keep their partition-relative layout, so multi-file IMAS datasets (HDF5, ASCII, MDSplus backends) stay contained within their own directory and are reconstructed correctly on the server, while standalone files (such as an IMAS netcdf `.nc`) are not given a spurious enclosing folder. + ## Pulling simulations from a remote The mirror to pushing simulations is the `pull` command. This command will pull the simulation metadata from the SimDB remote to your local SimDB database and download the simulation data into a directory of your choosing. Once you have pulled a simulation it will appear in any local SimDB queries you perform. The command looks as follows: From 3c78c7b81fff2f5b2c5ef6b5741fb84d120e6b49 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Fri, 19 Jun 2026 09:52:49 +0200 Subject: [PATCH 20/31] Checksums --- docs/user_guide.md | 2 + pyproject.toml | 1 + src/simdb/checksum.py | 39 +++++-- src/simdb/cli/remote_api.py | 43 ++++++-- src/simdb/cli/resumable_upload.py | 26 ++++- src/simdb/database/models/file.py | 6 +- src/simdb/imas/checksum.py | 11 +- src/simdb/imas/utils.py | 5 +- src/simdb/remote/apis/files.py | 8 +- src/simdb/remote/apis/v1_3/upload.py | 55 ++++++++++ src/simdb/workers/tasks.py | 14 ++- tests/cli/test_push_http.py | 21 +++- .../remote/api/v1.3/test_resumable_upload.py | 103 ++++++++++++++++++ tests/workers/test_tasks.py | 11 ++ uv.lock | 2 + 15 files changed, 305 insertions(+), 42 deletions(-) diff --git a/docs/user_guide.md b/docs/user_guide.md index 21493ca0..6dcfb12e 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -359,6 +359,8 @@ Because the protocol is resumable, an interrupted upload (lost connection, Ctrl- The size of each upload chunk is governed by the server: it advertises a maximum append size via the `Upload-Limit` response header, and the client sizes its chunks to stay within that bound. The limit defaults to 8 MiB and can be tuned on the server with the `server.max_append_size` config option (for example, to fit within a reverse proxy's request body limit). +Each chunk is integrity-checked using the RFC 9530 digest field. The client sends a `Content-Digest` (SHA-256) with every chunk so the server can verify that chunk before appending it. A digest mismatch is rejected with a `400` response and the offending bytes are not stored, so corruption in transit cannot be silently committed. + #### Configuring the `http` Partition on the Server Unlike `push_local`, `push_http` needs **no client-side partition configuration** - the file paths are taken directly from the local simulation. The server, however, must define where uploaded bytes are staged by configuring a partition named `http` in its `simdb.cfg`: diff --git a/pyproject.toml b/pyproject.toml index 44a29a20..ad4c8ab3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ dependencies = [ "sqlalchemy>=1.2.12,<2.0", "alembic~=1.13", "rich>=14.3.3", + "xxhash>=3.7.0", ] [project.optional-dependencies] diff --git a/src/simdb/checksum.py b/src/simdb/checksum.py index f85aea1f..91867922 100644 --- a/src/simdb/checksum.py +++ b/src/simdb/checksum.py @@ -1,27 +1,42 @@ import hashlib from pathlib import Path +from typing import Callable, Optional from simdb.imas.utils import SimDBUrl +#: Algorithm used for all catalog checksums. +CHECKSUM_ALGORITHM = "sha1" +#: Buffer size for reading files while hashing. Larger reads mean far fewer +#: syscalls on big files, which noticeably speeds up checksumming. +READ_CHUNK_SIZE = 1024 * 1024 -def calculate_checksum(path: Path) -> str: - """Generate a SHA1 checksum from the file at the given path. - :param path: the path of the file to checksum - :return: a string containing the hex representation of the computed SHA1 checksum +def hash_file( + path: Path, + algorithm: str = CHECKSUM_ALGORITHM, + progress: Optional[Callable[[int], None]] = None, +) -> str: + """Return the hex digest of ``path`` computed with ``algorithm``. + + @param progress: optional callback invoked with the number of bytes read for + each block, suitable for advancing a progress bar. """ - sha1 = hashlib.sha1() + digest = hashlib.new(algorithm) with path.open("rb") as file: - for chunk in iter(lambda: file.read(4096), b""): - sha1.update(chunk) - return sha1.hexdigest() + for chunk in iter(lambda: file.read(READ_CHUNK_SIZE), b""): + digest.update(chunk) + if progress is not None: + progress(len(chunk)) + return digest.hexdigest() + +def file_checksum(uri: SimDBUrl, algorithm: str = CHECKSUM_ALGORITHM) -> str: + """Generate a checksum for the file at ``uri``. -def sha1_checksum(uri: SimDBUrl) -> str: - """Generate a SHA1 checksum from the given file. + Checksums use :data:`CHECKSUM_ALGORITHM` (SHA-1). :param uri: the URI of the file to checksum - :return: a string containing the hex representation of the computed SHA1 checksum + :return: a string containing the hex representation of the computed checksum """ if uri.scheme != "file": raise ValueError(f"invalid scheme for file checksum: {uri.scheme}") @@ -34,4 +49,4 @@ def sha1_checksum(uri: SimDBUrl) -> str: if not path.is_file(): raise ValueError("File appears to be a directory") - return calculate_checksum(path) + return hash_file(path, algorithm) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 8aa1a4f1..dd260483 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -42,7 +42,7 @@ ) from semantic_version import Version -from simdb.checksum import calculate_checksum +from simdb.checksum import CHECKSUM_ALGORITHM, READ_CHUNK_SIZE, hash_file from simdb.cli.resumable_upload import resumable_upload from simdb.config import Config from simdb.database.models import Simulation @@ -280,7 +280,7 @@ def _file_data_for_partition( return FileData( type=file.type, uri=new_uri.encoded_string(), - checksum=calculate_checksum(source), + checksum=hash_file(source), datetime=file.datetime, usage=file.usage, purpose=file.purpose, @@ -365,6 +365,10 @@ def _make_http_entry( The relative path is taken from :func:`_find_partition_for_file` (the same mapping ``push_local`` uses) and namespaced under ``//`` so uploads from different partitions never collide on the server. + + The checksum is left empty here and filled in later by + :func:`_compute_checksums`, so that hashing (a full read of every file) can be + reported with a progress bar instead of stalling silently before the upload. """ scheme, rel = _find_partition_for_file(local_path, partitions) rel_posix = rel.as_posix().lstrip("/") @@ -375,7 +379,7 @@ def _make_http_entry( FileData( type=file_type, uri=new_uri.encoded_string(), - checksum=calculate_checksum(local_path), + checksum="", datetime=template.datetime, usage=template.usage, purpose=template.purpose, @@ -388,6 +392,30 @@ def _make_http_entry( ) +def _compute_checksums(files: List[Tuple[FileData, Path, str]]) -> None: + """Compute and store the SHA-1 checksum of each file, reporting progress. + + Hashing reads every file in full and is the main delay before the upload + starts, so surface it with a byte-level progress bar (mirroring the upload + bars). The computed checksum is stored as the catalog checksum. + """ + total_bytes = sum(local_path.stat().st_size for _, local_path, _ in files) + with Progress( + TextColumn("[bold blue]{task.description}"), + BarColumn(), + DownloadColumn(), + TransferSpeedColumn(), + TimeRemainingColumn(), + ) as progress: + task = progress.add_task("Calculating checksums", total=total_bytes) + for file_data, local_path, _target in files: + progress.update(task, description=f"Hashing {local_path.name}") + file_data.checksum = hash_file( + local_path, progress=lambda n: progress.advance(task, n) + ) + progress.update(task, description="Calculated checksums") + + class RemoteAPI: """ Class to represent connection to remote API. @@ -1091,6 +1119,7 @@ def push_http_simulation(self, simulation: Simulation): files = list(itertools.chain(inputs, outputs)) upload_headers = {"User-Agent": "it_script_basic"} if files: + _compute_checksums(files) self._upload_files(files, upload_headers) sim_data.inputs.root = [file_data for file_data, _, _ in inputs] @@ -1319,7 +1348,7 @@ def _pull_file( response = self.get(f"file/download/{uuid.hex}/{index}", stream=True) to_path.parent.mkdir(parents=True, exist_ok=True) - sha1 = hashlib.sha1() + digest = hashlib.new(CHECKSUM_ALGORITHM) with to_path.open("wb") as f: total_length = response.headers.get("content-length") @@ -1328,8 +1357,8 @@ def _pull_file( else: downloaded = 0 total_length = int(total_length) - for data in response.iter_content(chunk_size=4096): - sha1.update(data) + for data in response.iter_content(chunk_size=READ_CHUNK_SIZE): + digest.update(data) downloaded += len(data) f.write(data) done = int(50 * downloaded / total_length) @@ -1345,7 +1374,7 @@ def _pull_file( ) print("\r", file=out_stream, end="", flush=True) - if sha1.hexdigest() != checksum: + if digest.hexdigest() != checksum: raise APIError(f"Checksum failed for file {from_path}") @versioned_method("v1.2", "v1.3") diff --git a/src/simdb/cli/resumable_upload.py b/src/simdb/cli/resumable_upload.py index a08513da..eefe2bf6 100644 --- a/src/simdb/cli/resumable_upload.py +++ b/src/simdb/cli/resumable_upload.py @@ -1,8 +1,7 @@ """Client for the IETF "Resumable Uploads for HTTP" protocol. This is a small, dependency-free (uses ``requests``, already a dependency) -implementation of draft-ietf-httpbis-resumable-upload-11 (interop version 8) - -the same protocol implemented by https://github.com/Yannicked/pyrufh. +implementation of draft-ietf-httpbis-resumable-upload-11 (interop version 8) The single public entry point :func:`resumable_upload` uploads a local file to a server endpoint that speaks the same protocol. The upload resource is identified @@ -11,6 +10,8 @@ the server (via ``HEAD``) how many bytes it already has and continues from there. """ +import base64 +import hashlib import logging from pathlib import Path from typing import Callable, Mapping, Optional, Tuple, Union @@ -25,6 +26,8 @@ INTEROP_HEADER = "Upload-Draft-Interop-Version" #: Content type used for the body of append (``PATCH``) requests. PARTIAL_UPLOAD_CONTENT_TYPE = "application/partial-upload" +DIGEST_ALGORITHM = "sha-256" +_HASHLIB_NAME = "sha256" #: Default size of a single ``PATCH`` chunk (kept below the 10 MB request cap #: enforced on the ITER network, see ``RemoteAPI.push_simulation``). DEFAULT_CHUNK_SIZE = 8 * 1024 * 1024 @@ -54,6 +57,21 @@ def _parse_bool_field(value: Optional[str]) -> Optional[bool]: return None +def _format_digest(digest: bytes) -> str: + """Render a raw digest as an RFC 9530 structured-field dictionary value. + + The single member uses :data:`DIGEST_ALGORITHM` as its key and the digest as + a base64-encoded byte sequence, e.g. ``sha-256=:47DEQpj8HBSa...:``. + """ + encoded = base64.b64encode(digest).decode("ascii") + return f"{DIGEST_ALGORITHM}=:{encoded}:" + + +def _content_digest(data: bytes) -> str: + """``Content-Digest`` value for the bytes of a single request body.""" + return _format_digest(hashlib.new(_HASHLIB_NAME, data).digest()) + + def _header_int(resp: "requests.Response", name: str) -> Optional[int]: raw = resp.headers.get(name) if raw is None: @@ -143,7 +161,8 @@ def resumable_upload( with path.open("rb") as f: _send_chunks( - url, f, offset, total, chunk_size, auth, cookies, headers, progress + url, f, offset, total, chunk_size, + auth, cookies, headers, progress, ) @@ -204,6 +223,7 @@ def _send_chunks( patch_headers["Content-Type"] = PARTIAL_UPLOAD_CONTENT_TYPE patch_headers["Upload-Offset"] = str(offset) patch_headers["Upload-Complete"] = _bool_field(complete) + patch_headers["Content-Digest"] = _content_digest(chunk) try: resp = requests.patch( diff --git a/src/simdb/database/models/file.py b/src/simdb/database/models/file.py index 202b94dc..92228dfe 100644 --- a/src/simdb/database/models/file.py +++ b/src/simdb/database/models/file.py @@ -7,7 +7,7 @@ from sqlalchemy import Column from sqlalchemy import types as sql_types -from simdb.checksum import sha1_checksum +from simdb.checksum import file_checksum from simdb.cli.manifest import DataType from simdb.config.config import Config from simdb.docstrings import inherit_docstrings @@ -78,7 +78,7 @@ def generate_checksum(self, config, ids_list: list): elif self.type == DataType.IMAS: checksum = imas_checksum(self.uri, ids_list) elif self.type == DataType.FILE: - checksum = sha1_checksum(self.uri) + checksum = file_checksum(self.uri) else: raise NotImplementedError(f"Cannot generate checksum for type {self.type}.") return checksum @@ -139,7 +139,7 @@ def to_model_with_path(self) -> FileGetDataResponse: files = [FileInfo(path=Path(self.uri.path), checksum=self.checksum)] else: files = [ - FileInfo(path=path, checksum=sha1_checksum(SimDBUrl(f"file:{path}"))) + FileInfo(path=path, checksum=file_checksum(SimDBUrl(f"file:{path}"))) for path in imas_files(self.uri) ] return FileGetDataResponse( diff --git a/src/simdb/imas/checksum.py b/src/simdb/imas/checksum.py index d9d403ef..52717a5c 100644 --- a/src/simdb/imas/checksum.py +++ b/src/simdb/imas/checksum.py @@ -1,6 +1,7 @@ import hashlib from pathlib import Path +from simdb.checksum import CHECKSUM_ALGORITHM, READ_CHUNK_SIZE from simdb.imas.utils import SimDBUrl from .utils import imas_files, list_idss, open_imas @@ -8,8 +9,8 @@ IGNORED_FIELDS = ("data_dictionary", "access_layer", "access_layer_language") -def checksum(uri: SimDBUrl, ids_list: list) -> str: - sha1 = hashlib.sha1() +def checksum(uri: SimDBUrl, ids_list: list, algorithm: str = CHECKSUM_ALGORITHM) -> str: + digest = hashlib.new(algorithm) if not ids_list: entry = open_imas(uri) @@ -25,6 +26,6 @@ def checksum(uri: SimDBUrl, ids_list: list) -> str: and ids_name[0] not in ids_list ): continue - for chunk in iter(lambda: file.read(4096), b""): - sha1.update(chunk) - return sha1.hexdigest() + for chunk in iter(lambda: file.read(READ_CHUNK_SIZE), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/src/simdb/imas/utils.py b/src/simdb/imas/utils.py index 016bc400..08353d7b 100644 --- a/src/simdb/imas/utils.py +++ b/src/simdb/imas/utils.py @@ -103,10 +103,11 @@ def list_idss(entry: DBEntry) -> List[str]: for ids_name in entry.factory.ids_names(): occurrences = entry.list_all_occurrences(ids_name) if occurrences and len(occurrences) > 0: - for occurrence in range(len(occurrences)): + for occurrence in occurrences: if occurrence > 0: idss.append(ids_name + "_" + str(occurrence)) - idss.append(ids_name) + else: + idss.append(ids_name) return idss diff --git a/src/simdb/remote/apis/files.py b/src/simdb/remote/apis/files.py index cd7cad5e..61286623 100644 --- a/src/simdb/remote/apis/files.py +++ b/src/simdb/remote/apis/files.py @@ -8,7 +8,7 @@ from flask_restx import Namespace, Resource from werkzeug.datastructures import FileStorage -from simdb.checksum import sha1_checksum +from simdb.checksum import file_checksum from simdb.cli.manifest import DataType from simdb.database import DatabaseError, models from simdb.imas.checksum import checksum as imas_checksum @@ -49,7 +49,9 @@ def _verify_file( path = secure_path(Path(sim_file.uri.path), common_root, staging_dir) if not path.exists(): raise ValueError(f"file {path} does not exist") - checksum = sha1_checksum(SimDBUrl.build(scheme="file", path=path.as_posix())) + checksum = file_checksum( + SimDBUrl.build(scheme="file", host="", path=path.as_posix()), + ) if sim_file.checksum != checksum: raise ValueError(f"checksum failed for file {sim_file!r}") elif sim_file.type == DataType.IMAS: @@ -66,7 +68,7 @@ def _verify_file( else: path_value = str(staging_dir) new_uri = uri.build( - scheme=uri.scheme, path=uri.path, query=f"path={path_value}" + scheme=uri.scheme, host="", path=uri.path, query=f"path={path_value}" ) checksum = imas_checksum(new_uri, ids_list or []) if sim_file.checksum != checksum: diff --git a/src/simdb/remote/apis/v1_3/upload.py b/src/simdb/remote/apis/v1_3/upload.py index 702c19d1..79f92ea1 100644 --- a/src/simdb/remote/apis/v1_3/upload.py +++ b/src/simdb/remote/apis/v1_3/upload.py @@ -13,7 +13,10 @@ when the upload completes. The current offset is simply the size of that file. """ +import base64 +import binascii import contextlib +import hashlib from pathlib import Path from typing import Optional, Tuple @@ -34,6 +37,7 @@ #: clients via the ``Upload-Limit`` header. Overridable with the #: ``server.max_append_size`` config option. DEFAULT_MAX_APPEND_SIZE = 8 * 1024 * 1024 +_DIGEST_ALGORITHMS = {"sha-256": "sha256", "sha-512": "sha512"} def _max_append_size() -> int: @@ -61,6 +65,47 @@ def _parse_bool_field(value: Optional[str]) -> Optional[bool]: return None +def _parse_digest_header(value: Optional[str]) -> dict: + """Parse an RFC 9530 digest structured-field dictionary into ``{algo: bytes}``. + + Members are ``algo=:base64:`` items; only the algorithms in + :data:`_DIGEST_ALGORITHMS` are kept. Unparseable members are skipped. Commas + are safe separators here because base64 never contains them. + """ + digests: dict = {} + if not value: + return digests + for member in value.split(","): + member = member.strip() + if "=" not in member: + continue + key, _, raw = member.partition("=") + key = key.strip().lower() + if key not in _DIGEST_ALGORITHMS: + continue + raw = raw.strip() + if len(raw) >= 2 and raw.startswith(":") and raw.endswith(":"): + raw = raw[1:-1] + try: + digests[key] = base64.b64decode(raw, validate=True) + except (binascii.Error, ValueError): + continue + return digests + + +def _digest_matches(value: Optional[str], data: bytes) -> bool: + """Return whether the digest header ``value`` matches ``data``. + + Passes when no recognised algorithm is present (nothing to verify) and when + every recognised algorithm's digest matches; fails on any mismatch. + """ + provided = _parse_digest_header(value) + return all( + hashlib.new(_DIGEST_ALGORITHMS[algo], data).digest() == expected + for algo, expected in provided.items() + ) + + def _partition_base() -> Path: base = current_app.simdb_config.get_string_option("partition.http", default=None) if not base: @@ -115,6 +160,10 @@ def post(self, target: str, user: User) -> Response: data = request.get_data() or b"" if len(data) > _max_append_size(): return Response(status=413, headers=_headers(0, False)) + # Per-request integrity: reject before writing anything if the body does + # not match the client's Content-Digest. + if not _digest_matches(request.headers.get("Content-Digest"), data): + return Response(status=400, headers=_headers(0, False)) with partial.open("wb") as f: f.write(data) offset = len(data) @@ -160,6 +209,12 @@ def patch(self, target: str, user: User) -> Response: if len(data) > _max_append_size(): return Response(status=413, headers=_headers(offset, False)) + # Per-request integrity: reject (without appending) if the body does not + # match the client's Content-Digest. Offset is left unchanged so the + # client can safely retry the same chunk. + if not _digest_matches(request.headers.get("Content-Digest"), data): + return Response(status=400, headers=_headers(offset, False)) + partial.parent.mkdir(parents=True, exist_ok=True) with partial.open("ab") as f: f.write(data) diff --git a/src/simdb/workers/tasks.py b/src/simdb/workers/tasks.py index 01adf8fc..3895898e 100644 --- a/src/simdb/workers/tasks.py +++ b/src/simdb/workers/tasks.py @@ -7,7 +7,7 @@ from typing import Iterable, List from uuid import UUID -from simdb.checksum import calculate_checksum as _calculate_checksum +from simdb.checksum import hash_file from simdb.config import Config from simdb.database.database import get_db from simdb.database.models import File @@ -113,6 +113,12 @@ def _copy_files( shutil.copy2(source, destination) +def _checksum_matches(path: Path, expected: str) -> bool: + """Whether ``path`` matches ``expected``.""" + return hash_file(path) == expected + + + def _get_imas_identifier_path(path: Path) -> Path: if path.suffix == ".nc": return path @@ -125,8 +131,7 @@ def _create_file_from_data( uri = SimDBUrl(data.uri) path = _resolve_uri_to_path(uri, config) - checksum = _calculate_checksum(path) - if data.checksum != checksum: + if not _checksum_matches(path, data.checksum): raise ValueError("Hash of file does not match provided checksum") file = File.from_data_model(data) @@ -154,8 +159,7 @@ def _create_files_from_data_list( seen_imas_paths.add(imas_path) file = _create_file_from_data(file_data, config, imas_path) else: - checksum = _calculate_checksum(path) - if file_data.checksum != checksum: + if not _checksum_matches(path, file_data.checksum): raise ValueError("Hash of file does not match provided checksum") file = File.from_data_model(file_data) file.uri = SimDBUrl.build(scheme="file", path=path.as_posix()) diff --git a/tests/cli/test_push_http.py b/tests/cli/test_push_http.py index c480a6b6..27e99022 100644 --- a/tests/cli/test_push_http.py +++ b/tests/cli/test_push_http.py @@ -1,5 +1,6 @@ """Tests for the HTTP push client helpers and CLI command.""" +import hashlib import uuid from datetime import datetime, timezone from unittest import mock @@ -7,7 +8,7 @@ from click.testing import CliRunner from utils import config_test_file -from simdb.cli.remote_api import _expand_directories_http +from simdb.cli.remote_api import _compute_checksums, _expand_directories_http from simdb.cli.simdb import cli from simdb.imas.utils import SimDBUrl from simdb.remote.models import FileData @@ -43,7 +44,23 @@ def test_expand_directories_http_uses_partition_relative_paths(tmp_path): assert parsed.host == sim_uuid.hex assert parsed.path == "/data/subdir/file.txt" assert file_data.type == "FILE" - assert file_data.checksum != "ignored" + assert file_data.checksum == "" + + +def test_compute_checksums_populates_sha1(tmp_path): + f1 = tmp_path / "a.txt" + f1.write_bytes(b"hello") + f2 = tmp_path / "b.txt" + f2.write_bytes(b"world!!") + + fd1 = _file_data(f1) + fd2 = _file_data(f2) + files = [(fd1, f1, "a"), (fd2, f2, "b")] + + _compute_checksums(files) + + assert fd1.checksum == hashlib.sha1(b"hello").hexdigest() + assert fd2.checksum == hashlib.sha1(b"world!!").hexdigest() def test_expand_directories_http_keeps_imas_directory(tmp_path): diff --git a/tests/remote/api/v1.3/test_resumable_upload.py b/tests/remote/api/v1.3/test_resumable_upload.py index 343320eb..1b7ed6a0 100644 --- a/tests/remote/api/v1.3/test_resumable_upload.py +++ b/tests/remote/api/v1.3/test_resumable_upload.py @@ -1,5 +1,7 @@ """Tests for the resumable HTTP upload endpoint (/v1.3/upload/).""" +import base64 +import hashlib import uuid from pathlib import Path from urllib.parse import urlsplit @@ -13,6 +15,12 @@ INTEROP_HEADER = "Upload-Draft-Interop-Version" +def _digest(data): + """RFC 9530 ``sha-256`` digest structured-field value for ``data``.""" + encoded = base64.b64encode(hashlib.sha256(data).digest()).decode("ascii") + return f"sha-256=:{encoded}:" + + @pytest.fixture def http_partition(client, tmp_path): """Point the ``http`` partition at a temporary directory for the test.""" @@ -270,3 +278,98 @@ def test_client_respects_server_append_limit( ) assert (http_partition / sim_hex / "big.bin").read_bytes() == payload + + +def test_resumable_upload_completes_multi_chunk(client, http_partition, monkeypatch): + head, post, patch = _flask_transport(client) + monkeypatch.setattr(ru.requests, "head", head) + monkeypatch.setattr(ru.requests, "post", post) + monkeypatch.setattr(ru.requests, "patch", patch) + + sim_hex = uuid.uuid4().hex + src = http_partition.parent / "reuse.bin" + payload = b"0123456789" * 100 + src.write_bytes(payload) + + # The file is uploaded across several chunks and assembled on the server. + ru.resumable_upload( + f"http://localhost/v1.3/upload/{sim_hex}/reuse.bin", + src, + chunk_size=256, + ) + assert (http_partition / sim_hex / "reuse.bin").read_bytes() == payload + + +def test_patch_content_digest_match_accepted(client, http_partition): + target = f"{uuid.uuid4().hex}/file.txt" + client.post( + f"/v1.3/upload/{target}", data=b"", headers={**HEADERS, "Upload-Complete": "?0"} + ) + rv = _patch( + client, target, 0, b"hello", complete=False, + headers={**HEADERS, "Content-Digest": _digest(b"hello")}, + ) + assert rv.status_code == 204 + assert rv.headers["Upload-Offset"] == "5" + + +def test_patch_content_digest_mismatch_rejected(client, http_partition): + target = f"{uuid.uuid4().hex}/file.txt" + client.post( + f"/v1.3/upload/{target}", data=b"", headers={**HEADERS, "Upload-Complete": "?0"} + ) + # Digest of different bytes than the body -> 400 and nothing appended. + rv = _patch( + client, target, 0, b"hello", complete=False, + headers={**HEADERS, "Content-Digest": _digest(b"goodbye")}, + ) + assert rv.status_code == 400 + assert rv.headers["Upload-Offset"] == "0" + assert not (http_partition / target).exists() + # The partial exists (created empty by POST) but the rejected body was not + # appended. + assert (http_partition / (target + ".partial")).read_bytes() == b"" + + +def test_multi_chunk_upload_finalizes(client, http_partition): + sim_hex = uuid.uuid4().hex + target = f"{sim_hex}/file.txt" + client.post( + f"/v1.3/upload/{target}", data=b"", headers={**HEADERS, "Upload-Complete": "?0"} + ) + _patch( + client, target, 0, b"hello", complete=False, + headers={**HEADERS, "Content-Digest": _digest(b"hello")}, + ) + rv = _patch( + client, target, 5, b" world", complete=True, + headers={**HEADERS, "Content-Digest": _digest(b" world")}, + ) + assert rv.status_code == 200 + assert (http_partition / sim_hex / "file.txt").read_bytes() == b"hello world" + + +def test_post_content_digest_mismatch_rejected(client, http_partition): + sim_hex = uuid.uuid4().hex + target = f"{sim_hex}/file.txt" + rv = client.post( + f"/v1.3/upload/{target}", + data=b"hello", + headers={**HEADERS, "Upload-Complete": "?0", "Content-Digest": _digest(b"x")}, + ) + assert rv.status_code == 400 + assert not (http_partition / sim_hex / "file.txt.partial").exists() + + +def test_unknown_digest_algorithm_ignored(client, http_partition): + # A digest using an algorithm the server cannot recompute is ignored rather + # than rejected, so the upload still succeeds. + target = f"{uuid.uuid4().hex}/file.txt" + client.post( + f"/v1.3/upload/{target}", data=b"", headers={**HEADERS, "Upload-Complete": "?0"} + ) + rv = _patch( + client, target, 0, b"hello", complete=False, + headers={**HEADERS, "Content-Digest": "unixsum=:0061:"}, + ) + assert rv.status_code == 204 diff --git a/tests/workers/test_tasks.py b/tests/workers/test_tasks.py index c52ecb20..8168d626 100644 --- a/tests/workers/test_tasks.py +++ b/tests/workers/test_tasks.py @@ -1,3 +1,4 @@ +import hashlib from datetime import datetime, timezone from unittest import mock from uuid import uuid1 @@ -11,6 +12,7 @@ from simdb.workers import tasks as simdb_tasks from simdb.workers.tasks import ( _calculate_checksum, + _checksum_matches, _copy_files, _create_file_from_data, _get_imas_identifier_path, @@ -150,6 +152,15 @@ def test_create_file_from_data_raises_on_checksum_mismatch(tmp_path): _create_file_from_data(file_data, config, data_file) +def test_checksum_matches_uses_sha1(tmp_path): + data_file = tmp_path / "testfile.txt" + content = b"content" + data_file.write_bytes(content) + + assert _checksum_matches(data_file, hashlib.sha1(content).hexdigest()) + assert not _checksum_matches(data_file, hashlib.sha1(b"other").hexdigest()) + + @pytest.fixture def task_environment(tmp_path): """Set up Config, mocked DB, and directory layout for copy_files_task tests.""" diff --git a/uv.lock b/uv.lock index 532738e0..b84ab266 100644 --- a/uv.lock +++ b/uv.lock @@ -3013,6 +3013,7 @@ dependencies = [ { name = "rich", version = "15.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, { name = "semantic-version" }, { name = "sqlalchemy" }, + { name = "xxhash" }, ] [package.optional-dependencies] @@ -3167,6 +3168,7 @@ requires-dist = [ { name = "sphinx-autodoc-typehints", marker = "extra == 'build-docs'", specifier = ">=1.12.0" }, { name = "sphinx-immaterial", marker = "extra == 'build-docs'", specifier = ">=0.11.14" }, { name = "sqlalchemy", specifier = ">=1.2.12,<2.0" }, + { name = "xxhash", specifier = ">=3.7.0" }, ] provides-extras = ["server", "auth-ad", "auth-keycloak", "auth-ldap", "auth", "imas-validator", "build-docs", "postgres", "all"] From 08d52b33f4edec9d4dc0e025c72a10dd872cc750 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Fri, 19 Jun 2026 11:24:50 +0200 Subject: [PATCH 21/31] Ty fixes --- src/simdb/cli/resumable_upload.py | 14 +++++++-- .../remote/api/v1.3/test_resumable_upload.py | 30 +++++++++++++++---- tests/workers/test_tasks.py | 6 ++-- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/simdb/cli/resumable_upload.py b/src/simdb/cli/resumable_upload.py index eefe2bf6..d6457ce5 100644 --- a/src/simdb/cli/resumable_upload.py +++ b/src/simdb/cli/resumable_upload.py @@ -18,6 +18,7 @@ import requests from requests.auth import AuthBase +from requests.cookies import RequestsCookieJar logger = logging.getLogger(__name__) @@ -123,7 +124,7 @@ def resumable_upload( path: Union[str, Path], *, auth: Optional[Union[AuthBase, Tuple[str, str]]] = None, - cookies: Optional[Mapping[str, str]] = None, + cookies: Optional[Union[Mapping[str, str], RequestsCookieJar]] = None, headers: Optional[Mapping[str, str]] = None, chunk_size: int = DEFAULT_CHUNK_SIZE, progress: Optional[Callable[[int], None]] = None, @@ -161,8 +162,15 @@ def resumable_upload( with path.open("rb") as f: _send_chunks( - url, f, offset, total, chunk_size, - auth, cookies, headers, progress, + url, + f, + offset, + total, + chunk_size, + auth, + cookies, + headers, + progress, ) diff --git a/tests/remote/api/v1.3/test_resumable_upload.py b/tests/remote/api/v1.3/test_resumable_upload.py index 1b7ed6a0..486c7f4f 100644 --- a/tests/remote/api/v1.3/test_resumable_upload.py +++ b/tests/remote/api/v1.3/test_resumable_upload.py @@ -306,7 +306,11 @@ def test_patch_content_digest_match_accepted(client, http_partition): f"/v1.3/upload/{target}", data=b"", headers={**HEADERS, "Upload-Complete": "?0"} ) rv = _patch( - client, target, 0, b"hello", complete=False, + client, + target, + 0, + b"hello", + complete=False, headers={**HEADERS, "Content-Digest": _digest(b"hello")}, ) assert rv.status_code == 204 @@ -320,7 +324,11 @@ def test_patch_content_digest_mismatch_rejected(client, http_partition): ) # Digest of different bytes than the body -> 400 and nothing appended. rv = _patch( - client, target, 0, b"hello", complete=False, + client, + target, + 0, + b"hello", + complete=False, headers={**HEADERS, "Content-Digest": _digest(b"goodbye")}, ) assert rv.status_code == 400 @@ -338,11 +346,19 @@ def test_multi_chunk_upload_finalizes(client, http_partition): f"/v1.3/upload/{target}", data=b"", headers={**HEADERS, "Upload-Complete": "?0"} ) _patch( - client, target, 0, b"hello", complete=False, + client, + target, + 0, + b"hello", + complete=False, headers={**HEADERS, "Content-Digest": _digest(b"hello")}, ) rv = _patch( - client, target, 5, b" world", complete=True, + client, + target, + 5, + b" world", + complete=True, headers={**HEADERS, "Content-Digest": _digest(b" world")}, ) assert rv.status_code == 200 @@ -369,7 +385,11 @@ def test_unknown_digest_algorithm_ignored(client, http_partition): f"/v1.3/upload/{target}", data=b"", headers={**HEADERS, "Upload-Complete": "?0"} ) rv = _patch( - client, target, 0, b"hello", complete=False, + client, + target, + 0, + b"hello", + complete=False, headers={**HEADERS, "Content-Digest": "unixsum=:0061:"}, ) assert rv.status_code == 204 diff --git a/tests/workers/test_tasks.py b/tests/workers/test_tasks.py index 8168d626..716014f1 100644 --- a/tests/workers/test_tasks.py +++ b/tests/workers/test_tasks.py @@ -174,7 +174,7 @@ def task_environment(tmp_path): config.set_option("database.file", str(tmp_path / "test.db")) config.set_option("server.upload_folder", str(upload_dir)) config.set_option("partition.data", str(partition_dir)) - config.load = mock.MagicMock() + config.load = mock.MagicMock() # ty: ignore[invalid-assignment] simulation_uuid = uuid1() mock_simulation = mock.MagicMock(uuid=simulation_uuid, inputs=[], outputs=[]) @@ -309,7 +309,7 @@ def test_cleanup_http_staging_task_removes_simulation_dir(tmp_path): config = Config() config.set_option("partition.http", str(partition_path)) - config.load = mock.MagicMock() + config.load = mock.MagicMock() # ty: ignore[invalid-assignment] with mock.patch("simdb.workers.tasks.Config", return_value=config): cleanup_http_staging_task(sim_uuid) @@ -320,7 +320,7 @@ def test_cleanup_http_staging_task_removes_simulation_dir(tmp_path): def test_cleanup_http_staging_task_without_partition_is_noop(tmp_path): config = Config() - config.load = mock.MagicMock() + config.load = mock.MagicMock() # ty: ignore[invalid-assignment] with mock.patch("simdb.workers.tasks.Config", return_value=config): # Should not raise even though partition.http is unset. From aba8c302eb4aa9986186b641bc8e82a84bfeb6b9 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Fri, 19 Jun 2026 15:55:05 +0200 Subject: [PATCH 22/31] Fix small issues --- src/simdb/cli/remote_api.py | 31 ++++++++++++++++++++--- src/simdb/cli/resumable_upload.py | 25 +++++++++++++----- src/simdb/database/models/file.py | 7 ++++- src/simdb/imas/utils.py | 28 +++++++++++++------- src/simdb/remote/apis/v1_3/simulations.py | 11 +++++--- src/simdb/remote/apis/v1_3/upload.py | 16 +++++++++--- 6 files changed, 89 insertions(+), 29 deletions(-) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index dd260483..9f07b4fc 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -24,7 +24,6 @@ Optional, Tuple, Union, - cast, ) from urllib.parse import ParseResult, quote, urlparse @@ -40,13 +39,14 @@ TimeRemainingColumn, TransferSpeedColumn, ) +from netCDF4 import Dataset from semantic_version import Version from simdb.checksum import CHECKSUM_ALGORITHM, READ_CHUNK_SIZE, hash_file from simdb.cli.resumable_upload import resumable_upload from simdb.config import Config from simdb.database.models import Simulation -from simdb.imas.utils import SimDBUrl, imas_files +from simdb.imas.utils import SimDBUrl, imas_backend_for_directory, imas_files from simdb.json import CustomDecoder, CustomEncoder from simdb.remote import CLIENT_API_VERSIONS, APIConstants from simdb.remote.models import FileData, SimulationPostData @@ -247,6 +247,29 @@ def _get_paths(file: "File") -> Iterable[Path]: return imas_files(file.uri) +def _check_file_is_imas(file: Path) -> bool: + # NetCDF is identified by the IMAS "Conventions" attribute + if file.suffix == ".nc": + try: + with Dataset(file, "r") as ds: + if getattr(ds, "Conventions", None) == "IMAS": + return True + except OSError: + # Not a readable NetCDF file; fall back to the directory heuristics + pass + + try: + imas_backend_for_directory(file.parent) + except ValueError: + return False + return True + + +def _partition_roots(config: Config) -> dict[str, str]: + section = config.get_section("partition", default={}) + return {k: str(v) for k, v in section.items()} + + def _find_partition_for_file( file: Path, partitions: Dict[str, str] ) -> Tuple[str, Path]: @@ -1042,7 +1065,7 @@ def _send_chunk( def push_local_simulation(self, simulation: Simulation, add_watcher: bool = False): sim_data = simulation.to_model(recurse=True) - partitions = cast(Dict[str, str], self._config.get_section("partition")) + partitions = _partition_roots(self._config) sim_data.inputs.root = _expand_directories(sim_data.inputs.root, partitions) sim_data.outputs.root = _expand_directories(sim_data.outputs.root, partitions) @@ -1108,7 +1131,7 @@ def push_http_simulation(self, simulation: Simulation): """ sim_data = simulation.to_model(recurse=True) - partitions = cast(dict[str, str], self._config.get_section("partition")) + partitions = _partition_roots(self._config) inputs = _expand_directories_http( sim_data.inputs.root, simulation.uuid, partitions ) diff --git a/src/simdb/cli/resumable_upload.py b/src/simdb/cli/resumable_upload.py index d6457ce5..368ad535 100644 --- a/src/simdb/cli/resumable_upload.py +++ b/src/simdb/cli/resumable_upload.py @@ -244,7 +244,9 @@ def _send_chunks( f"Upload failed after {_MAX_RETRIES} retries: {err}" ) from err logger.warning("Upload chunk failed (%s), resuming from server offset", err) - offset = _query_offset(url, auth, cookies, headers) + server_offset = _query_offset(url, auth, cookies, headers) + if server_offset is not None: + offset = server_offset continue if resp.status_code == 409: @@ -276,10 +278,19 @@ def _send_chunks( return -def _query_offset(url: str, auth, cookies, headers) -> int: - resp = requests.head( - url, headers=_base_headers(headers), auth=auth, cookies=cookies - ) +def _query_offset(url: str, auth, cookies, headers) -> Optional[int]: + """Return the server's current offset, or ``None`` if it can't be determined. + + A ``None`` result (failed/ambiguous HEAD, or a missing ``Upload-Offset`` + header) means "unknown" - the caller must keep its current offset rather than + restart the upload. + """ + try: + resp = requests.head( + url, headers=_base_headers(headers), auth=auth, cookies=cookies + ) + except (requests.ConnectionError, requests.Timeout): + return None if resp.status_code in (200, 204): - return _header_int(resp, "Upload-Offset") or 0 - return 0 + return _header_int(resp, "Upload-Offset") + return None diff --git a/src/simdb/database/models/file.py b/src/simdb/database/models/file.py index 92228dfe..2bf22948 100644 --- a/src/simdb/database/models/file.py +++ b/src/simdb/database/models/file.py @@ -139,7 +139,12 @@ def to_model_with_path(self) -> FileGetDataResponse: files = [FileInfo(path=Path(self.uri.path), checksum=self.checksum)] else: files = [ - FileInfo(path=path, checksum=file_checksum(SimDBUrl(f"file:{path}"))) + FileInfo( + path=path, + checksum=file_checksum( + SimDBUrl.build(scheme="file", path=path.as_posix()) + ), + ) for path in imas_files(self.uri) ] return FileGetDataResponse( diff --git a/src/simdb/imas/utils.py b/src/simdb/imas/utils.py index 08353d7b..8d8538fa 100644 --- a/src/simdb/imas/utils.py +++ b/src/simdb/imas/utils.py @@ -35,15 +35,16 @@ def build( fragment: Optional[str] = None, **kwargs, ) -> "SimDBUrl": - url_str = f"{scheme}:" - + authority = "" if host: - url_str += f"//{host}" - if port: - url_str += f":{port}" - url_str += "/" - - url_str += path or "" + authority = f"//{host}" + if port is not None: + authority += f":{port}" + path_part = path or "" + if authority and path_part and not path_part.startswith("/"): + path_part = f"/{path_part}" + + url_str = f"{scheme}:{authority}{path_part}" if query: url_str += f"?{query}" if fragment: @@ -223,10 +224,19 @@ def open_imas(uri: SimDBUrl) -> DBEntry: if uri.scheme == "file": imas_uri = uri.path elif uri.scheme == "imas": + # Access Layer 4 / legacy entries are opened through the dedicated + # DBEntry constructor rather than a URI string. + if not _is_al5(): + return _open_legacy(uri) + qs = dict(uri.query_params()) path = qs.get("path") if path is None: - raise ValueError(f"invalid imas URI: {uri} - no path found") + # A legacy-style URI (no explicit path query): resolve the on-disk + # path and rebuild an AL5 URI before opening. + path = get_path_for_legacy_uri(uri) + backend = qs.get("backend", "mdsplus") + uri = SimDBUrl.build(scheme="imas", path=backend, query=f"path={path}") imas_uri = str(uri) else: raise ValueError(f"invalid imas URI: {uri} - invalid scheme") diff --git a/src/simdb/remote/apis/v1_3/simulations.py b/src/simdb/remote/apis/v1_3/simulations.py index 8e7da0de..7611f655 100644 --- a/src/simdb/remote/apis/v1_3/simulations.py +++ b/src/simdb/remote/apis/v1_3/simulations.py @@ -118,13 +118,16 @@ def post( # The complete job will set simulation.ingestion_status = Completed complete = complete_ingestion_task.si(simulation.uuid) - chain = copy_files | complete - # Files uploaded over HTTP are staged in the ``http`` partition; once # copied into the upload folder, remove those staged duplicates. all_files = [*body.simulation.inputs.root, *body.simulation.outputs.root] - if any(SimDBUrl(f.uri).scheme == "http" for f in all_files): - chain = chain | cleanup_http_staging_task.si(simulation.uuid) + if all(SimDBUrl(f.uri).scheme == "http" for f in all_files): + cleanup = cleanup_http_staging_task.si(simulation.uuid) + copy_files.link_error(cleanup) + complete.link_error(cleanup) + chain = copy_files | complete | cleanup + else: + chain = copy_files | complete try: _ = chain.apply_async() diff --git a/src/simdb/remote/apis/v1_3/upload.py b/src/simdb/remote/apis/v1_3/upload.py index 79f92ea1..640b4987 100644 --- a/src/simdb/remote/apis/v1_3/upload.py +++ b/src/simdb/remote/apis/v1_3/upload.py @@ -50,6 +50,14 @@ def _max_append_size() -> int: return DEFAULT_MAX_APPEND_SIZE +def _read_capped_body(limit: int) -> Optional[bytes]: + """Read the request body, capped at ``limit`` bytes.""" + data = request.stream.read(limit + 1) + if len(data) > limit: + return None + return data + + def _bool_field(value: bool) -> str: return "?1" if value else "?0" @@ -157,8 +165,8 @@ def post(self, target: str, user: User) -> Response: final, partial = _resolve_target(target) partial.parent.mkdir(parents=True, exist_ok=True) - data = request.get_data() or b"" - if len(data) > _max_append_size(): + data = _read_capped_body(_max_append_size()) + if data is None: return Response(status=413, headers=_headers(0, False)) # Per-request integrity: reject before writing anything if the body does # not match the client's Content-Digest. @@ -205,8 +213,8 @@ def patch(self, target: str, user: User) -> Response: # Offset mismatch - tell the client our current offset so it resyncs. return Response(status=409, headers=_headers(offset, False)) - data = request.get_data() or b"" - if len(data) > _max_append_size(): + data = _read_capped_body(_max_append_size()) + if data is None: return Response(status=413, headers=_headers(offset, False)) # Per-request integrity: reject (without appending) if the body does not From 31d2aabc0e24303996b3c13b5deb8b965bd0c8a7 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Wed, 15 Jul 2026 13:25:41 +0200 Subject: [PATCH 23/31] Use hash_file in tests --- tests/remote/api/v1.3/test_simulations3.py | 4 ++-- tests/workers/test_tasks.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/remote/api/v1.3/test_simulations3.py b/tests/remote/api/v1.3/test_simulations3.py index 9fb6d9b6..ab3b63bc 100644 --- a/tests/remote/api/v1.3/test_simulations3.py +++ b/tests/remote/api/v1.3/test_simulations3.py @@ -9,6 +9,7 @@ generate_simulation_data, ) +from simdb.checksum import hash_file from simdb.cli.manifest import Manifest from simdb.config import Config from simdb.database.models import Simulation @@ -19,7 +20,6 @@ ) from simdb.workers import tasks as simdb_tasks from simdb.workers.celery import celery_app -from simdb.workers.tasks import _calculate_checksum @pytest.fixture(autouse=True) @@ -82,7 +82,7 @@ def generate_simulation_file(path) -> FileData: file_path = path / "partition/file.txt" file_path.parent.mkdir(exist_ok=True) file_path.write_text("test data") - checksum = _calculate_checksum(file_path) + checksum = hash_file(file_path) return FileData( type="FILE", uri="data:///file.txt", diff --git a/tests/workers/test_tasks.py b/tests/workers/test_tasks.py index 716014f1..d5f8950a 100644 --- a/tests/workers/test_tasks.py +++ b/tests/workers/test_tasks.py @@ -5,13 +5,13 @@ import pytest +from simdb.checksum import hash_file from simdb.config import Config from simdb.enums import IngestionStatus from simdb.imas.utils import SimDBUrl from simdb.remote.models import FileData from simdb.workers import tasks as simdb_tasks from simdb.workers.tasks import ( - _calculate_checksum, _checksum_matches, _copy_files, _create_file_from_data, @@ -201,7 +201,7 @@ def test_copy_files_task_copies_inputs_and_marks_copied(task_environment): input_files = [ _make_file_data( - f"data:/{source_file.name}", checksum=_calculate_checksum(source_file) + f"data:/{source_file.name}", checksum=hash_file(source_file) ) ] @@ -261,14 +261,14 @@ def test_copy_files_task_http_keeps_imas_folder_flattens_sibling(task_environmen output_files = [ _make_file_data( f"http://{sim_hex}/data/subdir/test_hdf5/master.h5", - checksum=_calculate_checksum(master), + checksum=hash_file(master), ), _make_file_data( f"http://{sim_hex}/data/subdir/test_hdf5/0001.h5", - checksum=_calculate_checksum(extra), + checksum=hash_file(extra), ), _make_file_data( - f"http://{sim_hex}/data/subdir/test.nc", checksum=_calculate_checksum(nc) + f"http://{sim_hex}/data/subdir/test.nc", checksum=hash_file(nc) ), ] From acc47ff1135762183f38af9ea919c872e22f9fa7 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Wed, 15 Jul 2026 13:25:41 +0200 Subject: [PATCH 24/31] Do not use partitions for push_http --- src/simdb/cli/remote_api.py | 42 ++++++++++++++++--------------------- tests/cli/test_push_http.py | 34 +++++++++++++----------------- 2 files changed, 32 insertions(+), 44 deletions(-) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 9f07b4fc..7ad6020f 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -342,17 +342,18 @@ def _expand_directories(files: Iterable[FileData], partitions: Dict[str, str]): def _expand_directories_http( - files: Iterable[FileData], sim_uuid: uuid.UUID, partitions: dict[str, str] + files: Iterable[FileData], sim_uuid: uuid.UUID ) -> List[Tuple[FileData, Path, str]]: """Expand directories / IMAS data into individual files for HTTP upload. - Returns ``(file_data, local_source_path, target)`` triples. Each file keeps - the same partition-relative layout that :func:`_expand_directories` produces - for ``push_local`` - so structure handling (IMAS directories stay grouped, - standalone files stay flat) is identical to local push. The layout is then - namespaced under ``//`` and assigned an ``http://`` URI so - the server stages it into the ``http`` partition; the server's existing copy - step strips the common root exactly as it does for local push. + Returns ``(file_data, local_source_path, target)`` triples. Structure + handling (IMAS directories stay grouped, standalone files stay flat) is + identical to local push, but unlike ``push_local`` the file bytes are + uploaded, so partitions play no role: each file keeps its absolute local + path, namespaced under ``/file/``, and is assigned an ``http://`` + URI so the server stages it into the ``http`` partition. The server's + existing copy step strips the common root exactly as it does for local + push. """ result: List[Tuple[FileData, Path, str]] = [] for file in files: @@ -371,9 +372,9 @@ def _expand_directories_http( for sub_file in file_path.iterdir(): if sub_file.is_dir(): raise ValueError("Nested directory found") - result.append(_make_http_entry(file, sub_file, sim_uuid, partitions)) + result.append(_make_http_entry(file, sub_file, sim_uuid)) else: - result.append(_make_http_entry(file, file_path, sim_uuid, partitions)) + result.append(_make_http_entry(file, file_path, sim_uuid)) return result @@ -381,21 +382,19 @@ def _make_http_entry( template: FileData, local_path: Path, sim_uuid: uuid.UUID, - partitions: dict[str, str], ) -> Tuple[FileData, Path, str]: """Build the HTTP upload entry for a single local file. - The relative path is taken from :func:`_find_partition_for_file` (the same - mapping ``push_local`` uses) and namespaced under ``//`` so - uploads from different partitions never collide on the server. + HTTP uploads carry the file bytes from the local system, so partitions are + not consulted: the file's absolute path is namespaced under + ``/file/``, which keeps targets unique on the server. The checksum is left empty here and filled in later by :func:`_compute_checksums`, so that hashing (a full read of every file) can be reported with a progress bar instead of stalling silently before the upload. """ - scheme, rel = _find_partition_for_file(local_path, partitions) - rel_posix = rel.as_posix().lstrip("/") - target = f"{sim_uuid.hex}/{scheme}/{rel_posix}" + rel_posix = local_path.as_posix().lstrip("/") + target = f"{sim_uuid.hex}/file/{rel_posix}" new_uri = SimDBUrl.build(scheme="http", path=target, host="") file_type = "IMAS" if _check_file_is_imas(local_path) else template.type return ( @@ -1131,13 +1130,8 @@ def push_http_simulation(self, simulation: Simulation): """ sim_data = simulation.to_model(recurse=True) - partitions = _partition_roots(self._config) - inputs = _expand_directories_http( - sim_data.inputs.root, simulation.uuid, partitions - ) - outputs = _expand_directories_http( - sim_data.outputs.root, simulation.uuid, partitions - ) + inputs = _expand_directories_http(sim_data.inputs.root, simulation.uuid) + outputs = _expand_directories_http(sim_data.outputs.root, simulation.uuid) files = list(itertools.chain(inputs, outputs)) upload_headers = {"User-Agent": "it_script_basic"} diff --git a/tests/cli/test_push_http.py b/tests/cli/test_push_http.py index 27e99022..24ceadc8 100644 --- a/tests/cli/test_push_http.py +++ b/tests/cli/test_push_http.py @@ -23,26 +23,24 @@ def _file_data(path) -> FileData: ) -def test_expand_directories_http_uses_partition_relative_paths(tmp_path): - # Files under a configured partition keep their partition-relative layout, - # namespaced under // (mirrors local push mapping). - partition = tmp_path / "data" - (partition / "subdir").mkdir(parents=True) - f = partition / "subdir" / "file.txt" +def test_expand_directories_http_uses_absolute_paths(tmp_path): + # HTTP uploads carry the file bytes, so partitions play no role: files keep + # their absolute local path, namespaced under /file/. + (tmp_path / "subdir").mkdir() + f = tmp_path / "subdir" / "file.txt" f.write_text("hello") sim_uuid = uuid.uuid4() - partitions = {"data": str(partition)} - result = _expand_directories_http([_file_data(f)], sim_uuid, partitions) + result = _expand_directories_http([_file_data(f)], sim_uuid) assert len(result) == 1 file_data, local_path, target = result[0] assert local_path == f - assert target == f"{sim_uuid.hex}/data/subdir/file.txt" + assert target == f"{sim_uuid.hex}/file/{str(f).lstrip('/')}" parsed = SimDBUrl(file_data.uri) assert parsed.scheme == "http" assert parsed.host == sim_uuid.hex - assert parsed.path == "/data/subdir/file.txt" + assert parsed.path == f"/file/{str(f).lstrip('/')}" assert file_data.type == "FILE" assert file_data.checksum == "" @@ -65,13 +63,11 @@ def test_compute_checksums_populates_sha1(tmp_path): def test_expand_directories_http_keeps_imas_directory(tmp_path): # An IMAS (hdf5) directory must stay contained in its own folder. - partition = tmp_path / "data" - imas_dir = partition / "run" / "myids" + imas_dir = tmp_path / "run" / "myids" imas_dir.mkdir(parents=True) (imas_dir / "master.h5").write_text("m") (imas_dir / "0001.h5").write_text("d") sim_uuid = uuid.uuid4() - partitions = {"data": str(partition)} imas_file = FileData( type="IMAS", @@ -82,23 +78,21 @@ def test_expand_directories_http_keeps_imas_directory(tmp_path): datetime=datetime.now(timezone.utc), ) - result = _expand_directories_http([imas_file], sim_uuid, partitions) + result = _expand_directories_http([imas_file], sim_uuid) + prefix = f"{sim_uuid.hex}/file/{str(imas_dir).lstrip('/')}" targets = sorted(t for _, _, t in result) - assert targets == [ - f"{sim_uuid.hex}/data/run/myids/0001.h5", - f"{sim_uuid.hex}/data/run/myids/master.h5", - ] + assert targets == [f"{prefix}/0001.h5", f"{prefix}/master.h5"] assert all(file_data.type == "IMAS" for file_data, _, _ in result) def test_expand_directories_http_unpartitioned_file_uses_file_scheme(tmp_path): - # A file outside any partition falls back to the "file" namespace. + # No partition configuration is needed for HTTP uploads. f = tmp_path / "loose.txt" f.write_text("x") sim_uuid = uuid.uuid4() - result = _expand_directories_http([_file_data(f)], sim_uuid, {}) + result = _expand_directories_http([_file_data(f)], sim_uuid) _, _, target = result[0] assert target == f"{sim_uuid.hex}/file/{str(f).lstrip('/')}" From 903d211ca829330e003310de7985b67c0ac49b66 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Wed, 15 Jul 2026 13:28:35 +0200 Subject: [PATCH 25/31] Ruff --- src/simdb/cli/remote_api.py | 4 ++-- src/simdb/workers/tasks.py | 1 - tests/workers/test_tasks.py | 5 ++--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 7ad6020f..3d69bcf9 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -25,11 +25,12 @@ Tuple, Union, ) -from urllib.parse import ParseResult, quote, urlparse +from urllib.parse import quote, urlparse import appdirs import click import requests +from netCDF4 import Dataset from requests.auth import AuthBase from rich.progress import ( BarColumn, @@ -39,7 +40,6 @@ TimeRemainingColumn, TransferSpeedColumn, ) -from netCDF4 import Dataset from semantic_version import Version from simdb.checksum import CHECKSUM_ALGORITHM, READ_CHUNK_SIZE, hash_file diff --git a/src/simdb/workers/tasks.py b/src/simdb/workers/tasks.py index 3895898e..9243cddb 100644 --- a/src/simdb/workers/tasks.py +++ b/src/simdb/workers/tasks.py @@ -118,7 +118,6 @@ def _checksum_matches(path: Path, expected: str) -> bool: return hash_file(path) == expected - def _get_imas_identifier_path(path: Path) -> Path: if path.suffix == ".nc": return path diff --git a/tests/workers/test_tasks.py b/tests/workers/test_tasks.py index d5f8950a..22de3a33 100644 --- a/tests/workers/test_tasks.py +++ b/tests/workers/test_tasks.py @@ -200,9 +200,7 @@ def test_copy_files_task_copies_inputs_and_marks_copied(task_environment): source_file.write_text("test content") input_files = [ - _make_file_data( - f"data:/{source_file.name}", checksum=hash_file(source_file) - ) + _make_file_data(f"data:/{source_file.name}", checksum=hash_file(source_file)) ] copy_files_task(env["simulation_uuid"], input_files, []) @@ -239,6 +237,7 @@ def test_notify_watchers_noop_without_watchers(): delay.assert_not_called() + def test_copy_files_task_http_keeps_imas_folder_flattens_sibling(task_environment): """HTTP-staged files are copied like local push: a shared root is stripped, so an IMAS directory keeps its folder while a sibling file stays flat.""" From daae3982bf770846e7bae2d4849a789c4d941ff5 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 13:49:59 +0200 Subject: [PATCH 26/31] Pass add_watcher flag through push_http --- src/simdb/cli/commands/simulation.py | 2 +- src/simdb/cli/remote_api.py | 4 ++-- tests/cli/test_push_http.py | 34 +++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/simdb/cli/commands/simulation.py b/src/simdb/cli/commands/simulation.py index cb1992f8..0679fd46 100644 --- a/src/simdb/cli/commands/simulation.py +++ b/src/simdb/cli/commands/simulation.py @@ -370,7 +370,7 @@ def simulation_push_http( except ValidationError as err: raise click.ClickException(f"Simulation does not validate: {err}") from err - api.push_http_simulation(simulation) + api.push_http_simulation(simulation, add_watcher=add_watcher) click.echo("Waiting for ingestion to complete...", nl=False) last_status = None diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 3d69bcf9..82dcf623 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -1121,7 +1121,7 @@ def _on_progress(completed: int, _base: int = uploaded) -> None: progress.update(overall, completed=uploaded) @try_request - def push_http_simulation(self, simulation: Simulation): + def push_http_simulation(self, simulation: Simulation, add_watcher: bool = False): """Push a simulation by uploading its files over resumable HTTP. Unlike :meth:`push_local_simulation` (which requires a filesystem shared @@ -1146,7 +1146,7 @@ def push_http_simulation(self, simulation: Simulation): headers = {"Content-type": "application/json", "User-Agent": "it_script_basic"} post_data = SimulationPostData( - simulation=sim_data, add_watcher=False, uploaded_by=uploaded_by + simulation=sim_data, add_watcher=add_watcher, uploaded_by=uploaded_by ).model_dump_json() res = requests.post( f"{self._url}/v1.3/simulations", diff --git a/tests/cli/test_push_http.py b/tests/cli/test_push_http.py index 24ceadc8..69c3bee1 100644 --- a/tests/cli/test_push_http.py +++ b/tests/cli/test_push_http.py @@ -120,10 +120,42 @@ def test_push_http_command_pushes_and_reports_success(tmp_path): ) assert result.exit_code == 0, result.output - fake_api.push_http_simulation.assert_called_once_with(sim) + fake_api.push_http_simulation.assert_called_once_with(sim, add_watcher=False) assert "Successfully pushed simulation" in result.output +def test_push_http_command_passes_add_watcher(tmp_path): + runner = CliRunner() + config_file = config_test_file() + + fake_api = mock.MagicMock() + fake_api.get_validation_schemas.return_value = [] + fake_api.get_ingestion_status.return_value = "completed" + + sim = mock.MagicMock() + sim.uuid = uuid.uuid4() + fake_db = mock.MagicMock() + fake_db.get_simulation.return_value = sim + + with mock.patch( + "simdb.cli.commands.simulation.RemoteAPI", return_value=fake_api + ), mock.patch("simdb.cli.commands.simulation.get_local_db", return_value=fake_db): + result = runner.invoke( + cli, + [ + f"--config-file={config_file}", + "simulation", + "push_http", + "iter", + "sim1", + "--add-watcher", + ], + ) + + assert result.exit_code == 0, result.output + fake_api.push_http_simulation.assert_called_once_with(sim, add_watcher=True) + + def test_push_http_command_fails_on_failed_status(tmp_path): runner = CliRunner() config_file = config_test_file() From e89ecc9764c4b51871362bd209ec8016dcc06c3e Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 13:50:22 +0200 Subject: [PATCH 27/31] Do not record uploaded_by as the string 'None' in push_http str() around a missing meta value produced a truthy "None" that suppressed the server's fallback to the authenticated user. --- src/simdb/cli/remote_api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 82dcf623..3842b31e 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -1142,11 +1142,13 @@ def push_http_simulation(self, simulation: Simulation, add_watcher: bool = False sim_data.inputs.root = [file_data for file_data, _, _ in inputs] sim_data.outputs.root = [file_data for file_data, _, _ in outputs] - uploaded_by = str(simulation.meta_dict().get("uploaded_by", None)) + uploaded_by = simulation.meta_dict().get("uploaded_by") headers = {"Content-type": "application/json", "User-Agent": "it_script_basic"} post_data = SimulationPostData( - simulation=sim_data, add_watcher=add_watcher, uploaded_by=uploaded_by + simulation=sim_data, + add_watcher=add_watcher, + uploaded_by=str(uploaded_by) if uploaded_by is not None else None, ).model_dump_json() res = requests.post( f"{self._url}/v1.3/simulations", From 9954e927b6344c6051a6e880aca05b26383f98f4 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 13:51:06 +0200 Subject: [PATCH 28/31] Use request helpers and negotiated URL in push_http_simulation The hand-rolled requests.post bypassed the auth gating on self._server_auth, the gzip compression for large simulations payloads, and the negotiated self._api_url. The resumable upload calls now gate auth the same way. --- src/simdb/cli/remote_api.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 3842b31e..f2033aa1 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -1102,7 +1102,7 @@ def _upload_files( progress.reset( file_task, total=size, description=f" {local_path.name}" ) - url = f"{self._url}/v1.3/upload/{quote(target)}" + url = f"{self._api_url}upload/{quote(target)}" def _on_progress(completed: int, _base: int = uploaded) -> None: progress.update(file_task, completed=completed) @@ -1111,7 +1111,7 @@ def _on_progress(completed: int, _base: int = uploaded) -> None: resumable_upload( url, local_path, - auth=self._get_auth(), + auth=self._get_auth() if self._server_auth != "None" else None, cookies=self._cookies, headers=upload_headers, progress=_on_progress, @@ -1120,6 +1120,7 @@ def _on_progress(completed: int, _base: int = uploaded) -> None: progress.update(file_task, completed=size) progress.update(overall, completed=uploaded) + @versioned_method("v1.3") @try_request def push_http_simulation(self, simulation: Simulation, add_watcher: bool = False): """Push a simulation by uploading its files over resumable HTTP. @@ -1144,20 +1145,12 @@ def push_http_simulation(self, simulation: Simulation, add_watcher: bool = False uploaded_by = simulation.meta_dict().get("uploaded_by") - headers = {"Content-type": "application/json", "User-Agent": "it_script_basic"} post_data = SimulationPostData( simulation=sim_data, add_watcher=add_watcher, uploaded_by=str(uploaded_by) if uploaded_by is not None else None, - ).model_dump_json() - res = requests.post( - f"{self._url}/v1.3/simulations", - data=post_data, - headers=headers, - auth=self._get_auth(), - cookies=self._cookies, ) - check_return(res) + self.post("simulations", data=post_data.model_dump(mode="json")) @versioned_method("v1.3") @try_request From 05299c7a936fa4de16888d1264cd0751fdc622b6 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 13:51:58 +0200 Subject: [PATCH 29/31] Remove unused xxhash dependency --- pyproject.toml | 1 - uv.lock | 2 -- 2 files changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ad4c8ab3..44a29a20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,6 @@ dependencies = [ "sqlalchemy>=1.2.12,<2.0", "alembic~=1.13", "rich>=14.3.3", - "xxhash>=3.7.0", ] [project.optional-dependencies] diff --git a/uv.lock b/uv.lock index b84ab266..532738e0 100644 --- a/uv.lock +++ b/uv.lock @@ -3013,7 +3013,6 @@ dependencies = [ { name = "rich", version = "15.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, { name = "semantic-version" }, { name = "sqlalchemy" }, - { name = "xxhash" }, ] [package.optional-dependencies] @@ -3168,7 +3167,6 @@ requires-dist = [ { name = "sphinx-autodoc-typehints", marker = "extra == 'build-docs'", specifier = ">=1.12.0" }, { name = "sphinx-immaterial", marker = "extra == 'build-docs'", specifier = ">=0.11.14" }, { name = "sqlalchemy", specifier = ">=1.2.12,<2.0" }, - { name = "xxhash", specifier = ">=3.7.0" }, ] provides-extras = ["server", "auth-ad", "auth-keycloak", "auth-ldap", "auth", "imas-validator", "build-docs", "postgres", "all"] From 46f57580687b1cff6f2ced19c84bdf14624d75b2 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Tue, 4 Aug 2026 15:23:44 +0200 Subject: [PATCH 30/31] Fix 404 on versioned API root endpoints --- src/simdb/remote/apis/__init__.py | 36 ++++++++++++++++++------------- tests/remote/api/test_index.py | 16 ++++++++++++++ 2 files changed, 37 insertions(+), 15 deletions(-) create mode 100644 tests/remote/api/test_index.py diff --git a/src/simdb/remote/apis/__init__.py b/src/simdb/remote/apis/__init__.py index b75a9680..2f8a7803 100644 --- a/src/simdb/remote/apis/__init__.py +++ b/src/simdb/remote/apis/__init__.py @@ -36,6 +36,26 @@ def register(api, version, namespaces): version_str = version.replace(".", "_") blueprint = Blueprint(f"api_{version_str}", f"{__name__}.{version_str}") blueprints[version] = blueprint + + def index(): + return jsonify( + { + "api": "simdb", + "api_version": api.version, + "server_version": __version__, + "endpoints": [ + request.url + "simulations", + request.url + "files", + request.url + "validation_schema", + request.url + "metadata", + request.url + "upload_options", + ], + "documentation": request.url + "docs", + } + ) + + api.render_root = index + api.init_app(blueprint) for namespace in namespaces: @@ -78,21 +98,7 @@ def handle_authentication_error(err: Exception): class Index(Resource): @api.doc(security=[]) def get(self): - return jsonify( - { - "api": "simdb", - "api_version": api.version, - "server_version": __version__, - "endpoints": [ - request.url + "simulations", - request.url + "files", - request.url + "validation_schema", - request.url + "metadata", - request.url + "upload_options", - ], - "documentation": request.url + "docs", - } - ) + return index() @api.route("/token") class Token(Resource): diff --git a/tests/remote/api/test_index.py b/tests/remote/api/test_index.py new file mode 100644 index 00000000..608fadcc --- /dev/null +++ b/tests/remote/api/test_index.py @@ -0,0 +1,16 @@ +import pytest + +from simdb.remote.apis import blueprints + + +@pytest.mark.parametrize("version", list(blueprints)) +def test_versioned_index(client, version): + """The versioned root must serve the index JSON, not flask-restx's 404 root.""" + rv = client.get(f"/{version}/") + + assert rv.status_code == 200 + assert rv.json["api"] == "simdb" + # v1 reports "1.0" for blueprint key "v1", so only compare the prefix + assert rv.json["api_version"].startswith(version.lstrip("v")) + assert "server_version" in rv.json + assert any(url.endswith("simulations") for url in rv.json["endpoints"]) From 1db3f4fdc3219495377d2e9eac5a7da557f13f10 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Wed, 5 Aug 2026 16:30:12 +0200 Subject: [PATCH 31/31] Use status enum --- src/simdb/cli/commands/simulation.py | 32 +++++++++++++--------------- src/simdb/cli/remote_api.py | 3 ++- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/simdb/cli/commands/simulation.py b/src/simdb/cli/commands/simulation.py index 0679fd46..7aab39dc 100644 --- a/src/simdb/cli/commands/simulation.py +++ b/src/simdb/cli/commands/simulation.py @@ -264,12 +264,6 @@ def simulation_push_local( api.push_local_simulation(simulation, add_watcher=add_watcher) - terminal_statuses = { - IngestionStatus.COMPLETED.value, - IngestionStatus.COPY_FAILED.value, - IngestionStatus.VALIDATION_FAILED.value, - } - max_consecutive_failures = 5 click.echo("Waiting for ingestion to complete...", nl=False) @@ -300,28 +294,30 @@ def simulation_push_local( if status != last_status: if last_status is not None: - click.echo(f" -> {status}", nl=False) + click.echo(f" -> {status.value}", nl=False) else: - click.echo(f" {status}", nl=False) + click.echo(f" {status.value}", nl=False) last_status = status - if status in terminal_statuses: + if status.is_terminal(): break if time.monotonic() >= deadline: click.echo() raise click.ClickException( f"Timed out after {timeout:g}s waiting for ingestion to complete " - f"(last status: {status})" + f"(last status: {status.value})" ) time.sleep(1) click.echo() - if status == IngestionStatus.COMPLETED.value: + if status == IngestionStatus.COMPLETED: click.echo(f"Successfully pushed simulation {simulation.uuid}") else: - raise click.ClickException(f"Simulation ingestion failed with status: {status}") + raise click.ClickException( + f"Simulation ingestion failed with status: {status.value}" + ) @simulation.command("push_http", cls=n_required_args_adaptor(1)) @@ -385,21 +381,23 @@ def simulation_push_http( if status != last_status: if last_status is not None: - click.echo(f" -> {status}", nl=False) + click.echo(f" -> {status.value}", nl=False) else: - click.echo(f" {status}", nl=False) + click.echo(f" {status.value}", nl=False) last_status = status - if status in ("completed", "copy_failed", "validation_failed"): + if status.is_terminal(): break time.sleep(1) click.echo() - if status == "completed": + if status == IngestionStatus.COMPLETED: click.echo(f"Successfully pushed simulation {simulation.uuid}") else: - raise click.ClickException(f"Simulation ingestion failed with status: {status}") + raise click.ClickException( + f"Simulation ingestion failed with status: {status.value}" + ) @simulation.command("push", cls=n_required_args_adaptor(1)) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index f2033aa1..abb1e290 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -46,6 +46,7 @@ from simdb.cli.resumable_upload import resumable_upload from simdb.config import Config from simdb.database.models import Simulation +from simdb.enums import IngestionStatus from simdb.imas.utils import SimDBUrl, imas_backend_for_directory, imas_files from simdb.json import CustomDecoder, CustomEncoder from simdb.remote import CLIENT_API_VERSIONS, APIConstants @@ -1156,7 +1157,7 @@ def push_http_simulation(self, simulation: Simulation, add_watcher: bool = False @try_request def get_ingestion_status(self, sim_id: str) -> str: res = self.get(f"simulation/status/{sim_id}") - return res.json()["status"] + return IngestionStatus(res.json()["status"]) @versioned_method("v1.2", "v1.3") @try_request