From 0dfc9a2e59bfc7c234adc8f76b2b98791fe0f556 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 30 Jul 2026 11:05:04 +0200 Subject: [PATCH 1/2] Add model for simulation filters --- src/simdb/remote/apis/v1_2/simulations.py | 3 ++ src/simdb/remote/models.py | 51 +++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/src/simdb/remote/apis/v1_2/simulations.py b/src/simdb/remote/apis/v1_2/simulations.py index 169248af..cce8e2dd 100644 --- a/src/simdb/remote/apis/v1_2/simulations.py +++ b/src/simdb/remote/apis/v1_2/simulations.py @@ -24,6 +24,7 @@ from simdb.remote.core.pydantic_utils import ( Body, Header, + Query, ResponseException, pydantic_validate, ) @@ -42,6 +43,7 @@ SimulationPatchResponse, SimulationPostData, SimulationPostResponse, + SimulationQueryParams, SimulationTraceData, StatusPatchData, ValidationResult, @@ -170,6 +172,7 @@ def get( self, user: User, pagination: Annotated[PaginationData, Header()], + filters: Annotated[SimulationQueryParams, Query()], ) -> PaginatedResponse[SimulationListItem]: names = [] constraints = [] diff --git a/src/simdb/remote/models.py b/src/simdb/remote/models.py index eb3827ae..9c37b22a 100644 --- a/src/simdb/remote/models.py +++ b/src/simdb/remote/models.py @@ -12,6 +12,7 @@ List, Literal, Optional, + Tuple, TypeVar, Union, ) @@ -36,6 +37,7 @@ ) from simdb.cli.manifest import DataType +from simdb.query import QueryType, parse_query_arg HexUUID = Annotated[UUID, PlainSerializer(lambda x: x.hex, return_type=str)] """UUID serialized as a hex string.""" @@ -626,6 +628,55 @@ def _strip_path(cls, v: Any) -> str: return v +class SimulationQueryParams(BaseModel): + """Metadata filters for the simulation-list endpoint. + + Every query parameter is treated as a metadata constraint: the parameter + name is a metadata key and its value is a query expression (for example + ``status=passed`` or ``runtime=gt:100``). The same key may be supplied more + than once to apply several constraints. The ``alias`` and ``uuid`` keys are + declared explicitly because they filter on a simulation's identity and are + not reported as additional metadata columns in the response. Any other + (arbitrary) key is accepted and captured as an extra field. + """ + + model_config = ConfigDict(extra="allow", use_attribute_docstrings=True) + + alias: Optional[str] = None + """Filter by simulation alias.""" + + uuid: Optional[str] = None + """Filter by simulation UUID.""" + + def constraints(self) -> Tuple[List[str], List[Tuple[str, str, QueryType]]]: + """Parse the query parameters into metadata query constraints. + + Returns a ``(names, constraints)`` pair where *names* is the list of + metadata keys to include in the response (every filtered key except the + identity keys ``alias`` and ``uuid``) and *constraints* is the list of + ``(key, value, type)`` tuples to query the database with. + """ + # Arbitrary metadata keys arrive as extra fields; the identity keys are + # declared fields, so merge them back in before building constraints. + filters = dict(self.model_extra or {}) + for key in ("alias", "uuid"): + value = getattr(self, key) + if value is not None: + filters[key] = value + + names: List[str] = [] + constraints: List[Tuple[str, str, QueryType]] = [] + for name, raw in filters.items(): + if name not in ("alias", "uuid"): + names.append(name) + values = raw if isinstance(raw, list) else [raw] + for value in values: + constraint = parse_query_arg(str(value)) + if constraint[0] or constraint[1] == QueryType.EXIST: + constraints.append((name, *constraint)) + return names, constraints + + class QuantityData(BaseModel): """A named, unit-bearing data quantity (field value or coordinate).""" From 48b78c7243520c634db65dff3a9fe94af5f012c3 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Thu, 30 Jul 2026 11:05:26 +0200 Subject: [PATCH 2/2] Add documentation --- src/simdb/remote/apis/__init__.py | 23 +++++ src/simdb/remote/apis/files.py | 30 ++++++ src/simdb/remote/apis/metadata.py | 11 +++ src/simdb/remote/apis/v1_2/__init__.py | 7 ++ src/simdb/remote/apis/v1_2/simulations.py | 115 +++++++++++++++++++--- src/simdb/remote/apis/watchers.py | 18 ++++ 6 files changed, 189 insertions(+), 15 deletions(-) diff --git a/src/simdb/remote/apis/__init__.py b/src/simdb/remote/apis/__init__.py index b75a9680..e1cb8b21 100644 --- a/src/simdb/remote/apis/__init__.py +++ b/src/simdb/remote/apis/__init__.py @@ -78,6 +78,12 @@ def handle_authentication_error(err: Exception): class Index(Resource): @api.doc(security=[]) def get(self): + """Describe the API root. + + Returns basic information about this API version, including the + server version and the URLs of the top-level endpoints and the + documentation. Requires no authentication. + """ return jsonify( { "api": "simdb", @@ -101,6 +107,12 @@ class Token(Resource): @api.response(401, "Unauthorized") @requires_auth() def get(self, user: User): + """Issue an authentication token. + + Exchanges HTTP basic-auth credentials for a signed JWT that can be + used to authenticate subsequent requests. The token expires after + the server-configured lifetime. + """ auth = request.authorization if auth is None: return error("Authorization invalid") @@ -128,6 +140,11 @@ def get(self, user: User): class ValidationSchema(Resource): @requires_auth() def get(self, user: User): + """Return the configured validation schemas. + + Returns the metadata validation schemas the server applies to + simulations, as configured on the server. + """ config = current_app.simdb_config return jsonify(Validator.validation_schemas(config, None)) @@ -135,6 +152,12 @@ def get(self, user: User): class UploadOptions(Resource): @requires_auth() def get(self, user: User): + """Return the server's upload options. + + Returns the server-side upload behaviour flags clients should honour + when pushing simulations, such as whether files and IMAS data are + copied onto the server. + """ config = current_app.simdb_config options = { "copy_files": config.get_option("server.copy_files", default=True), diff --git a/src/simdb/remote/apis/files.py b/src/simdb/remote/apis/files.py index cd7cad5e..48a9df17 100644 --- a/src/simdb/remote/apis/files.py +++ b/src/simdb/remote/apis/files.py @@ -177,11 +177,24 @@ class FileList(Resource): @requires_auth() @pydantic_validate(api) def get(self, user: User) -> FileDataList: + """List all registered files. + + Returns every input and output file known to the database across all + simulations. + """ files = current_app.db.list_files() return FileDataList.model_validate([file.to_model() for file in files]) @requires_auth() def post(self, user: User): + """Register or upload simulation files. + + Handles two content types. A JSON body registers file metadata and + verifies each file's checksum against the copy already present in the + simulation's staging directory. A multipart form upload streams file + content (optionally gzip-compressed and chunked) into the staging + directory ahead of registration. + """ try: if request.is_json: body = FileRegistrationData.model_validate_json(request.get_data()) @@ -197,6 +210,11 @@ class File(Resource): @requires_auth() @pydantic_validate(api) def get(self, file_uuid: str, user: Optional[User] = None) -> FileGetDataResponse: + """Retrieve a single file's metadata. + + Returns the stored record for the file identified by ``file_uuid``, + including its resolved on-disk path. + """ file = current_app.db.get_file(file_uuid) return file.to_model_with_path() @@ -205,6 +223,12 @@ def get(self, file_uuid: str, user: Optional[User] = None) -> FileGetDataRespons class NonIMASFileDownload(Resource): @requires_auth() def get(self, file_uuid: str, user: Optional[User] = None): + """Download a non-IMAS file. + + Streams the raw contents of the plain (non-IMAS) file identified by + ``file_uuid`` back to the client, with a MIME type inferred from the + file itself. + """ try: file: models.File = current_app.db.get_file(file_uuid) if file.type != DataType.FILE: @@ -221,6 +245,12 @@ def get(self, file_uuid: str, user: Optional[User] = None): class FileDownload(Resource): @requires_auth() def get(self, file_uuid: str, file_index: int, user: Optional[User] = None): + """Download one file from a file entry by index. + + Streams a single physical file back to the client. For a plain file + only index ``0`` is valid. For an IMAS entry, which maps to several + physical files, ``file_index`` selects which one to download. + """ try: file: models.File = current_app.db.get_file(file_uuid) if file.type == DataType.FILE: diff --git a/src/simdb/remote/apis/metadata.py b/src/simdb/remote/apis/metadata.py index ed593949..e36879a4 100644 --- a/src/simdb/remote/apis/metadata.py +++ b/src/simdb/remote/apis/metadata.py @@ -13,6 +13,12 @@ class MetaData(Resource): @cache.cached(key_prefix=cache_key) # type: ignore @pydantic_validate(api) def get(self) -> MetadataKeyInfoList: + """List all metadata keys. + + Returns every distinct metadata key present across the database, + together with information about each key. Use this to discover which + keys are available to filter or sort simulations by. + """ return MetadataKeyInfoList.model_validate(current_app.db.list_metadata_keys()) @@ -21,6 +27,11 @@ class MetaDataValues(Resource): @cache.cached(key_prefix=cache_key) # type: ignore @pydantic_validate(api) def get(self, name: str) -> MetadataValueList: + """List all values for a metadata key. + + Returns every distinct value stored across the database for the given + metadata key ``name``. + """ return MetadataValueList.model_validate( current_app.db.list_metadata_values(name) ) diff --git a/src/simdb/remote/apis/v1_2/__init__.py b/src/simdb/remote/apis/v1_2/__init__.py index 920f303b..5a31ce9f 100644 --- a/src/simdb/remote/apis/v1_2/__init__.py +++ b/src/simdb/remote/apis/v1_2/__init__.py @@ -40,6 +40,13 @@ class StagingDirectory(Resource): @requires_auth() @pydantic_validate(api) def get(self, sim_hex: str, user: User) -> StagingDirectoryResponse: + """Get (and create) a staging directory for file uploads. + + Returns the staging directory path clients should upload simulation + files to before ingesting a simulation. Without ``sim_hex`` the base + upload folder is returned. With a ``sim_hex`` the per-simulation + subdirectory is created if needed and its path returned. + """ upload_dir = current_app.simdb_config.get_string_option( "server.user_upload_folder", default=None ) diff --git a/src/simdb/remote/apis/v1_2/simulations.py b/src/simdb/remote/apis/v1_2/simulations.py index cce8e2dd..02f5846b 100644 --- a/src/simdb/remote/apis/v1_2/simulations.py +++ b/src/simdb/remote/apis/v1_2/simulations.py @@ -5,9 +5,9 @@ import tarfile from io import BytesIO from pathlib import Path -from typing import Annotated, List, Optional, Tuple +from typing import Annotated, Optional -from flask import request, send_file +from flask import send_file from flask_restx import Namespace, Resource from simdb.database import DatabaseError @@ -15,7 +15,6 @@ from simdb.database.models import watcher as models_watcher from simdb.email.server import EmailServer from simdb.imas.utils import SimDBUrl, convert_uri -from simdb.query import QueryType, parse_query_arg from simdb.remote.core.alias import create_alias_dir from simdb.remote.core.auth import User, requires_auth from simdb.remote.core.cache import cache, cache_key, clear_cache @@ -167,6 +166,23 @@ def get_meta_val(key, default=None): class SimulationList(Resource): @requires_auth() @pydantic_validate(api) + @api.doc( + params={ + "": { + "description": ( + "Any metadata key may be supplied as a query parameter to " + "filter on that metadata. The value is matched for equality " + "by default, or may use a ``comparator:value`` expression " + "(comparators: eq, ne, in, ni, gt, ge, lt, le, agt, age, " + "alt, ale, exist), e.g. ``status=passed`` or " + "``runtime=gt:100``. Repeat a key to apply several " + "constraints." + ), + "in": "query", + "type": "string", + }, + } + ) # @cache.cached(key_prefix=cache_key) def get( self, @@ -174,18 +190,21 @@ def get( pagination: Annotated[PaginationData, Header()], filters: Annotated[SimulationQueryParams, Query()], ) -> PaginatedResponse[SimulationListItem]: - names = [] - constraints = [] - if request.args: - constraints: List[Tuple[str, str, QueryType]] = [] - for name in request.args: - if name not in ("alias", "uuid"): - names.append(name) - values = request.args.getlist(name) - for value in values: - constraint = parse_query_arg(value) - if constraint[0] or constraint[1] == QueryType.EXIST: - constraints.append((name, *constraint)) + """List simulations, optionally filtered by metadata. + + Returns a paginated list of simulations. Query parameters are + interpreted as metadata constraints, so passing a metadata key and a + query value (for example ``status=passed`` or ``runtime=gt:100``) + filters the results to matching simulations. Values are matched for + equality by default, or may use a ``comparator:value`` expression + (comparators: ``eq``, ``ne``, ``in``, ``ni``, ``gt``, ``ge``, ``lt``, + ``le``, ``agt``, ``age``, ``alt``, ``ale``, ``exist``). The special + ``alias`` and ``uuid`` parameters filter on the simulation's identity + rather than its metadata. Without any query parameters all simulations + are returned. Use the pagination headers to control page size, page + number and sorting. + """ + names, constraints = filters.constraints() if constraints: count, data = current_app.db.query_meta_data( @@ -221,6 +240,17 @@ def post( user: User, body: Annotated[SimulationPostData, Body()], ) -> SimulationPostResponse: + """Ingest (upload) a new simulation. + + Registers a simulation and its input and output files in the database. + The upload timestamp and the uploading user are recorded automatically. + If the server is configured to copy files, referenced files are moved + from the per-simulation staging directory into permanent storage. When + auto-validation is enabled the simulation is validated on ingest and the + result is returned. If the simulation declares that it replaces an + earlier one, the replaced simulation is marked deprecated. An alias may + be requested; aliases ending in ``-`` or ``#`` are auto-numbered. + """ simulation = models_sim.Simulation.from_data_model(body.simulation) # Simulation Upload (Push) Date @@ -347,6 +377,13 @@ class Simulation(Resource): @cache.cached(key_prefix=cache_key) # type: ignore @pydantic_validate(api) def get(self, sim_id: str, user: User) -> SimulationDataResponse: + """Retrieve a single simulation by id or alias. + + Returns the full simulation record, including its input and output + files and metadata, together with references to its parent and child + simulations. The ``sim_id`` path parameter accepts either a simulation + UUID or an alias. + """ try: simulation = current_app.db.get_simulation(sim_id) except DatabaseError: @@ -368,6 +405,12 @@ def patch( user: Optional[User], body: Annotated[StatusPatchData, Body()], ) -> SimulationPatchResponse: + """Update a simulation's status. + + Sets the simulation's status to the value given in the request body. + If the status changes, any users watching the simulation are notified + by email. Requires admin privileges. + """ simulation = current_app.db.get_simulation(sim_id) if simulation is None: raise ResponseException(f"Simulation {sim_id} not found.") @@ -380,6 +423,13 @@ def patch( @requires_auth("admin") @pydantic_validate(api) def delete(self, sim_id: str, user: User) -> SimulationDeleteResponse: + """Delete a simulation and its stored files. + + Removes the simulation from the database and deletes its staging + directory and any alias symlink from disk. Returns the deleted + simulation's id and the list of files that were removed. Requires admin + privileges. + """ simulation = current_app.db.delete_simulation(sim_id) clear_cache() @@ -409,6 +459,11 @@ class SimulationMeta(Resource): @cache.cached(key_prefix=cache_key) # type: ignore @pydantic_validate(api) def get(self, sim_id: str, user: User) -> MetadataDataList: + """List a simulation's metadata. + + Returns all metadata entries (key/value pairs) attached to the + simulation identified by ``sim_id`` (UUID or alias). + """ simulation = current_app.db.get_simulation(sim_id) if simulation: return MetadataDataList.model_validate( @@ -424,6 +479,13 @@ def patch( user: Optional[User], body: Annotated[MetadataPatchData, Body()], ) -> MetadataDataList: + """Set or update a metadata entry on a simulation. + + Writes the given metadata key/value pair to the simulation and returns + the previous value(s) for that key. Updating the ``status`` key routes + through the status-change logic and notifies watchers. Requires admin + privileges. + """ key = body.key value = body.value.lower() simulation = current_app.db.get_simulation(sim_id) @@ -450,6 +512,11 @@ def delete( user: Optional[User], body: Annotated[MetadataDeleteData, Body()], ) -> MetadataDeleteResponse: + """Remove a metadata entry from a simulation. + + Deletes the metadata key given in the request body from the simulation. + Requires admin privileges. + """ simulation = current_app.db.get_simulation(sim_id) if simulation is None: raise ResponseException(f"Simulation {sim_id} not found.") @@ -465,6 +532,12 @@ class ValidateSimulation(Resource): @requires_auth() @pydantic_validate(api) def post(self, sim_id, user: User) -> ValidationResult: + """Validate a simulation against its schemas. + + Runs the configured metadata and file validators against the simulation + and updates its status to passed or failed accordingly. Returns whether + validation passed along with any validation error message. + """ simulation = current_app.db.get_simulation(sim_id) result = _validate(simulation, user) current_app.db.insert_simulation(simulation) @@ -478,6 +551,13 @@ class SimulationTrace(Resource): @cache.cached(key_prefix=cache_key) # type: ignore @pydantic_validate(api) def get(self, sim_id: str, user: User) -> SimulationTraceData: + """Trace a simulation's provenance chain. + + Returns the simulation's trace data, recursively resolving the chain of + simulations it replaces so the full deprecation and replacement history + can be followed. Includes status, status timestamps and replacement + reasons. + """ return _build_trace(sim_id) @@ -485,6 +565,11 @@ def get(self, sim_id: str, user: User) -> SimulationTraceData: class SimulationPackage(Resource): @requires_auth() def get(self, sim_id: str, user: User): + """Download a simulation's files as a gzipped tar archive. + + Packages the simulation's staging directory into a ``.tar.gz`` archive + and streams it back as an ``application/x-gzip`` download. + """ try: simulation = current_app.db.get_simulation(sim_id) diff --git a/src/simdb/remote/apis/watchers.py b/src/simdb/remote/apis/watchers.py index 51940aad..77674be5 100644 --- a/src/simdb/remote/apis/watchers.py +++ b/src/simdb/remote/apis/watchers.py @@ -26,6 +26,13 @@ class Watcher(Resource): def post( self, sim_id: str, user: User, data: Annotated[WatcherPostRequest, Body()] ) -> WatcherPostResponse: + """Add a watcher to a simulation. + + Registers a user to be notified (by email) about changes to the + simulation identified by ``sim_id``. The watcher's username, email and + notification level default to those of the authenticated user when not + supplied in the request body. + """ username = data.user or user.name email = data.email or user.email @@ -48,6 +55,12 @@ def post( def delete( self, sim_id: str, user: User, data: Annotated[WatcherDeleteRequest, Body()] ) -> WatcherDeleteResponse: + """Remove a watcher from a simulation. + + Stops notifying the given user about changes to the simulation + identified by ``sim_id``. The username defaults to the authenticated + user when not supplied in the request body. + """ username = data.user or user.name current_app.db.remove_watcher(sim_id, username) @@ -59,6 +72,11 @@ def delete( @requires_auth() @pydantic_validate(api) def get(self, sim_id: str, user: User) -> WatcherGetResponse: + """List the watchers of a simulation. + + Returns every user currently watching the simulation identified by + ``sim_id``, along with their notification settings. + """ return WatcherGetResponse( [watcher.to_model() for watcher in current_app.db.list_watchers(sim_id)] )