diff --git a/examples/BESSY2_example/BESSY2Orbit.yaml b/examples/BESSY2_example/BESSY2Orbit.yaml index e4e5947e..4d9fabed 100644 --- a/examples/BESSY2_example/BESSY2Orbit.yaml +++ b/examples/BESSY2_example/BESSY2Orbit.yaml @@ -277,7 +277,7 @@ devices: vcorr_array_name: VCorr name: DEFAULT_ORBIT_CORRECTION singular_values: 16 - response_matrix: file:orm.json + response_matrix: ${path:orm.json} - type: pyaml.rf.rf_plant name: RF masterclock: (MCLKHX251C:freq)[KHz] diff --git a/examples/BESSY2_example/BESSY2Tune.yaml b/examples/BESSY2_example/BESSY2Tune.yaml index 23a5bfd4..754ab9cd 100644 --- a/examples/BESSY2_example/BESSY2Tune.yaml +++ b/examples/BESSY2_example/BESSY2Tune.yaml @@ -668,7 +668,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: file:trm.json + response_matrix: ${path:trm.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune diff --git a/examples/SOLEIL_examples/tuning_tools.yaml b/examples/SOLEIL_examples/tuning_tools.yaml index 4b5eefbd..3d531b0b 100644 --- a/examples/SOLEIL_examples/tuning_tools.yaml +++ b/examples/SOLEIL_examples/tuning_tools.yaml @@ -6,7 +6,7 @@ name: DEFAULT_TUNE_CORRECTION quad_array_name: QCORR betatron_tune_name: BETATRON_TUNE - response_matrix: file:trm.json + response_matrix: ${path:trm.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QCORR @@ -40,4 +40,4 @@ vcorr_array_name: VCORR name: DEFAULT_ORBIT_CORRECTION singular_values: 16 - response_matrix: file:orm.json + response_matrix: ${path:file:orm.json} diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index ab48372c..65f66a10 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -1,10 +1,12 @@ """PyAML configuration file loader.""" -import collections.abc import io import json import logging +import os +import re from abc import ABC, abstractmethod +from collections.abc import Callable, Hashable from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path @@ -19,11 +21,11 @@ logger = logging.getLogger(__name__) -ACCEPTED_SUFFIXES = (".yaml", ".yml", ".json") -FILE_PREFIX = "file:" LOCATION_KEY = "__location__" FIELD_LOCATIONS_KEY = "__fieldlocations__" +ACCEPTED_SUFFIXES = (".yaml", ".yml", ".json") +RESOLVER_PATTERN = re.compile(r"\$\{([^{}]+)\}") class RootFolder: @@ -110,6 +112,81 @@ def loading(self, path: Path): self.include_stack.pop() +Resolver = Callable[[str, LoadContext | None], Any] +RESOLVERS: dict[str, Resolver] = {} + + +def resolver(name: str): + """Register a function as a configuration value resolver. + + Args: + name: Prefix used to invoke the resolver (for example ``"env"`` + or ``"file"``). + + Returns: + A decorator that registers the decorated function in the global + resolver registry. + """ + + def decorate(func: Resolver) -> Resolver: + RESOLVERS[name] = func + return func + + return decorate + + +@resolver("env") +def resolve_env(value: str, _context: LoadContext | None = None) -> str: + """Resolve an environment variable. + + Args: + value: Name of the environment variable. + context: Unused loading context. Present to match the resolver + interface. + + Raises: + PyAMLException: If the environment variable is not set. + """ + try: + return os.environ[value] + except KeyError as exc: + raise PyAMLException(f"Environment variable '{value}' is not set") from exc + + +@resolver("path") +def resolve_path(value: str, _context: LoadContext | None = None) -> str: + """Resolve a configuration path without loading the file. + + Relative paths are expanded using the configured root folder. + + Args: + value: Path to resolve. + context: Unused loading context. Present to match the resolver + interface. + + Returns: + The absolute, normalized path as a string. + """ + return str(ROOT.expand_path(value)) + + +@resolver("file") +def resolve_file(value: str, context: LoadContext) -> Any: + """Load and return the contents of a configuration file. + + Args: + value: Path to the configuration file. + context: Shared loading context used to track recursive includes + and detect inclusion cycles. + + Raises: + RuntimeError: If no loading context is provided. + """ + if context is None: + raise RuntimeError("File resolver requires LoadContext") + return _load(value, context) + + def load(filename: str, include_locations: bool = False) -> Union[dict, list]: """Load a configuration file. @@ -154,40 +231,103 @@ def __init__(self, path: Path, context: LoadContext): self.context = context def expand(self, obj: Union[dict, list, Any]) -> Union[dict, list, Any]: - """Recursively expand nested dictionaries and lists.""" + """Recursively expand configuration values. + + Dictionaries and lists are traversed recursively, while string values + are resolved using the registered resolvers. All other values are + returned unchanged. + """ if isinstance(obj, dict): return self._expand_dict(obj) if isinstance(obj, list): return self._expand_list(obj) + if isinstance(obj, str): + return self._expand_string(obj) return obj - def _expand_dict(self, d: dict) -> dict: - """Expand any supported file references found inside a dictionary.""" + def _expand_string(self, value: str) -> Any: + """Expand resolver expressions and file references in a string. - for key, value in list(d.items()): - try: - if _is_supported_file(value): - # If it is a reference to another file - resolved = self._resolve_file_reference(value) - if resolved is not None: - d[key] = resolved - else: - d[key] = _load(value, self.context) - else: - d[key] = self.expand(value) - except PyAMLConfigCyclingException as exc: - self._raise_cycle_error(exc, d, key) + If the entire string is a resolver expression (for example + ``"${env:HOME}"`` or ``"${file:config.yaml}"``), the resolved value is + returned directly and may be of any type. - return d + Resolver expressions embedded inside a larger string are interpolated + into the surrounding text. Only scalar values may be interpolated; + attempting to embed a dictionary or list raises a ``PyAMLException``. - def _resolve_file_reference(self, value: str) -> str | None: - """Return an absolute path for FILE_PREFIX references, otherwise None.""" + After resolver expansion, plain strings that refer to supported + configuration files are loaded automatically. - if value.startswith(FILE_PREFIX): - stripped_value = value[len(FILE_PREFIX) :] - return str(ROOT.get() / Path(stripped_value)) - return None + Args: + value: The string value to expand. + + Returns: + The expanded value, which may be a string or another object if the + input consists solely of a resolver expression. + """ + + full_match = RESOLVER_PATTERN.fullmatch(value) + if full_match: + return self._resolve_resolver_expression(full_match.group(1).strip()) + + # Handle embedded case + def replace(match: re.Match[str]) -> str: + resolved = self._resolve_resolver_expression(match.group(1).strip()) + + if isinstance(resolved, (dict, list)): + raise PyAMLException( + f"Resolver '{match.group(1)}' returned a {type(resolved).__name__}, " + "which cannot be interpolated into a string." + ) + + return str(resolved) + + value = RESOLVER_PATTERN.sub(replace, value) + + if _is_supported_file(value): + return RESOLVERS["file"](value, self.context) + + return value + + def _resolve_resolver_expression(self, expr: str) -> Any: + """Resolve a single resolver expression. + + The expression must have the form ``":"``, for + example ``"env:HOME"`` or ``"file:config.yaml"``. The resolver is + looked up in the global resolver registry and invoked with the + supplied payload. + + Args: + expr: Resolver expression without the surrounding ``"${...}"``. + + Returns: + The value returned by the matching resolver. + + Raises: + PyAMLException: If the expression is malformed or references an + unknown resolver. + """ + if ":" not in expr: + raise PyAMLException(f"Invalid resolver expression '{expr}'") + + prefix, payload = expr.split(":", 1) + resolver = RESOLVERS.get(prefix.strip()) + if resolver is None: + raise PyAMLException(f"Unknown resolver '{prefix.strip()}'") + + return resolver(payload.strip(), self.context) + + def _expand_dict(self, data: dict) -> dict: + """Recursively expand dictionary values and enrich cycle errors with location information.""" + + for key, value in list(data.items()): + try: + data[key] = self.expand(value) + except PyAMLConfigCyclingException as exc: + self._raise_cycle_error(exc, data, key) + return data def _raise_cycle_error(self, exc: PyAMLConfigCyclingException, obj: dict, key: Any) -> None: """Re-raise a cycle error with file location information, if available.""" @@ -205,18 +345,34 @@ def _raise_cycle_error(self, exc: PyAMLConfigCyclingException, obj: dict, key: A raise PyAMLException(f"Circular file inclusion of {exc.error_filename}{location_str}") from exc def _expand_list(self, items: list) -> list: - """Expand supported file references inside a list.""" + """Recursively expand the elements of a list. - expanded = [] - for value in items: - if _is_supported_file(value): - loaded = _load(value, self.context) + Plain string values that refer to supported configuration files are + treated as list includes. If the referenced file loads to a list, its + elements are spliced into the current list. Otherwise, the loaded + object is appended as a single element. + + All other items are expanded recursively using :meth:`expand`. + + Args: + items: The list to expand. + + Returns: + The expanded list. + """ + expanded: list[Any] = [] + + for item in items: + if isinstance(item, str) and _is_supported_file(item): + loaded = RESOLVERS["file"](item, self.context) if isinstance(loaded, list): expanded.extend(loaded) else: expanded.append(loaded) - else: - expanded.append(self.expand(value)) + continue + + expanded.append(self.expand(item)) + return expanded @abstractmethod @@ -281,7 +437,7 @@ def construct_mapping(self, node, deep=False): for key_node, value_node in node.value: key = self.construct_object(key_node, deep=deep) - if not isinstance(key, collections.abc.Hashable): + if not isinstance(key, Hashable): raise ConstructorError( "while constructing a mapping", node.start_mark, diff --git a/pyaml/configuration/restfetcher.py b/pyaml/configuration/restfetcher.py index 380da617..78d4aca7 100644 --- a/pyaml/configuration/restfetcher.py +++ b/pyaml/configuration/restfetcher.py @@ -15,12 +15,14 @@ from yaml import CLoader from ..common.exception import PyAMLConfigException -from .fileloader import ACCEPTED_SUFFIXES, FILE_PREFIX, SafeLineLoader +from .fileloader import ACCEPTED_SUFFIXES, SafeLineLoader REMOTE_BASE_URL_KEY = "__baseurl__" SourceRoot = Path | str | None _REMOTE_SCHEMES = {"http", "https"} +FILE_PREFIX = "${path:" + class _NamedStringIO(io.StringIO): def __init__(self, value: str, name: str): diff --git a/tests/config/EBSOrbit.yaml b/tests/config/EBSOrbit.yaml index a3c66132..9b66318a 100644 --- a/tests/config/EBSOrbit.yaml +++ b/tests/config/EBSOrbit.yaml @@ -55,7 +55,7 @@ devices: name: DEFAULT_CHROMATICITY_CORRECTION chromaticty_monitor_name: CHROMATICITY_MONITOR sextu_array_name: Sext - response_matrix: file:ideal_chroma_resp.json + response_matrix: ${path:ideal_chroma_resp.json} - type: pyaml.tuning_tools.orbit bpm_array_name: BPM hcorr_array_name: HCorr @@ -63,7 +63,7 @@ devices: rf_plant_name: RF name: DEFAULT_ORBIT_CORRECTION singular_values: 162 - response_matrix: file:ideal_orm_disp.json + response_matrix: ${path:ideal_orm_disp.json} - type: pyaml.tuning_tools.orbit_response_matrix bpm_array_name: BPM hcorr_array_name: HCorr diff --git a/tests/config/EBSTune-patterns.yaml b/tests/config/EBSTune-patterns.yaml index abab79d8..0c258bde 100644 --- a/tests/config/EBSTune-patterns.yaml +++ b/tests/config/EBSTune-patterns.yaml @@ -36,7 +36,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: file:tune_response.json + response_matrix: ${path:tune_response.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune diff --git a/tests/config/EBSTune.yaml b/tests/config/EBSTune.yaml index 373e83c3..f5a8d5c0 100644 --- a/tests/config/EBSTune.yaml +++ b/tests/config/EBSTune.yaml @@ -29,7 +29,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: file:tune_response.json + response_matrix: ${path:tune_response.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune diff --git a/tests/config/sr_serialized_magnets.yaml b/tests/config/sr_serialized_magnets.yaml index e3609e8a..a8a3c486 100644 --- a/tests/config/sr_serialized_magnets.yaml +++ b/tests/config/sr_serialized_magnets.yaml @@ -225,7 +225,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: file:tune_response.json + response_matrix: ${path:tune_response.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune diff --git a/tests/configuration/test_fileloader.py b/tests/configuration/test_fileloader.py index b76273fc..53b234a6 100644 --- a/tests/configuration/test_fileloader.py +++ b/tests/configuration/test_fileloader.py @@ -5,7 +5,6 @@ from pyaml import PyAMLException from pyaml.configuration.fileloader import ( FIELD_LOCATIONS_KEY, - FILE_PREFIX, LOCATION_KEY, ROOT, LoadContext, @@ -133,6 +132,92 @@ def test_load_nested_json(tmp_path): assert result["child"]["answer"] == 42 +def test_load_file_resolver_loads_nested_file(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "subdir").mkdir() + (tmp_path / "subdir" / "child.yaml").write_text("answer: 42\n") + (tmp_path / "parent.yaml").write_text('target: "${file:subdir/child.yaml}"\n') + + result = load("parent.yaml") + + assert result["target"]["answer"] == 42 + + +def test_load_env_resolver_resolves_environment_variable(tmp_path, monkeypatch): + ROOT.set(tmp_path) + + monkeypatch.setenv("TANGO_HOST", "localhost:10000") + (tmp_path / "config.yaml").write_text("host: ${env:TANGO_HOST}\n") + + result = load("config.yaml") + + assert result["host"] == "localhost:10000" + + +def test_load_env_resolver_missing_variable_raises(tmp_path, monkeypatch): + ROOT.set(tmp_path) + + monkeypatch.delenv("TANGO_HOST", raising=False) + (tmp_path / "config.yaml").write_text("host: ${env:TANGO_HOST}\n") + + with pytest.raises(PyAMLException, match="Environment variable 'TANGO_HOST' is not set"): + load("config.yaml") + + +def test_interpolates_env_inside_string(tmp_path, monkeypatch): + ROOT.set(tmp_path) + monkeypatch.setenv("HOST", "localhost") + monkeypatch.setenv("PORT", "5432") + + (tmp_path / "config.yaml").write_text("url: http://${env:HOST}:${env:PORT}\n") + + result = load("config.yaml") + + assert result["url"] == "http://localhost:5432" + + +def test_load_interpolates_env_multiple_times_in_one_string(tmp_path, monkeypatch): + ROOT.set(tmp_path) + + monkeypatch.setenv("USER", "alice") + + (tmp_path / "config.yaml").write_text("message: hello ${env:USER}, ${env:USER}!\n") + + result = load("config.yaml") + + assert result["message"] == "hello alice, alice!" + + +def test_load_path_resolver_resolves_without_loading_file(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "config.yaml").write_text("target: ${path:subdir/missing.json}\n") + + result = load("config.yaml") + + assert result["target"] == str((tmp_path / "subdir" / "missing.json").resolve()) + + +def test_load_interpolated_file_resolver_inside_string_raises(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "child.yaml").write_text("answer: 42\n") + (tmp_path / "config.yaml").write_text('value: "prefix-${file:child.yaml}-suffix"\n') + + with pytest.raises(PyAMLException, match="cannot be interpolated into a string"): + load("config.yaml") + + +def test_load_interpolation_unknown_resolver_raises(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "config.yaml").write_text("value: prefix-${missing:VALUE}-suffix\n") + + with pytest.raises(PyAMLException, match="Unknown resolver 'missing'"): + load("config.yaml") + + def test_load_list_include_extends_list(tmp_path): ROOT.set(tmp_path) @@ -157,16 +242,6 @@ def test_load_list_include_extends_list(tmp_path): ] -def test_load_file_prefix_returns_absolute_path(tmp_path): - ROOT.set(tmp_path) - - (tmp_path / "config.yaml").write_text(f'target: "{FILE_PREFIX}subdir/file.yaml"\n') - - result = load("config.yaml") - - assert result["target"] == str((tmp_path / "subdir" / "file.yaml").resolve()) - - def test_load_yaml_omits_locations_by_default(tmp_path): ROOT.set(tmp_path)