diff --git a/examples/BESSY2_example/BESSY2Orbit.yaml b/examples/BESSY2_example/BESSY2Orbit.yaml index e4e5947ec..4d9fabed5 100644 --- a/examples/BESSY2_example/BESSY2Orbit.yaml +++ b/examples/BESSY2_example/BESSY2Orbit.yaml @@ -277,7 +277,7 @@ devices: vcorr_array_name: VCorr name: DEFAULT_ORBIT_CORRECTION singular_values: 16 - response_matrix: file:orm.json + response_matrix: ${path:orm.json} - type: pyaml.rf.rf_plant name: RF masterclock: (MCLKHX251C:freq)[KHz] diff --git a/examples/BESSY2_example/BESSY2Tune.yaml b/examples/BESSY2_example/BESSY2Tune.yaml index 23a5bfd4c..754ab9cd8 100644 --- a/examples/BESSY2_example/BESSY2Tune.yaml +++ b/examples/BESSY2_example/BESSY2Tune.yaml @@ -668,7 +668,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: file:trm.json + response_matrix: ${path:trm.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune diff --git a/examples/SOLEIL_examples/tuning_tools.yaml b/examples/SOLEIL_examples/tuning_tools.yaml index 4b5eefbdb..3d531b0ba 100644 --- a/examples/SOLEIL_examples/tuning_tools.yaml +++ b/examples/SOLEIL_examples/tuning_tools.yaml @@ -6,7 +6,7 @@ name: DEFAULT_TUNE_CORRECTION quad_array_name: QCORR betatron_tune_name: BETATRON_TUNE - response_matrix: file:trm.json + response_matrix: ${path:trm.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QCORR @@ -40,4 +40,4 @@ vcorr_array_name: VCORR name: DEFAULT_ORBIT_CORRECTION singular_values: 16 - response_matrix: file:orm.json + response_matrix: ${path:file:orm.json} diff --git a/pyaml/accelerator.py b/pyaml/accelerator.py index a35dd4e24..b813c721e 100644 --- a/pyaml/accelerator.py +++ b/pyaml/accelerator.py @@ -275,7 +275,7 @@ def from_dict(config_dict: dict, ignore_external=False) -> "Accelerator": return Factory.build(config_dict, ignore_external) @staticmethod - def load(filename: str, use_fast_loader: bool = False, ignore_external=False) -> "Accelerator": + def load(filename: str, include_locations: bool = False, ignore_external=False) -> "Accelerator": """ Load an accelerator from a config file. @@ -283,11 +283,11 @@ def load(filename: str, use_fast_loader: bool = False, ignore_external=False) -> ---------- filename : str Configuration file name, yaml or json. - use_fast_loader : bool - Use fast yaml loader. When specified, - no line number are reported in case of error, + include_locations : bool + When False, use faster loader but no line number + are reported in case of error, only the element name that triggered the error - will be reported in the exception) + will be reported in the exception ignore_external : bool Ignore external modules and return None for object that cannot be created. pydantic schema that support that an @@ -295,7 +295,7 @@ def load(filename: str, use_fast_loader: bool = False, ignore_external=False) -> """ manager = ConfigurationManager() try: - manager.add(filename, use_fast_loader=use_fast_loader) + manager.add(filename, include_locations=include_locations) except UnsupportedConfigurationRootError as ex: raise PyAMLConfigException( "Accelerator.load() expects a 'pyaml.accelerator' root configuration. " diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index c23497bde..65f66a105 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -1,11 +1,16 @@ -""" "PyAML configuration file loader.""" +"""PyAML configuration file loader.""" -import collections.abc import io import json import logging +import os +import re +from abc import ABC, abstractmethod +from collections.abc import Callable, Hashable +from contextlib import contextmanager +from dataclasses import dataclass, field from pathlib import Path -from typing import Union +from typing import Any, Union import yaml from yaml import CLoader @@ -16,158 +21,430 @@ logger = logging.getLogger(__name__) -accepted_suffixes = [".yaml", ".yml", ".json"] -FILE_PREFIX = "file:" + +LOCATION_KEY = "__location__" +FIELD_LOCATIONS_KEY = "__fieldlocations__" +ACCEPTED_SUFFIXES = (".yaml", ".yml", ".json") +RESOLVER_PATTERN = re.compile(r"\$\{([^{}]+)\}") class RootFolder: - """ - Manage the root directory used to resolve relative configuration paths. - """ + """Manage the root directory used to resolve configuration paths.""" def __init__(self, path: str | Path | None = None): - # Resolve the path is given otherwise set to the current working directory + """Create a root folder. + + If no path is provided, the current working directory is used. + """ if path is None: self._path = Path.cwd().resolve() else: self._path = Path(path).resolve() def set(self, path: str | Path) -> None: - """ - Set the root path for configuration files. - """ + """Set the root path used for resolving relative configuration files.""" self._path = Path(path).resolve() def get(self) -> Path: - """ - Get the root path for configuration files. - """ + """Return the current root path.""" return self._path def expand_path(self, path: str | Path) -> Path: - """ - Return an absolute configuration path. + """Return an absolute, normalized configuration path. - Relative paths are interpreted relative to the configured root - folder. Absolute paths are returned unchanged. + Relative paths are interpreted relative to the configured root folder. """ path = Path(path) - return path if path.is_absolute() else self._path / path + if not path.is_absolute(): + path = self._path / path + return path.resolve() ROOT = RootFolder() class PyAMLConfigCyclingException(PyAMLException): + """Raised when a configuration file includes itself through a cycle.""" + def __init__(self, error_filename: str, path_stack: list[Path]): + """Create a circular-include error. + + Args: + error_filename: The file that triggered the cycle. + path_stack: The include chain leading to the cycle. + """ + self.error_filename = error_filename + parent_file_stack = [parent_path.name for parent_path in path_stack] super().__init__(f"Circular file inclusion of {error_filename}. File list before reaching it: {parent_file_stack}") - pass +@dataclass +class LoadContext: + """Track state for one recursive configuration-loading session.""" -def load(filename: str, paths_stack: list = None, use_fast_loader: bool = False) -> Union[dict, list]: - """Load recursively a configuration setup""" - if filename.endswith(".yaml") or filename.endswith(".yml"): - l = YAMLLoader(filename, paths_stack, use_fast_loader) - elif filename.endswith(".json"): - l = JSONLoader(filename, paths_stack, use_fast_loader) - else: - raise PyAMLException(f"{filename} File format not supported (only .yaml .yml or .json)") - return l.load() + include_locations: bool = False + include_stack: list[Path] = field(default_factory=list) + @contextmanager + def loading(self, path: Path): + """Register a file as currently being loaded and remove it afterward. -# Expand condition -def hasToLoad(value): - return isinstance(value, str) and any(value.endswith(suffix) for suffix in accepted_suffixes) + Raises: + PyAMLConfigCyclingException: If the file is already on the active + include stack. + """ + # Check if the file is currently in the chain + # Raise a cycling error if that is the case + if path in self.include_stack: + raise PyAMLConfigCyclingException(path.name, self.include_stack) -# Loader base class (nested files expansion) -class Loader: - def __init__(self, filename: str, parent_path_stack: list[Path]): - self.path: Path = ROOT.expand_path(filename) - self.files_stack: list[Path] = [] - if parent_path_stack: - if any(self.path.samefile(parent_path) for parent_path in parent_path_stack): - raise PyAMLConfigCyclingException(filename, parent_path_stack) - self.files_stack.extend(parent_path_stack) - self.files_stack.append(self.path) + # Add the file to the stack to record that it is now being loaded + self.include_stack.append(path) + try: + # Run the code in the with block + yield + finally: + # Remove the file to stack to record that it has finished loading + self.include_stack.pop() + + +Resolver = Callable[[str, LoadContext | None], Any] +RESOLVERS: dict[str, Resolver] = {} + + +def resolver(name: str): + """Register a function as a configuration value resolver. + + Args: + name: Prefix used to invoke the resolver (for example ``"env"`` + or ``"file"``). + + Returns: + A decorator that registers the decorated function in the global + resolver registry. + """ + + def decorate(func: Resolver) -> Resolver: + RESOLVERS[name] = func + return func + + return decorate + + +@resolver("env") +def resolve_env(value: str, _context: LoadContext | None = None) -> str: + """Resolve an environment variable. + + Args: + value: Name of the environment variable. + context: Unused loading context. Present to match the resolver + interface. + + Raises: + PyAMLException: If the environment variable is not set. + """ + try: + return os.environ[value] + except KeyError as exc: + raise PyAMLException(f"Environment variable '{value}' is not set") from exc + + +@resolver("path") +def resolve_path(value: str, _context: LoadContext | None = None) -> str: + """Resolve a configuration path without loading the file. + + Relative paths are expanded using the configured root folder. + + Args: + value: Path to resolve. + context: Unused loading context. Present to match the resolver + interface. + + Returns: + The absolute, normalized path as a string. + """ + return str(ROOT.expand_path(value)) + + +@resolver("file") +def resolve_file(value: str, context: LoadContext) -> Any: + """Load and return the contents of a configuration file. + + Args: + value: Path to the configuration file. + context: Shared loading context used to track recursive includes + and detect inclusion cycles. + + Raises: + RuntimeError: If no loading context is provided. + """ + if context is None: + raise RuntimeError("File resolver requires LoadContext") + return _load(value, context) + + +def load(filename: str, include_locations: bool = False) -> Union[dict, list]: + """Load a configuration file. + + When include_locations is False, uses the faster C-based YAML loader + and skips including source location metadata. + """ + + # Create a new context + context = LoadContext(include_locations=include_locations) + + return _load(filename, context) + + +def _load(filename: str, context: LoadContext) -> Union[dict, list]: + """Load a single configuration file using the appropriate parser.""" + + path = ROOT.expand_path(filename) + + with context.loading(path): + if filename.endswith((".yaml", ".yml")): + loader = YAMLLoader(path, context) + elif filename.endswith(".json"): + loader = JSONLoader(path, context) + else: + raise PyAMLException(f"{filename} File format not supported (only .yaml .yml or .json)") + + return loader.load() + + +def _is_supported_file(value: Any) -> bool: + """Return True if the value looks like a supported configuration file name.""" + return isinstance(value, str) and value.endswith(ACCEPTED_SUFFIXES) + + +class ConfigLoader(ABC): + """Base class for loaders that expand nested configuration references.""" + + def __init__(self, path: Path, context: LoadContext): + """Store the file path and shared loading context.""" + + self.path = path + self.context = context + + def expand(self, obj: Union[dict, list, Any]) -> Union[dict, list, Any]: + """Recursively expand configuration values. + + Dictionaries and lists are traversed recursively, while string values + are resolved using the registered resolvers. All other values are + returned unchanged. + """ - # Recursively expand a dict - def expand_dict(self, d: dict): - for key, value in d.items(): - try: - if hasToLoad(value): - if value.startswith(FILE_PREFIX): - # remove prefix - stripped_value = value[len(FILE_PREFIX) :] - d[key] = str(ROOT.get() / Path(stripped_value)) - else: - d[key] = load(value, self.files_stack, self.use_fast_loader) - else: - self.expand(value) - except PyAMLConfigCyclingException as pyaml_ex: - location = d.pop("__location__", None) - field_locations = d.pop("__fieldlocations__", None) - location_str = "" - if location: - file, line, col = location - if field_locations and key in field_locations: - location = field_locations[key] - file, line, col = location - location_str = f" in {file} at line {line}, column {col}" - raise PyAMLException(f"Circular file inclusion of {pyaml_ex.error_filename}{location_str}") from pyaml_ex - - # Recursively expand a list - def expand_list(self, l: list): - idx = 0 - while idx < len(l): - value = l[idx] - if hasToLoad(value): - obj = load(value, self.files_stack) - if isinstance(obj, list): - l[idx : idx + 1] = obj - idx += len(obj) - else: - l[idx] = obj - idx += 1 - else: - self.expand(value) - idx += 1 - - # Recursively expand an object - def expand(self, obj: Union[dict, list]): if isinstance(obj, dict): - self.expand_dict(obj) - elif isinstance(obj, list): - self.expand_list(obj) + return self._expand_dict(obj) + if isinstance(obj, list): + return self._expand_list(obj) + if isinstance(obj, str): + return self._expand_string(obj) return obj - # Load a file + def _expand_string(self, value: str) -> Any: + """Expand resolver expressions and file references in a string. + + If the entire string is a resolver expression (for example + ``"${env:HOME}"`` or ``"${file:config.yaml}"``), the resolved value is + returned directly and may be of any type. + + Resolver expressions embedded inside a larger string are interpolated + into the surrounding text. Only scalar values may be interpolated; + attempting to embed a dictionary or list raises a ``PyAMLException``. + + After resolver expansion, plain strings that refer to supported + configuration files are loaded automatically. + + Args: + value: The string value to expand. + + Returns: + The expanded value, which may be a string or another object if the + input consists solely of a resolver expression. + """ + + full_match = RESOLVER_PATTERN.fullmatch(value) + if full_match: + return self._resolve_resolver_expression(full_match.group(1).strip()) + + # Handle embedded case + def replace(match: re.Match[str]) -> str: + resolved = self._resolve_resolver_expression(match.group(1).strip()) + + if isinstance(resolved, (dict, list)): + raise PyAMLException( + f"Resolver '{match.group(1)}' returned a {type(resolved).__name__}, " + "which cannot be interpolated into a string." + ) + + return str(resolved) + + value = RESOLVER_PATTERN.sub(replace, value) + + if _is_supported_file(value): + return RESOLVERS["file"](value, self.context) + + return value + + def _resolve_resolver_expression(self, expr: str) -> Any: + """Resolve a single resolver expression. + + The expression must have the form ``":"``, for + example ``"env:HOME"`` or ``"file:config.yaml"``. The resolver is + looked up in the global resolver registry and invoked with the + supplied payload. + + Args: + expr: Resolver expression without the surrounding ``"${...}"``. + + Returns: + The value returned by the matching resolver. + + Raises: + PyAMLException: If the expression is malformed or references an + unknown resolver. + """ + if ":" not in expr: + raise PyAMLException(f"Invalid resolver expression '{expr}'") + + prefix, payload = expr.split(":", 1) + resolver = RESOLVERS.get(prefix.strip()) + if resolver is None: + raise PyAMLException(f"Unknown resolver '{prefix.strip()}'") + + return resolver(payload.strip(), self.context) + + def _expand_dict(self, data: dict) -> dict: + """Recursively expand dictionary values and enrich cycle errors with location information.""" + + for key, value in list(data.items()): + try: + data[key] = self.expand(value) + except PyAMLConfigCyclingException as exc: + self._raise_cycle_error(exc, data, key) + return data + + def _raise_cycle_error(self, exc: PyAMLConfigCyclingException, obj: dict, key: Any) -> None: + """Re-raise a cycle error with file location information, if available.""" + + location = obj.get(LOCATION_KEY) + field_locations = obj.get(FIELD_LOCATIONS_KEY) + + location_str = "" + if location: + file, line, col = location + if field_locations and key in field_locations: + file, line, col = field_locations[key] + location_str = f" in {file} at line {line}, column {col}" + + raise PyAMLException(f"Circular file inclusion of {exc.error_filename}{location_str}") from exc + + def _expand_list(self, items: list) -> list: + """Recursively expand the elements of a list. + + Plain string values that refer to supported configuration files are + treated as list includes. If the referenced file loads to a list, its + elements are spliced into the current list. Otherwise, the loaded + object is appended as a single element. + + All other items are expanded recursively using :meth:`expand`. + + Args: + items: The list to expand. + + Returns: + The expanded list. + """ + expanded: list[Any] = [] + + for item in items: + if isinstance(item, str) and _is_supported_file(item): + loaded = RESOLVERS["file"](item, self.context) + if isinstance(loaded, list): + expanded.extend(loaded) + else: + expanded.append(loaded) + continue + + expanded.append(self.expand(item)) + + return expanded + + @abstractmethod def load(self) -> Union[dict, list]: - raise PyAMLException(str(self.path) + ": load() method not implemented") + """Load and parse the current configuration file.""" + ... + + +class YAMLLoader(ConfigLoader): + """Load and expand YAML configuration files.""" + + def __init__(self, path: Path, context: LoadContext): + """Create a YAML loader for the given file.""" + + super().__init__(path, context) + self._loader = SafeLineLoader if context.include_locations else CLoader + + def load(self) -> Union[dict, list]: + """Parse the YAML file and expand nested configuration references.""" + + logger.log(logging.DEBUG, f"Loading YAML file '{self.path}'") + with open(self.path) as file: + try: + return self.expand(yaml.load(file, Loader=self._loader)) + except yaml.YAMLError as exc: + raise PyAMLException(f"{self.path}: {exc}") from exc + + +class JSONLoader(ConfigLoader): + """Load and expand JSON configuration files.""" + + def __init__(self, path: Path, context: LoadContext): + """Create a JSON loader for the given file.""" + + super().__init__(path, context) + + def load(self) -> Union[dict, list]: + """Parse the JSON file and expand nested configuration references.""" + + logger.log(logging.DEBUG, f"Loading JSON file '{self.path}'") + with open(self.path) as file: + try: + return self.expand(json.load(file)) + except json.JSONDecodeError as exc: + raise PyAMLException(f"{self.path}: {exc}") from exc class SafeLineLoader(SafeLoader): + """YAML loader that preserves line and column information for mappings.""" + def __init__(self, stream): + """Create the YAML loader and record the source filename.""" + super().__init__(stream) self.filename = stream.name if isinstance(stream, io.TextIOWrapper) else "" def construct_mapping(self, node, deep=False): + """Build a mapping and attach location metadata to it.""" + mapping = {} field_mapping = {} for key_node, value_node in node.value: key = self.construct_object(key_node, deep=deep) - if not isinstance(key, collections.abc.Hashable): + if not isinstance(key, Hashable): raise ConstructorError( "while constructing a mapping", node.start_mark, "found unhashable key", key_node.start_mark, ) + value = self.construct_object(value_node, deep=deep) mapping[key] = value field_mapping[key] = ( @@ -177,41 +454,10 @@ def construct_mapping(self, node, deep=False): ) # Add location information inside the dict - mapping["__location__"] = ( + mapping[LOCATION_KEY] = ( self.filename, node.start_mark.line + 1, node.start_mark.column + 1, ) - mapping["__fieldlocations__"] = field_mapping + mapping[FIELD_LOCATIONS_KEY] = field_mapping return mapping - - -# YAML loader -class YAMLLoader(Loader): - def __init__(self, filename: str, parent_paths_stack: list, use_fast_loader: bool): - super().__init__(filename, parent_paths_stack) - self._loader = SafeLineLoader if not use_fast_loader else CLoader - self.use_fast_loader = use_fast_loader - - def load(self) -> Union[dict, list]: - logger.log(logging.DEBUG, f"Loading YAML file '{self.path}'") - with open(self.path) as file: - try: - return self.expand(yaml.load(file, Loader=self._loader)) - except yaml.YAMLError as e: - raise PyAMLException(str(self.path) + ": " + str(e)) from e - - -# JSON loader -class JSONLoader(Loader): - def __init__(self, filename: str, parent_paths_stack: list, use_fast_loader: bool): - super().__init__(filename, parent_paths_stack) - self.use_fast_loader = False - - def load(self) -> Union[dict, list]: - logger.log(logging.DEBUG, f"Loading JSON file '{self.path}'") - with open(self.path) as file: - try: - return self.expand(json.load(file)) - except json.JSONDecodeError as e: - raise PyAMLException(str(self.path) + ": " + str(e)) from e diff --git a/pyaml/configuration/manager.py b/pyaml/configuration/manager.py index 8b3d5b226..61e844ca1 100644 --- a/pyaml/configuration/manager.py +++ b/pyaml/configuration/manager.py @@ -122,7 +122,7 @@ def add(self, payload, **kwargs) -> None: Fragment to merge into the current aggregated state. source_name : str, optional Explicit source label to associate with the fragment. - use_fast_loader : bool, optional + include_locations : bool, optional Forwarded to the configuration loader. Examples @@ -145,7 +145,7 @@ def add(self, payload, **kwargs) -> None: ... ) """ source_name = kwargs.pop("source_name", None) - use_fast_loader = kwargs.pop("use_fast_loader", False) + include_locations = kwargs.pop("include_locations", False) if kwargs: unknown = ", ".join(sorted(kwargs)) raise PyAMLConfigException(f"Unsupported ConfigurationManager.add() arguments: {unknown}") @@ -153,7 +153,7 @@ def add(self, payload, **kwargs) -> None: fragment, inferred_source_name, source_root = self._load_payload( payload, source_name=source_name, - use_fast_loader=use_fast_loader, + include_locations=include_locations, ) prepared = self._prepare_fragment(fragment, source_root, inferred_source_name) self._merge_fragment(prepared, inferred_source_name) @@ -599,14 +599,14 @@ def _load_payload( payload, *, source_name: str | None, - use_fast_loader: bool, + include_locations: bool, ) -> tuple[dict[str, Any], str, SourceRoot]: if isinstance(payload, dict): return copy.deepcopy(payload), source_name or "", None payload_path = self._coerce_path(payload) if is_remote_url(payload_path): - fragment, source_root = fetch_remote_config(payload_path, use_fast_loader=use_fast_loader) + fragment, source_root = fetch_remote_config(payload_path, include_locations=include_locations) if not self._build_root_locked: self._build_root = source_root self._build_root_locked = True @@ -628,7 +628,7 @@ def _load_payload( source_root = resolved_path.parent try: ROOT.set(source_root) - fragment = load(resolved_path.name, None, use_fast_loader) + fragment = load(resolved_path.name, include_locations) finally: ROOT.set(previous_root) diff --git a/pyaml/configuration/restfetcher.py b/pyaml/configuration/restfetcher.py index ea80b58e7..78d4aca7e 100644 --- a/pyaml/configuration/restfetcher.py +++ b/pyaml/configuration/restfetcher.py @@ -15,12 +15,14 @@ from yaml import CLoader from ..common.exception import PyAMLConfigException -from .fileloader import FILE_PREFIX, SafeLineLoader, accepted_suffixes +from .fileloader import ACCEPTED_SUFFIXES, SafeLineLoader REMOTE_BASE_URL_KEY = "__baseurl__" SourceRoot = Path | str | None _REMOTE_SCHEMES = {"http", "https"} +FILE_PREFIX = "${path:" + class _NamedStringIO(io.StringIO): def __init__(self, value: str, name: str): @@ -32,9 +34,9 @@ def is_remote_url(value: str) -> bool: return urlparse(value).scheme in _REMOTE_SCHEMES -def fetch_remote_config(url: str, *, use_fast_loader: bool = False) -> tuple[dict[str, Any] | list[Any], str]: +def fetch_remote_config(url: str, *, include_locations: bool = True) -> tuple[dict[str, Any] | list[Any], str]: normalized_url = _normalize_remote_url(url) - expanded = _load_remote_document(normalized_url, use_fast_loader=use_fast_loader, stack=[]) + expanded = _load_remote_document(normalized_url, include_locations=include_locations, stack=[]) return expanded, _remote_base_url(normalized_url) @@ -61,15 +63,15 @@ def _normalize_remote_url(url: str) -> str: def _load_remote_document( url: str, *, - use_fast_loader: bool, + include_locations: bool, stack: list[str], ) -> dict[str, Any] | list[Any]: if url in stack: raise PyAMLConfigException(f"Circular remote configuration inclusion detected for '{url}'.") payload, content_type = _download_text(url) - document = _parse_remote_document(url, payload, content_type, use_fast_loader=use_fast_loader) - return _expand_remote_value(document, _remote_base_url(url), stack + [url], use_fast_loader=use_fast_loader) + document = _parse_remote_document(url, payload, content_type, include_locations=include_locations) + return _expand_remote_value(document, _remote_base_url(url), stack + [url], include_locations=include_locations) def _download_text(url: str) -> tuple[str, str]: @@ -93,7 +95,7 @@ def _parse_remote_document( payload: str, content_type: str, *, - use_fast_loader: bool, + include_locations: bool, ) -> dict[str, Any] | list[Any]: suffix = Path(urlparse(url).path).suffix.lower() @@ -103,7 +105,7 @@ def _parse_remote_document( except json.JSONDecodeError as ex: raise PyAMLConfigException(f"{url}: {ex}") from ex - loader = CLoader if use_fast_loader else SafeLineLoader + loader = CLoader if include_locations else SafeLineLoader try: stream = _NamedStringIO(payload, url) return yaml.load(stream, Loader=loader) @@ -111,11 +113,11 @@ def _parse_remote_document( raise PyAMLConfigException(f"{url}: {ex}") from ex -def _expand_remote_value(value, base_url: str, stack: list[str], *, use_fast_loader: bool): +def _expand_remote_value(value, base_url: str, stack: list[str], *, include_locations: bool): if isinstance(value, dict): - return _expand_remote_dict(value, base_url, stack, use_fast_loader=use_fast_loader) + return _expand_remote_dict(value, base_url, stack, include_locations=include_locations) if isinstance(value, list): - return _expand_remote_list(value, base_url, stack, use_fast_loader=use_fast_loader) + return _expand_remote_list(value, base_url, stack, include_locations=include_locations) return value @@ -124,14 +126,14 @@ def _expand_remote_dict( base_url: str, stack: list[str], *, - use_fast_loader: bool, + include_locations: bool, ) -> dict[str, Any]: values.setdefault(REMOTE_BASE_URL_KEY, base_url) for key, value in list(values.items()): if _is_config_reference(value): values[key] = _load_remote_document( _resolve_remote_config_reference(value, base_url), - use_fast_loader=use_fast_loader, + include_locations=include_locations, stack=stack, ) continue @@ -140,18 +142,18 @@ def _expand_remote_dict( values[key] = resolve_reference(value[len(FILE_PREFIX) :], base_url) continue - values[key] = _expand_remote_value(value, base_url, stack, use_fast_loader=use_fast_loader) + values[key] = _expand_remote_value(value, base_url, stack, include_locations=include_locations) return values -def _expand_remote_list(values: list[Any], base_url: str, stack: list[str], *, use_fast_loader: bool) -> list[Any]: +def _expand_remote_list(values: list[Any], base_url: str, stack: list[str], *, include_locations: bool) -> list[Any]: index = 0 while index < len(values): value = values[index] if _is_config_reference(value): expanded = _load_remote_document( _resolve_remote_config_reference(value, base_url), - use_fast_loader=use_fast_loader, + include_locations=include_locations, stack=stack, ) if isinstance(expanded, list): @@ -167,7 +169,7 @@ def _expand_remote_list(values: list[Any], base_url: str, stack: list[str], *, u index += 1 continue - values[index] = _expand_remote_value(value, base_url, stack, use_fast_loader=use_fast_loader) + values[index] = _expand_remote_value(value, base_url, stack, include_locations=include_locations) index += 1 return values @@ -181,7 +183,7 @@ def _is_config_reference(value: Any) -> bool: parsed = urlparse(value) path = parsed.path if parsed.scheme in _REMOTE_SCHEMES else value - return any(path.endswith(suffix) for suffix in accepted_suffixes) + return any(path.endswith(suffix) for suffix in ACCEPTED_SUFFIXES) def _resolve_remote_config_reference(reference: str, base_url: str) -> str: diff --git a/tests/arrays/test_arrays.py b/tests/arrays/test_arrays.py index 526d4ab5b..eb72fdc87 100644 --- a/tests/arrays/test_arrays.py +++ b/tests/arrays/test_arrays.py @@ -201,7 +201,7 @@ def test_arrays(install_test_package): # Test dynamic arrays - sr: Accelerator = Accelerator.load("tests/config/EBSOrbit.yaml", use_fast_loader=True) + sr: Accelerator = Accelerator.load("tests/config/EBSOrbit.yaml", include_locations=False) ae = ElementArray("All", sr.design.get_all_elements()) acfm = ElementArray("AllCFM", sr.design.get_all_cfm_magnets(), use_aggregator=False) @@ -240,7 +240,7 @@ def test_arrays(install_test_package): ], ) def test_serialized_magnets_arrays(sr_file): - sr: Accelerator = Accelerator.load(sr_file, use_fast_loader=True, ignore_external=True) + sr: Accelerator = Accelerator.load(sr_file, include_locations=False, ignore_external=True) the_serie = sr.design.get_serialized_magnets("series") strength = the_serie.strengths.get() assert len(strength) == 1 diff --git a/tests/common/test_errors.py b/tests/common/test_errors.py index 77a1c5a2d..f7a9672d8 100644 --- a/tests/common/test_errors.py +++ b/tests/common/test_errors.py @@ -13,24 +13,24 @@ ) def test_tune(install_test_package): with pytest.raises(PyAMLConfigException) as exc: - ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_1.yaml") + ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_1.yaml", include_locations=True) assert "MagnetArray HCORR : duplicate name SH1A-C02-H @index 2" in str(exc) with pytest.raises(PyAMLConfigException) as exc: - ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_2.yaml") + ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_2.yaml", include_locations=True) assert "BPMArray BPM : duplicate name BPM_C04-06 @index 3" in str(exc) with pytest.raises(PyAMLConfigException) as exc: - ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_3.yaml") + ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_3.yaml", include_locations=True) assert "Configuration entry 'BPM_C04-06' is duplicated inside category 'devices'" in str(exc) assert "bad_conf_duplicate_3.yaml" in str(exc) assert "line 43, column 3" in str(exc) with pytest.raises(PyAMLConfigException) as exc: - ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_4.yaml") + ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_4.yaml", include_locations=True) assert "MagnetArray HCORR : duplicate name SH1A-C02-H @index 2" in str(exc) - sr: Accelerator = Accelerator.load("tests/config/EBSTune.yaml") + sr: Accelerator = Accelerator.load("tests/config/EBSTune.yaml", include_locations=True) m1 = sr.live.get_magnet("QF1E-C04") m2 = sr.design.get_magnet("QF1A-C05") with pytest.raises(PyAMLException) as exc: @@ -64,7 +64,7 @@ def test_duplicate_error_reports_source_line_and_column_across_files(tmp_path): ) with pytest.raises(PyAMLConfigException) as exc: - Accelerator.load(str(root)) + Accelerator.load(str(root), include_locations=True) message = str(exc.value) assert "Configuration entry 'BPM_DUPLICATE' is duplicated inside category 'devices'" in message diff --git a/tests/config/EBSOrbit.yaml b/tests/config/EBSOrbit.yaml index a3c66132e..9b66318ad 100644 --- a/tests/config/EBSOrbit.yaml +++ b/tests/config/EBSOrbit.yaml @@ -55,7 +55,7 @@ devices: name: DEFAULT_CHROMATICITY_CORRECTION chromaticty_monitor_name: CHROMATICITY_MONITOR sextu_array_name: Sext - response_matrix: file:ideal_chroma_resp.json + response_matrix: ${path:ideal_chroma_resp.json} - type: pyaml.tuning_tools.orbit bpm_array_name: BPM hcorr_array_name: HCorr @@ -63,7 +63,7 @@ devices: rf_plant_name: RF name: DEFAULT_ORBIT_CORRECTION singular_values: 162 - response_matrix: file:ideal_orm_disp.json + response_matrix: ${path:ideal_orm_disp.json} - type: pyaml.tuning_tools.orbit_response_matrix bpm_array_name: BPM hcorr_array_name: HCorr diff --git a/tests/config/EBSTune-patterns.yaml b/tests/config/EBSTune-patterns.yaml index abab79d85..0c258bdea 100644 --- a/tests/config/EBSTune-patterns.yaml +++ b/tests/config/EBSTune-patterns.yaml @@ -36,7 +36,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: file:tune_response.json + response_matrix: ${path:tune_response.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune diff --git a/tests/config/EBSTune.yaml b/tests/config/EBSTune.yaml index 373e83c31..f5a8d5c08 100644 --- a/tests/config/EBSTune.yaml +++ b/tests/config/EBSTune.yaml @@ -29,7 +29,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: file:tune_response.json + response_matrix: ${path:tune_response.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune diff --git a/tests/config/sr_serialized_magnets.yaml b/tests/config/sr_serialized_magnets.yaml index e3609e8ab..a8a3c486d 100644 --- a/tests/config/sr_serialized_magnets.yaml +++ b/tests/config/sr_serialized_magnets.yaml @@ -225,7 +225,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: file:tune_response.json + response_matrix: ${path:tune_response.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune diff --git a/tests/configuration/test_factory.py b/tests/configuration/test_factory.py index 97da045b6..e584bcde2 100644 --- a/tests/configuration/test_factory.py +++ b/tests/configuration/test_factory.py @@ -27,7 +27,7 @@ def test_error_location(test_file): try: with pytest.raises(PyAMLConfigException) as exc: ROOT.set("tests/config") - cfg = load("bad_conf.yml") + cfg = load("bad_conf.yml", include_locations=True) Factory.build(cfg, False) finally: ROOT.set(previous_root) @@ -46,7 +46,7 @@ def test_error_location(test_file): ) def test_error_cycles(test_file): with pytest.raises(PyAMLException) as exc: - ml: Accelerator = Accelerator.load(test_file) + ml: Accelerator = Accelerator.load(test_file, include_locations=True) assert "Circular file inclusion of " in str(exc.value) if not test_file.endswith(".json"): diff --git a/tests/configuration/test_fileloader.py b/tests/configuration/test_fileloader.py new file mode 100644 index 000000000..53b234a66 --- /dev/null +++ b/tests/configuration/test_fileloader.py @@ -0,0 +1,292 @@ +import json + +import pytest + +from pyaml import PyAMLException +from pyaml.configuration.fileloader import ( + FIELD_LOCATIONS_KEY, + LOCATION_KEY, + ROOT, + LoadContext, + RootFolder, + _is_supported_file, + load, +) + + +def test_rootfolder_uses_current_working_directory_when_no_path(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + + root = RootFolder() + + assert root.get() == tmp_path.resolve() + + +def test_rootfolder_sets_explicit_path(tmp_path): + root = RootFolder(tmp_path) + + assert root.get() == tmp_path.resolve() + + +def test_rootfolder_expands_relative_paths(tmp_path): + root = RootFolder(tmp_path) + + assert root.expand_path("config.yaml") == (tmp_path / "config.yaml").resolve() + + +def test_rootfolder_expands_absolute_paths(tmp_path): + root = RootFolder(tmp_path) + absolute = (tmp_path / "nested" / "config.yaml").resolve() + + assert root.expand_path(absolute) == absolute + + +def test_rootfolder_set_updates_root(tmp_path): + root = RootFolder(tmp_path) + new_root = tmp_path / "other" + + before = root.expand_path("config.yaml") + root.set(new_root) + after = root.expand_path("config.yaml") + + assert root.get() == new_root.resolve() + assert before == (tmp_path / "config.yaml").resolve() + assert after == (new_root / "config.yaml").resolve() + + +def test_is_supported_file(): + assert _is_supported_file("config.yaml") + assert _is_supported_file("config.yml") + assert _is_supported_file("config.json") + + assert not _is_supported_file("config.txt") + assert not _is_supported_file(1) + assert not _is_supported_file(None) + + +def test_load_context_adds_and_removes_paths(tmp_path): + ctx = LoadContext() + path = tmp_path / "config.yaml" + + with ctx.loading(path): + assert ctx.include_stack == [path] + + assert ctx.include_stack == [] + + +def test_load_context_cleans_up_after_exception(tmp_path): + ctx = LoadContext() + path = tmp_path / "config.yaml" + + with pytest.raises(RuntimeError): + with ctx.loading(path): + raise RuntimeError + + assert ctx.include_stack == [] + + +def test_load_yaml(tmp_path): + ROOT.set(tmp_path) + + config = tmp_path / "config.yaml" + config.write_text("a: 1\nb: hello\n") + + result = load("config.yaml") + + assert result["a"] == 1 + assert result["b"] == "hello" + + +def test_load_json(tmp_path): + ROOT.set(tmp_path) + + config = tmp_path / "config.json" + config.write_text(json.dumps({"a": 1, "b": "hello"})) + + result = load("config.json") + + assert result == {"a": 1, "b": "hello"} + + +def test_load_nested_yaml(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "child.yaml").write_text("answer: 42\n") + + (tmp_path / "parent.yaml").write_text("child: child.yaml\n") + + result = load("parent.yaml") + + assert result["child"]["answer"] == 42 + + +def test_load_nested_json(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "child.json").write_text(json.dumps({"answer": 42})) + + (tmp_path / "parent.json").write_text(json.dumps({"child": "child.json"})) + + result = load("parent.json") + + assert result["child"]["answer"] == 42 + + +def test_load_file_resolver_loads_nested_file(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "subdir").mkdir() + (tmp_path / "subdir" / "child.yaml").write_text("answer: 42\n") + (tmp_path / "parent.yaml").write_text('target: "${file:subdir/child.yaml}"\n') + + result = load("parent.yaml") + + assert result["target"]["answer"] == 42 + + +def test_load_env_resolver_resolves_environment_variable(tmp_path, monkeypatch): + ROOT.set(tmp_path) + + monkeypatch.setenv("TANGO_HOST", "localhost:10000") + (tmp_path / "config.yaml").write_text("host: ${env:TANGO_HOST}\n") + + result = load("config.yaml") + + assert result["host"] == "localhost:10000" + + +def test_load_env_resolver_missing_variable_raises(tmp_path, monkeypatch): + ROOT.set(tmp_path) + + monkeypatch.delenv("TANGO_HOST", raising=False) + (tmp_path / "config.yaml").write_text("host: ${env:TANGO_HOST}\n") + + with pytest.raises(PyAMLException, match="Environment variable 'TANGO_HOST' is not set"): + load("config.yaml") + + +def test_interpolates_env_inside_string(tmp_path, monkeypatch): + ROOT.set(tmp_path) + monkeypatch.setenv("HOST", "localhost") + monkeypatch.setenv("PORT", "5432") + + (tmp_path / "config.yaml").write_text("url: http://${env:HOST}:${env:PORT}\n") + + result = load("config.yaml") + + assert result["url"] == "http://localhost:5432" + + +def test_load_interpolates_env_multiple_times_in_one_string(tmp_path, monkeypatch): + ROOT.set(tmp_path) + + monkeypatch.setenv("USER", "alice") + + (tmp_path / "config.yaml").write_text("message: hello ${env:USER}, ${env:USER}!\n") + + result = load("config.yaml") + + assert result["message"] == "hello alice, alice!" + + +def test_load_path_resolver_resolves_without_loading_file(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "config.yaml").write_text("target: ${path:subdir/missing.json}\n") + + result = load("config.yaml") + + assert result["target"] == str((tmp_path / "subdir" / "missing.json").resolve()) + + +def test_load_interpolated_file_resolver_inside_string_raises(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "child.yaml").write_text("answer: 42\n") + (tmp_path / "config.yaml").write_text('value: "prefix-${file:child.yaml}-suffix"\n') + + with pytest.raises(PyAMLException, match="cannot be interpolated into a string"): + load("config.yaml") + + +def test_load_interpolation_unknown_resolver_raises(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "config.yaml").write_text("value: prefix-${missing:VALUE}-suffix\n") + + with pytest.raises(PyAMLException, match="Unknown resolver 'missing'"): + load("config.yaml") + + +def test_load_list_include_extends_list(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "items.yaml").write_text("- one\n- two\n") + + (tmp_path / "parent.yaml").write_text( + """ + values: + - start + - items.yaml + - end + """.strip() + ) + + result = load("parent.yaml") + + assert result["values"] == [ + "start", + "one", + "two", + "end", + ] + + +def test_load_yaml_omits_locations_by_default(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "config.yaml").write_text("a: 1\n") + + result = load("config.yaml") + + assert LOCATION_KEY not in result + assert FIELD_LOCATIONS_KEY not in result + + +def test_load_yaml_includes_locations_when_enables(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "config.yaml").write_text("a: 1\n") + + result = load("config.yaml", include_locations=True) + + assert LOCATION_KEY in result + assert FIELD_LOCATIONS_KEY in result + + +def test_load_invalid_yaml_raises_pyaml_exception(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "bad.yaml").write_text("a:\n - 1\n - 2\n") + + with pytest.raises(PyAMLException, match="bad.yaml"): + load("bad.yaml") + + +def test_load_invalid_json_raises_pyaml_exception(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "bad.json").write_text('{"a": }') + + with pytest.raises(PyAMLException, match="bad.json"): + load("bad.json") + + +def test_load_circular_include_raises_pyaml_exception(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "a.yaml").write_text("b: b.yaml\n") + (tmp_path / "b.yaml").write_text("a: a.yaml\n") + + with pytest.raises(PyAMLException, match="Circular file inclusion"): + load("a.yaml") diff --git a/tests/magnet/test_serialized_magnets.py b/tests/magnet/test_serialized_magnets.py index b0fc50baf..130379e3d 100644 --- a/tests/magnet/test_serialized_magnets.py +++ b/tests/magnet/test_serialized_magnets.py @@ -24,7 +24,7 @@ def check_no_diff(array: list[np.float64]) -> bool: ], ) def test_config_load(sr_file): - sr: Accelerator = Accelerator.load(sr_file, use_fast_loader=True, ignore_external=True) + sr: Accelerator = Accelerator.load(sr_file, include_locations=False, ignore_external=True) assert sr is not None magnets = [ sr.design.get_element("QF8B-C04"), @@ -55,7 +55,7 @@ def test_config_load(sr_file): ], ) def test_magnet_modification(sr_file): - sr = Accelerator.load(sr_file, use_fast_loader=True, ignore_external=True) + sr = Accelerator.load(sr_file, include_locations=False, ignore_external=True) print(sr.yellow_pages) @@ -107,7 +107,7 @@ def test_magnet_modification(sr_file): ], ) def test_tune(sr_file): - sr = Accelerator.load(sr_file, use_fast_loader=True, ignore_external=True) + sr = Accelerator.load(sr_file, include_locations=False, ignore_external=True) sr.design.get_lattice().disable_6d() m = sr.design.get_serialized_magnet("QF1A") @@ -161,7 +161,7 @@ def test_tune(sr_file): ], ) def test_get_device_names(sr_file): - sr: Accelerator = Accelerator.load(sr_file, use_fast_loader=True, ignore_external=True) + sr: Accelerator = Accelerator.load(sr_file, include_locations=False, ignore_external=True) sm: SerializedMagnets = sr.design.get_serialized_magnet("QF1A") device_names = sm.get_device_names() @@ -191,7 +191,7 @@ def get_strengths_from_lattice(sr: Accelerator, sm: SerializedMagnets) -> list: ], ) def test_strength_computation(sr_file): - sr: Accelerator = Accelerator.load(sr_file, use_fast_loader=True, ignore_external=True) + sr: Accelerator = Accelerator.load(sr_file, include_locations=False, ignore_external=True) sm: SerializedMagnets = sr.design.get_serialized_magnet("QF1A") assert sm.get_nb_magnets() == 31 sm.strength.set(24.0) diff --git a/tests/tuning_tools/test_tuning_tools.py b/tests/tuning_tools/test_tuning_tools.py index 72fdc5d3b..5c748b003 100644 --- a/tests/tuning_tools/test_tuning_tools.py +++ b/tests/tuning_tools/test_tuning_tools.py @@ -10,7 +10,7 @@ def callback(action: Action, data: dict): def test_tune_tool(): - sr = Accelerator.load("tests/config/EBSTune.yaml", use_fast_loader=True) + sr = Accelerator.load("tests/config/EBSTune.yaml", include_locations=False) sr.design.get_lattice().disable_6d() sr.design.trm.measure(callback=callback) sr.design.trm.save("tunemat.json") @@ -22,7 +22,7 @@ def test_tune_tool(): def test_tune_add(): - sr = Accelerator.load("tests/config/EBSTune.yaml", use_fast_loader=True) + sr = Accelerator.load("tests/config/EBSTune.yaml", include_locations=False) sr.design.get_lattice().disable_6d() sr.design.tune.load("tunemat.json") tune_initial = sr.design.tune.readback() @@ -33,7 +33,7 @@ def test_tune_add(): def test_chroma_tool(): - sr: Accelerator = Accelerator.load("tests/config/EBSOrbit.yaml", use_fast_loader=True) + sr: Accelerator = Accelerator.load("tests/config/EBSOrbit.yaml", include_locations=False) sr.design.get_lattice().enable_6d() sr.design.chromaticity.set([8.0, 5.0], iter=2) QpAT = sr.design.get_lattice().get_chrom()[:-1] @@ -45,7 +45,7 @@ def test_chroma_tool(): def test_chroma_add(): - sr: Accelerator = Accelerator.load("tests/config/EBSOrbit.yaml", use_fast_loader=True) + sr: Accelerator = Accelerator.load("tests/config/EBSOrbit.yaml", include_locations=False) sr.design.get_lattice().enable_6d() chromaAT = sr.design.get_lattice().get_chrom()[:-1] sr.design.chromaticity.add([0.5, 0.4])