From 51710ed8ad3797014f139d0d9cda47003a48da4f Mon Sep 17 00:00:00 2001 From: Teresia Olsson Date: Fri, 26 Jun 2026 14:25:41 +0200 Subject: [PATCH 01/14] Refactor fileloader for readability. --- pyaml/configuration/fileloader.py | 249 +++++++++++++++++------------ pyaml/configuration/manager.py | 2 +- pyaml/configuration/restfetcher.py | 4 +- 3 files changed, 147 insertions(+), 108 deletions(-) diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index c23497bde..85a1ac058 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -1,11 +1,14 @@ -""" "PyAML configuration file loader.""" +"""PyAML configuration file loader.""" import collections.abc import io import json import logging +from abc import ABC, abstractmethod +from contextlib import contextmanager +from dataclasses import dataclass, field from pathlib import Path -from typing import Union +from typing import Any, Union import yaml from yaml import CLoader @@ -16,9 +19,12 @@ logger = logging.getLogger(__name__) -accepted_suffixes = [".yaml", ".yml", ".json"] +ACCEPTED_SUFFIXES = (".yaml", ".yml", ".json") FILE_PREFIX = "file:" +LOCATION_KEY = "__location__" +FIELD_LOCATIONS_KEY = "__fieldlocations__" + class RootFolder: """ @@ -53,7 +59,9 @@ def expand_path(self, path: str | Path) -> Path: """ path = Path(path) - return path if path.is_absolute() else self._path / path + if not path.is_absolute(): + path = self._path / path + return path.resolve() ROOT = RootFolder() @@ -62,92 +70,153 @@ def expand_path(self, path: str | Path) -> Path: class PyAMLConfigCyclingException(PyAMLException): def __init__(self, error_filename: str, path_stack: list[Path]): self.error_filename = error_filename + parent_file_stack = [parent_path.name for parent_path in path_stack] super().__init__(f"Circular file inclusion of {error_filename}. File list before reaching it: {parent_file_stack}") - pass +@dataclass +class LoadContext: + use_fast_loader: bool = False + include_stack: list[Path] = field(default_factory=list) + + @contextmanager + def loading(self, path: Path): + # Check if the file is currently in the chain + # Raise a cycling error if that is the case + if path in self.include_stack: + raise PyAMLConfigCyclingException(path.name, self.include_stack) + + # Add the file to the stack to record that it is now being loaded + self.include_stack.append(path) + try: + # Run the code in the with block + yield + finally: + # Remove the file to stack to record that it has finished loading + self.include_stack.pop() + + +def load(filename: str, use_fast_loader: bool = False) -> Union[dict, list]: + """Load recursively a configuration setup.""" + + # Create a new context + context = LoadContext(use_fast_loader=use_fast_loader) + + return _load(filename, context) + + +def _load(filename: str, context: LoadContext) -> Union[dict, list]: + path = ROOT.expand_path(filename) + + with context.loading(path): + if filename.endswith((".yaml", ".yml")): + loader = YAMLLoader(path, context) + elif filename.endswith(".json"): + loader = JSONLoader(path, context) + else: + raise PyAMLException(f"{filename} File format not supported (only .yaml .yml or .json)") + + return loader.load() -def load(filename: str, paths_stack: list = None, use_fast_loader: bool = False) -> Union[dict, list]: - """Load recursively a configuration setup""" - if filename.endswith(".yaml") or filename.endswith(".yml"): - l = YAMLLoader(filename, paths_stack, use_fast_loader) - elif filename.endswith(".json"): - l = JSONLoader(filename, paths_stack, use_fast_loader) - else: - raise PyAMLException(f"{filename} File format not supported (only .yaml .yml or .json)") - return l.load() +def _is_supported_file(value: Any) -> bool: + """Return True if the value is a supported configuration file.""" + return isinstance(value, str) and value.endswith(ACCEPTED_SUFFIXES) -# Expand condition -def hasToLoad(value): - return isinstance(value, str) and any(value.endswith(suffix) for suffix in accepted_suffixes) +class ConfigLoader(ABC): + def __init__(self, path: Path, context: LoadContext): + self.path = path + self.context = context -# Loader base class (nested files expansion) -class Loader: - def __init__(self, filename: str, parent_path_stack: list[Path]): - self.path: Path = ROOT.expand_path(filename) - self.files_stack: list[Path] = [] - if parent_path_stack: - if any(self.path.samefile(parent_path) for parent_path in parent_path_stack): - raise PyAMLConfigCyclingException(filename, parent_path_stack) - self.files_stack.extend(parent_path_stack) - self.files_stack.append(self.path) + def expand(self, obj: Union[dict, list, Any]) -> Union[dict, list, Any]: + if isinstance(obj, dict): + return self._expand_dict(obj) + if isinstance(obj, list): + return self._expand_list(obj) + return obj - # Recursively expand a dict - def expand_dict(self, d: dict): - for key, value in d.items(): + def _expand_dict(self, d: dict) -> dict: + for key, value in list(d.items()): try: - if hasToLoad(value): - if value.startswith(FILE_PREFIX): - # remove prefix - stripped_value = value[len(FILE_PREFIX) :] - d[key] = str(ROOT.get() / Path(stripped_value)) + if _is_supported_file(value): + # If it is a reference to another file + resolved = self._resolve_file_reference(value) + if resolved is not None: + d[key] = resolved else: - d[key] = load(value, self.files_stack, self.use_fast_loader) + d[key] = _load(value, self.context) else: - self.expand(value) - except PyAMLConfigCyclingException as pyaml_ex: - location = d.pop("__location__", None) - field_locations = d.pop("__fieldlocations__", None) - location_str = "" - if location: - file, line, col = location - if field_locations and key in field_locations: - location = field_locations[key] - file, line, col = location - location_str = f" in {file} at line {line}, column {col}" - raise PyAMLException(f"Circular file inclusion of {pyaml_ex.error_filename}{location_str}") from pyaml_ex - - # Recursively expand a list - def expand_list(self, l: list): - idx = 0 - while idx < len(l): - value = l[idx] - if hasToLoad(value): - obj = load(value, self.files_stack) - if isinstance(obj, list): - l[idx : idx + 1] = obj - idx += len(obj) + d[key] = self.expand(value) + except PyAMLConfigCyclingException as exc: + self._raise_cycle_error(exc, d, key) + + return d + + def _resolve_file_reference(self, value: str) -> str | None: + """Return an absolute path for FILE_PREFIX references, otherwise None.""" + + if value.startswith(FILE_PREFIX): + stripped_value = value[len(FILE_PREFIX) :] + return str(ROOT.get() / Path(stripped_value)) + return None + + def _raise_cycle_error(self, exc: PyAMLConfigCyclingException, obj: dict, key: Any) -> None: + location = obj.get(LOCATION_KEY) + field_locations = obj.get(FIELD_LOCATIONS_KEY) + + location_str = "" + if location: + file, line, col = location + if field_locations and key in field_locations: + file, line, col = field_locations[key] + location_str = f" in {file} at line {line}, column {col}" + + raise PyAMLException(f"Circular file inclusion of {exc.error_filename}{location_str}") from exc + + def _expand_list(self, items: list) -> list: + expanded = [] + for value in items: + if _is_supported_file(value): + loaded = _load(value, self.context) + if isinstance(loaded, list): + expanded.extend(loaded) else: - l[idx] = obj - idx += 1 + expanded.append(loaded) else: - self.expand(value) - idx += 1 + expanded.append(self.expand(value)) + return expanded - # Recursively expand an object - def expand(self, obj: Union[dict, list]): - if isinstance(obj, dict): - self.expand_dict(obj) - elif isinstance(obj, list): - self.expand_list(obj) - return obj + @abstractmethod + def load(self) -> Union[dict, list]: ... + + +class YAMLLoader(ConfigLoader): + def __init__(self, path: Path, context: LoadContext): + super().__init__(path, context) + self._loader = CLoader if context.use_fast_loader else SafeLineLoader + + def load(self) -> Union[dict, list]: + logger.log(logging.DEBUG, f"Loading YAML file '{self.path}'") + with open(self.path) as file: + try: + return self.expand(yaml.load(file, Loader=self._loader)) + except yaml.YAMLError as exc: + raise PyAMLException(f"{self.path}: {exc}") from exc + + +class JSONLoader(ConfigLoader): + def __init__(self, path: Path, context: LoadContext): + super().__init__(path, context) - # Load a file def load(self) -> Union[dict, list]: - raise PyAMLException(str(self.path) + ": load() method not implemented") + logger.log(logging.DEBUG, f"Loading JSON file '{self.path}'") + with open(self.path) as file: + try: + return self.expand(json.load(file)) + except json.JSONDecodeError as exc: + raise PyAMLException(f"{self.path}: {exc}") from exc class SafeLineLoader(SafeLoader): @@ -168,6 +237,7 @@ def construct_mapping(self, node, deep=False): "found unhashable key", key_node.start_mark, ) + value = self.construct_object(value_node, deep=deep) mapping[key] = value field_mapping[key] = ( @@ -177,41 +247,10 @@ def construct_mapping(self, node, deep=False): ) # Add location information inside the dict - mapping["__location__"] = ( + mapping[LOCATION_KEY] = ( self.filename, node.start_mark.line + 1, node.start_mark.column + 1, ) - mapping["__fieldlocations__"] = field_mapping + mapping[FIELD_LOCATIONS_KEY] = field_mapping return mapping - - -# YAML loader -class YAMLLoader(Loader): - def __init__(self, filename: str, parent_paths_stack: list, use_fast_loader: bool): - super().__init__(filename, parent_paths_stack) - self._loader = SafeLineLoader if not use_fast_loader else CLoader - self.use_fast_loader = use_fast_loader - - def load(self) -> Union[dict, list]: - logger.log(logging.DEBUG, f"Loading YAML file '{self.path}'") - with open(self.path) as file: - try: - return self.expand(yaml.load(file, Loader=self._loader)) - except yaml.YAMLError as e: - raise PyAMLException(str(self.path) + ": " + str(e)) from e - - -# JSON loader -class JSONLoader(Loader): - def __init__(self, filename: str, parent_paths_stack: list, use_fast_loader: bool): - super().__init__(filename, parent_paths_stack) - self.use_fast_loader = False - - def load(self) -> Union[dict, list]: - logger.log(logging.DEBUG, f"Loading JSON file '{self.path}'") - with open(self.path) as file: - try: - return self.expand(json.load(file)) - except json.JSONDecodeError as e: - raise PyAMLException(str(self.path) + ": " + str(e)) from e diff --git a/pyaml/configuration/manager.py b/pyaml/configuration/manager.py index 8b3d5b226..8a568b14f 100644 --- a/pyaml/configuration/manager.py +++ b/pyaml/configuration/manager.py @@ -628,7 +628,7 @@ def _load_payload( source_root = resolved_path.parent try: ROOT.set(source_root) - fragment = load(resolved_path.name, None, use_fast_loader) + fragment = load(resolved_path.name, use_fast_loader) finally: ROOT.set(previous_root) diff --git a/pyaml/configuration/restfetcher.py b/pyaml/configuration/restfetcher.py index ea80b58e7..8140f953a 100644 --- a/pyaml/configuration/restfetcher.py +++ b/pyaml/configuration/restfetcher.py @@ -15,7 +15,7 @@ from yaml import CLoader from ..common.exception import PyAMLConfigException -from .fileloader import FILE_PREFIX, SafeLineLoader, accepted_suffixes +from .fileloader import ACCEPTED_SUFFIXES, FILE_PREFIX, SafeLineLoader REMOTE_BASE_URL_KEY = "__baseurl__" SourceRoot = Path | str | None @@ -181,7 +181,7 @@ def _is_config_reference(value: Any) -> bool: parsed = urlparse(value) path = parsed.path if parsed.scheme in _REMOTE_SCHEMES else value - return any(path.endswith(suffix) for suffix in accepted_suffixes) + return any(path.endswith(suffix) for suffix in ACCEPTED_SUFFIXES) def _resolve_remote_config_reference(reference: str, base_url: str) -> str: From 9947b2e0be943d4a01887801a657105156a12754 Mon Sep 17 00:00:00 2001 From: Teresia Olsson Date: Fri, 26 Jun 2026 14:32:42 +0200 Subject: [PATCH 02/14] Update docstrings in fileloader. --- pyaml/configuration/fileloader.py | 81 ++++++++++++++++++++++++------- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index 85a1ac058..1fa18e204 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -27,35 +27,30 @@ class RootFolder: - """ - Manage the root directory used to resolve relative configuration paths. - """ + """Manage the root directory used to resolve configuration paths.""" def __init__(self, path: str | Path | None = None): - # Resolve the path is given otherwise set to the current working directory + """Create a root folder. + + If no path is provided, the current working directory is used. + """ if path is None: self._path = Path.cwd().resolve() else: self._path = Path(path).resolve() def set(self, path: str | Path) -> None: - """ - Set the root path for configuration files. - """ + """Set the root path used for resolving relative configuration files.""" self._path = Path(path).resolve() def get(self) -> Path: - """ - Get the root path for configuration files. - """ + """Return the current root path.""" return self._path def expand_path(self, path: str | Path) -> Path: - """ - Return an absolute configuration path. + """Return an absolute, normalized configuration path. - Relative paths are interpreted relative to the configured root - folder. Absolute paths are returned unchanged. + Relative paths are interpreted relative to the configured root folder. """ path = Path(path) @@ -68,7 +63,16 @@ def expand_path(self, path: str | Path) -> Path: class PyAMLConfigCyclingException(PyAMLException): + """Raised when a configuration file includes itself through a cycle.""" + def __init__(self, error_filename: str, path_stack: list[Path]): + """Create a circular-include error. + + Args: + error_filename: The file that triggered the cycle. + path_stack: The include chain leading to the cycle. + """ + self.error_filename = error_filename parent_file_stack = [parent_path.name for parent_path in path_stack] @@ -77,11 +81,20 @@ def __init__(self, error_filename: str, path_stack: list[Path]): @dataclass class LoadContext: + """Track state for one recursive configuration-loading session.""" + use_fast_loader: bool = False include_stack: list[Path] = field(default_factory=list) @contextmanager def loading(self, path: Path): + """Register a file as currently being loaded and remove it afterward. + + Raises: + PyAMLConfigCyclingException: If the file is already on the active + include stack. + """ + # Check if the file is currently in the chain # Raise a cycling error if that is the case if path in self.include_stack: @@ -98,7 +111,7 @@ def loading(self, path: Path): def load(filename: str, use_fast_loader: bool = False) -> Union[dict, list]: - """Load recursively a configuration setup.""" + """Load a configuration file and recursively expand nested file references.""" # Create a new context context = LoadContext(use_fast_loader=use_fast_loader) @@ -107,6 +120,8 @@ def load(filename: str, use_fast_loader: bool = False) -> Union[dict, list]: def _load(filename: str, context: LoadContext) -> Union[dict, list]: + """Load a single configuration file using the appropriate parser.""" + path = ROOT.expand_path(filename) with context.loading(path): @@ -121,16 +136,22 @@ def _load(filename: str, context: LoadContext) -> Union[dict, list]: def _is_supported_file(value: Any) -> bool: - """Return True if the value is a supported configuration file.""" + """Return True if the value looks like a supported configuration file name.""" return isinstance(value, str) and value.endswith(ACCEPTED_SUFFIXES) class ConfigLoader(ABC): + """Base class for loaders that expand nested configuration references.""" + def __init__(self, path: Path, context: LoadContext): + """Store the file path and shared loading context.""" + self.path = path self.context = context def expand(self, obj: Union[dict, list, Any]) -> Union[dict, list, Any]: + """Recursively expand nested dictionaries and lists.""" + if isinstance(obj, dict): return self._expand_dict(obj) if isinstance(obj, list): @@ -138,6 +159,8 @@ def expand(self, obj: Union[dict, list, Any]) -> Union[dict, list, Any]: return obj def _expand_dict(self, d: dict) -> dict: + """Expand any supported file references found inside a dictionary.""" + for key, value in list(d.items()): try: if _is_supported_file(value): @@ -163,6 +186,8 @@ def _resolve_file_reference(self, value: str) -> str | None: return None def _raise_cycle_error(self, exc: PyAMLConfigCyclingException, obj: dict, key: Any) -> None: + """Re-raise a cycle error with file location information, if available.""" + location = obj.get(LOCATION_KEY) field_locations = obj.get(FIELD_LOCATIONS_KEY) @@ -176,6 +201,8 @@ def _raise_cycle_error(self, exc: PyAMLConfigCyclingException, obj: dict, key: A raise PyAMLException(f"Circular file inclusion of {exc.error_filename}{location_str}") from exc def _expand_list(self, items: list) -> list: + """Expand supported file references inside a list.""" + expanded = [] for value in items: if _is_supported_file(value): @@ -189,15 +216,23 @@ def _expand_list(self, items: list) -> list: return expanded @abstractmethod - def load(self) -> Union[dict, list]: ... + def load(self) -> Union[dict, list]: + """Load and parse the current configuration file.""" + ... class YAMLLoader(ConfigLoader): + """Load and expand YAML configuration files.""" + def __init__(self, path: Path, context: LoadContext): + """Create a YAML loader for the given file.""" + super().__init__(path, context) self._loader = CLoader if context.use_fast_loader else SafeLineLoader def load(self) -> Union[dict, list]: + """Parse the YAML file and expand nested configuration references.""" + logger.log(logging.DEBUG, f"Loading YAML file '{self.path}'") with open(self.path) as file: try: @@ -207,10 +242,16 @@ def load(self) -> Union[dict, list]: class JSONLoader(ConfigLoader): + """Load and expand JSON configuration files.""" + def __init__(self, path: Path, context: LoadContext): + """Create a JSON loader for the given file.""" + super().__init__(path, context) def load(self) -> Union[dict, list]: + """Parse the JSON file and expand nested configuration references.""" + logger.log(logging.DEBUG, f"Loading JSON file '{self.path}'") with open(self.path) as file: try: @@ -220,11 +261,17 @@ def load(self) -> Union[dict, list]: class SafeLineLoader(SafeLoader): + """YAML loader that preserves line and column information for mappings.""" + def __init__(self, stream): + """Create the YAML loader and record the source filename.""" + super().__init__(stream) self.filename = stream.name if isinstance(stream, io.TextIOWrapper) else "" def construct_mapping(self, node, deep=False): + """Build a mapping and attach location metadata to it.""" + mapping = {} field_mapping = {} From f83fe443f5f4a01df88a9fed4b8a8908efbb8fd1 Mon Sep 17 00:00:00 2001 From: Teresia Olsson Date: Fri, 26 Jun 2026 16:10:56 +0200 Subject: [PATCH 03/14] Rename use_fast_loader to include_locations. --- pyaml/accelerator.py | 12 +++++----- pyaml/configuration/fileloader.py | 30 +++++++++++++++++++---- pyaml/configuration/manager.py | 12 +++++----- pyaml/configuration/restfetcher.py | 32 ++++++++++++------------- tests/arrays/test_arrays.py | 4 ++-- tests/magnet/test_serialized_magnets.py | 10 ++++---- tests/tuning_tools/test_tuning_tools.py | 8 +++---- 7 files changed, 64 insertions(+), 44 deletions(-) diff --git a/pyaml/accelerator.py b/pyaml/accelerator.py index a35dd4e24..cbb2fcbd9 100644 --- a/pyaml/accelerator.py +++ b/pyaml/accelerator.py @@ -275,7 +275,7 @@ def from_dict(config_dict: dict, ignore_external=False) -> "Accelerator": return Factory.build(config_dict, ignore_external) @staticmethod - def load(filename: str, use_fast_loader: bool = False, ignore_external=False) -> "Accelerator": + def load(filename: str, include_locations: bool = True, ignore_external=False) -> "Accelerator": """ Load an accelerator from a config file. @@ -283,11 +283,11 @@ def load(filename: str, use_fast_loader: bool = False, ignore_external=False) -> ---------- filename : str Configuration file name, yaml or json. - use_fast_loader : bool - Use fast yaml loader. When specified, - no line number are reported in case of error, + include_locations : bool + When False, use faster loader but no line number + are reported in case of error, only the element name that triggered the error - will be reported in the exception) + will be reported in the exception ignore_external : bool Ignore external modules and return None for object that cannot be created. pydantic schema that support that an @@ -295,7 +295,7 @@ def load(filename: str, use_fast_loader: bool = False, ignore_external=False) -> """ manager = ConfigurationManager() try: - manager.add(filename, use_fast_loader=use_fast_loader) + manager.add(filename, include_locations=include_locations) except UnsupportedConfigurationRootError as ex: raise PyAMLConfigException( "Accelerator.load() expects a 'pyaml.accelerator' root configuration. " diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index 1fa18e204..77a1a6f4c 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -83,7 +83,7 @@ def __init__(self, error_filename: str, path_stack: list[Path]): class LoadContext: """Track state for one recursive configuration-loading session.""" - use_fast_loader: bool = False + include_locations: bool = True include_stack: list[Path] = field(default_factory=list) @contextmanager @@ -110,11 +110,15 @@ def loading(self, path: Path): self.include_stack.pop() -def load(filename: str, use_fast_loader: bool = False) -> Union[dict, list]: - """Load a configuration file and recursively expand nested file references.""" +def load(filename: str, include_locations: bool = True) -> Union[dict, list]: + """Load a configuration file. + + When include_locations is False, uses the faster C-based YAML loader + and skips including source location metadata. + """ # Create a new context - context = LoadContext(use_fast_loader=use_fast_loader) + context = LoadContext(include_locations=include_locations) return _load(filename, context) @@ -228,7 +232,7 @@ def __init__(self, path: Path, context: LoadContext): """Create a YAML loader for the given file.""" super().__init__(path, context) - self._loader = CLoader if context.use_fast_loader else SafeLineLoader + self._loader = SafeLineLoader if context.include_locations else CLoader def load(self) -> Union[dict, list]: """Parse the YAML file and expand nested configuration references.""" @@ -260,6 +264,22 @@ def load(self) -> Union[dict, list]: raise PyAMLException(f"{self.path}: {exc}") from exc +@dataclass(frozen=True) +class Location: + file: str + line: int + column: int + + +@dataclass +class LoadedConfig: + data: dict | list + locations: dict[tuple[Any, ...], Location] + + def location_for(self, path: tuple[Any, ...]) -> Location | None: + return self.locations.get(path) + + class SafeLineLoader(SafeLoader): """YAML loader that preserves line and column information for mappings.""" diff --git a/pyaml/configuration/manager.py b/pyaml/configuration/manager.py index 8a568b14f..61e844ca1 100644 --- a/pyaml/configuration/manager.py +++ b/pyaml/configuration/manager.py @@ -122,7 +122,7 @@ def add(self, payload, **kwargs) -> None: Fragment to merge into the current aggregated state. source_name : str, optional Explicit source label to associate with the fragment. - use_fast_loader : bool, optional + include_locations : bool, optional Forwarded to the configuration loader. Examples @@ -145,7 +145,7 @@ def add(self, payload, **kwargs) -> None: ... ) """ source_name = kwargs.pop("source_name", None) - use_fast_loader = kwargs.pop("use_fast_loader", False) + include_locations = kwargs.pop("include_locations", False) if kwargs: unknown = ", ".join(sorted(kwargs)) raise PyAMLConfigException(f"Unsupported ConfigurationManager.add() arguments: {unknown}") @@ -153,7 +153,7 @@ def add(self, payload, **kwargs) -> None: fragment, inferred_source_name, source_root = self._load_payload( payload, source_name=source_name, - use_fast_loader=use_fast_loader, + include_locations=include_locations, ) prepared = self._prepare_fragment(fragment, source_root, inferred_source_name) self._merge_fragment(prepared, inferred_source_name) @@ -599,14 +599,14 @@ def _load_payload( payload, *, source_name: str | None, - use_fast_loader: bool, + include_locations: bool, ) -> tuple[dict[str, Any], str, SourceRoot]: if isinstance(payload, dict): return copy.deepcopy(payload), source_name or "", None payload_path = self._coerce_path(payload) if is_remote_url(payload_path): - fragment, source_root = fetch_remote_config(payload_path, use_fast_loader=use_fast_loader) + fragment, source_root = fetch_remote_config(payload_path, include_locations=include_locations) if not self._build_root_locked: self._build_root = source_root self._build_root_locked = True @@ -628,7 +628,7 @@ def _load_payload( source_root = resolved_path.parent try: ROOT.set(source_root) - fragment = load(resolved_path.name, use_fast_loader) + fragment = load(resolved_path.name, include_locations) finally: ROOT.set(previous_root) diff --git a/pyaml/configuration/restfetcher.py b/pyaml/configuration/restfetcher.py index 8140f953a..380da6176 100644 --- a/pyaml/configuration/restfetcher.py +++ b/pyaml/configuration/restfetcher.py @@ -32,9 +32,9 @@ def is_remote_url(value: str) -> bool: return urlparse(value).scheme in _REMOTE_SCHEMES -def fetch_remote_config(url: str, *, use_fast_loader: bool = False) -> tuple[dict[str, Any] | list[Any], str]: +def fetch_remote_config(url: str, *, include_locations: bool = True) -> tuple[dict[str, Any] | list[Any], str]: normalized_url = _normalize_remote_url(url) - expanded = _load_remote_document(normalized_url, use_fast_loader=use_fast_loader, stack=[]) + expanded = _load_remote_document(normalized_url, include_locations=include_locations, stack=[]) return expanded, _remote_base_url(normalized_url) @@ -61,15 +61,15 @@ def _normalize_remote_url(url: str) -> str: def _load_remote_document( url: str, *, - use_fast_loader: bool, + include_locations: bool, stack: list[str], ) -> dict[str, Any] | list[Any]: if url in stack: raise PyAMLConfigException(f"Circular remote configuration inclusion detected for '{url}'.") payload, content_type = _download_text(url) - document = _parse_remote_document(url, payload, content_type, use_fast_loader=use_fast_loader) - return _expand_remote_value(document, _remote_base_url(url), stack + [url], use_fast_loader=use_fast_loader) + document = _parse_remote_document(url, payload, content_type, include_locations=include_locations) + return _expand_remote_value(document, _remote_base_url(url), stack + [url], include_locations=include_locations) def _download_text(url: str) -> tuple[str, str]: @@ -93,7 +93,7 @@ def _parse_remote_document( payload: str, content_type: str, *, - use_fast_loader: bool, + include_locations: bool, ) -> dict[str, Any] | list[Any]: suffix = Path(urlparse(url).path).suffix.lower() @@ -103,7 +103,7 @@ def _parse_remote_document( except json.JSONDecodeError as ex: raise PyAMLConfigException(f"{url}: {ex}") from ex - loader = CLoader if use_fast_loader else SafeLineLoader + loader = CLoader if include_locations else SafeLineLoader try: stream = _NamedStringIO(payload, url) return yaml.load(stream, Loader=loader) @@ -111,11 +111,11 @@ def _parse_remote_document( raise PyAMLConfigException(f"{url}: {ex}") from ex -def _expand_remote_value(value, base_url: str, stack: list[str], *, use_fast_loader: bool): +def _expand_remote_value(value, base_url: str, stack: list[str], *, include_locations: bool): if isinstance(value, dict): - return _expand_remote_dict(value, base_url, stack, use_fast_loader=use_fast_loader) + return _expand_remote_dict(value, base_url, stack, include_locations=include_locations) if isinstance(value, list): - return _expand_remote_list(value, base_url, stack, use_fast_loader=use_fast_loader) + return _expand_remote_list(value, base_url, stack, include_locations=include_locations) return value @@ -124,14 +124,14 @@ def _expand_remote_dict( base_url: str, stack: list[str], *, - use_fast_loader: bool, + include_locations: bool, ) -> dict[str, Any]: values.setdefault(REMOTE_BASE_URL_KEY, base_url) for key, value in list(values.items()): if _is_config_reference(value): values[key] = _load_remote_document( _resolve_remote_config_reference(value, base_url), - use_fast_loader=use_fast_loader, + include_locations=include_locations, stack=stack, ) continue @@ -140,18 +140,18 @@ def _expand_remote_dict( values[key] = resolve_reference(value[len(FILE_PREFIX) :], base_url) continue - values[key] = _expand_remote_value(value, base_url, stack, use_fast_loader=use_fast_loader) + values[key] = _expand_remote_value(value, base_url, stack, include_locations=include_locations) return values -def _expand_remote_list(values: list[Any], base_url: str, stack: list[str], *, use_fast_loader: bool) -> list[Any]: +def _expand_remote_list(values: list[Any], base_url: str, stack: list[str], *, include_locations: bool) -> list[Any]: index = 0 while index < len(values): value = values[index] if _is_config_reference(value): expanded = _load_remote_document( _resolve_remote_config_reference(value, base_url), - use_fast_loader=use_fast_loader, + include_locations=include_locations, stack=stack, ) if isinstance(expanded, list): @@ -167,7 +167,7 @@ def _expand_remote_list(values: list[Any], base_url: str, stack: list[str], *, u index += 1 continue - values[index] = _expand_remote_value(value, base_url, stack, use_fast_loader=use_fast_loader) + values[index] = _expand_remote_value(value, base_url, stack, include_locations=include_locations) index += 1 return values diff --git a/tests/arrays/test_arrays.py b/tests/arrays/test_arrays.py index 526d4ab5b..eb72fdc87 100644 --- a/tests/arrays/test_arrays.py +++ b/tests/arrays/test_arrays.py @@ -201,7 +201,7 @@ def test_arrays(install_test_package): # Test dynamic arrays - sr: Accelerator = Accelerator.load("tests/config/EBSOrbit.yaml", use_fast_loader=True) + sr: Accelerator = Accelerator.load("tests/config/EBSOrbit.yaml", include_locations=False) ae = ElementArray("All", sr.design.get_all_elements()) acfm = ElementArray("AllCFM", sr.design.get_all_cfm_magnets(), use_aggregator=False) @@ -240,7 +240,7 @@ def test_arrays(install_test_package): ], ) def test_serialized_magnets_arrays(sr_file): - sr: Accelerator = Accelerator.load(sr_file, use_fast_loader=True, ignore_external=True) + sr: Accelerator = Accelerator.load(sr_file, include_locations=False, ignore_external=True) the_serie = sr.design.get_serialized_magnets("series") strength = the_serie.strengths.get() assert len(strength) == 1 diff --git a/tests/magnet/test_serialized_magnets.py b/tests/magnet/test_serialized_magnets.py index b0fc50baf..130379e3d 100644 --- a/tests/magnet/test_serialized_magnets.py +++ b/tests/magnet/test_serialized_magnets.py @@ -24,7 +24,7 @@ def check_no_diff(array: list[np.float64]) -> bool: ], ) def test_config_load(sr_file): - sr: Accelerator = Accelerator.load(sr_file, use_fast_loader=True, ignore_external=True) + sr: Accelerator = Accelerator.load(sr_file, include_locations=False, ignore_external=True) assert sr is not None magnets = [ sr.design.get_element("QF8B-C04"), @@ -55,7 +55,7 @@ def test_config_load(sr_file): ], ) def test_magnet_modification(sr_file): - sr = Accelerator.load(sr_file, use_fast_loader=True, ignore_external=True) + sr = Accelerator.load(sr_file, include_locations=False, ignore_external=True) print(sr.yellow_pages) @@ -107,7 +107,7 @@ def test_magnet_modification(sr_file): ], ) def test_tune(sr_file): - sr = Accelerator.load(sr_file, use_fast_loader=True, ignore_external=True) + sr = Accelerator.load(sr_file, include_locations=False, ignore_external=True) sr.design.get_lattice().disable_6d() m = sr.design.get_serialized_magnet("QF1A") @@ -161,7 +161,7 @@ def test_tune(sr_file): ], ) def test_get_device_names(sr_file): - sr: Accelerator = Accelerator.load(sr_file, use_fast_loader=True, ignore_external=True) + sr: Accelerator = Accelerator.load(sr_file, include_locations=False, ignore_external=True) sm: SerializedMagnets = sr.design.get_serialized_magnet("QF1A") device_names = sm.get_device_names() @@ -191,7 +191,7 @@ def get_strengths_from_lattice(sr: Accelerator, sm: SerializedMagnets) -> list: ], ) def test_strength_computation(sr_file): - sr: Accelerator = Accelerator.load(sr_file, use_fast_loader=True, ignore_external=True) + sr: Accelerator = Accelerator.load(sr_file, include_locations=False, ignore_external=True) sm: SerializedMagnets = sr.design.get_serialized_magnet("QF1A") assert sm.get_nb_magnets() == 31 sm.strength.set(24.0) diff --git a/tests/tuning_tools/test_tuning_tools.py b/tests/tuning_tools/test_tuning_tools.py index 72fdc5d3b..5c748b003 100644 --- a/tests/tuning_tools/test_tuning_tools.py +++ b/tests/tuning_tools/test_tuning_tools.py @@ -10,7 +10,7 @@ def callback(action: Action, data: dict): def test_tune_tool(): - sr = Accelerator.load("tests/config/EBSTune.yaml", use_fast_loader=True) + sr = Accelerator.load("tests/config/EBSTune.yaml", include_locations=False) sr.design.get_lattice().disable_6d() sr.design.trm.measure(callback=callback) sr.design.trm.save("tunemat.json") @@ -22,7 +22,7 @@ def test_tune_tool(): def test_tune_add(): - sr = Accelerator.load("tests/config/EBSTune.yaml", use_fast_loader=True) + sr = Accelerator.load("tests/config/EBSTune.yaml", include_locations=False) sr.design.get_lattice().disable_6d() sr.design.tune.load("tunemat.json") tune_initial = sr.design.tune.readback() @@ -33,7 +33,7 @@ def test_tune_add(): def test_chroma_tool(): - sr: Accelerator = Accelerator.load("tests/config/EBSOrbit.yaml", use_fast_loader=True) + sr: Accelerator = Accelerator.load("tests/config/EBSOrbit.yaml", include_locations=False) sr.design.get_lattice().enable_6d() sr.design.chromaticity.set([8.0, 5.0], iter=2) QpAT = sr.design.get_lattice().get_chrom()[:-1] @@ -45,7 +45,7 @@ def test_chroma_tool(): def test_chroma_add(): - sr: Accelerator = Accelerator.load("tests/config/EBSOrbit.yaml", use_fast_loader=True) + sr: Accelerator = Accelerator.load("tests/config/EBSOrbit.yaml", include_locations=False) sr.design.get_lattice().enable_6d() chromaAT = sr.design.get_lattice().get_chrom()[:-1] sr.design.chromaticity.add([0.5, 0.4]) From 50062711a24346d0ca7e60ed8bc5d18c80d7ec88 Mon Sep 17 00:00:00 2001 From: Teresia Olsson Date: Fri, 26 Jun 2026 16:45:20 +0200 Subject: [PATCH 04/14] Change accelerator to use fast loader as default. --- pyaml/accelerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyaml/accelerator.py b/pyaml/accelerator.py index cbb2fcbd9..b813c721e 100644 --- a/pyaml/accelerator.py +++ b/pyaml/accelerator.py @@ -275,7 +275,7 @@ def from_dict(config_dict: dict, ignore_external=False) -> "Accelerator": return Factory.build(config_dict, ignore_external) @staticmethod - def load(filename: str, include_locations: bool = True, ignore_external=False) -> "Accelerator": + def load(filename: str, include_locations: bool = False, ignore_external=False) -> "Accelerator": """ Load an accelerator from a config file. From 2718f49d99ae8195337b9d0429902f54e87ed0d1 Mon Sep 17 00:00:00 2001 From: Teresia Olsson Date: Fri, 26 Jun 2026 17:11:29 +0200 Subject: [PATCH 05/14] Update tests to take into account that fast loader is now the default. --- tests/common/test_errors.py | 12 ++++++------ tests/configuration/test_factory.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/common/test_errors.py b/tests/common/test_errors.py index 77a1c5a2d..f7a9672d8 100644 --- a/tests/common/test_errors.py +++ b/tests/common/test_errors.py @@ -13,24 +13,24 @@ ) def test_tune(install_test_package): with pytest.raises(PyAMLConfigException) as exc: - ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_1.yaml") + ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_1.yaml", include_locations=True) assert "MagnetArray HCORR : duplicate name SH1A-C02-H @index 2" in str(exc) with pytest.raises(PyAMLConfigException) as exc: - ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_2.yaml") + ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_2.yaml", include_locations=True) assert "BPMArray BPM : duplicate name BPM_C04-06 @index 3" in str(exc) with pytest.raises(PyAMLConfigException) as exc: - ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_3.yaml") + ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_3.yaml", include_locations=True) assert "Configuration entry 'BPM_C04-06' is duplicated inside category 'devices'" in str(exc) assert "bad_conf_duplicate_3.yaml" in str(exc) assert "line 43, column 3" in str(exc) with pytest.raises(PyAMLConfigException) as exc: - ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_4.yaml") + ml: Accelerator = Accelerator.load("tests/config/bad_conf_duplicate_4.yaml", include_locations=True) assert "MagnetArray HCORR : duplicate name SH1A-C02-H @index 2" in str(exc) - sr: Accelerator = Accelerator.load("tests/config/EBSTune.yaml") + sr: Accelerator = Accelerator.load("tests/config/EBSTune.yaml", include_locations=True) m1 = sr.live.get_magnet("QF1E-C04") m2 = sr.design.get_magnet("QF1A-C05") with pytest.raises(PyAMLException) as exc: @@ -64,7 +64,7 @@ def test_duplicate_error_reports_source_line_and_column_across_files(tmp_path): ) with pytest.raises(PyAMLConfigException) as exc: - Accelerator.load(str(root)) + Accelerator.load(str(root), include_locations=True) message = str(exc.value) assert "Configuration entry 'BPM_DUPLICATE' is duplicated inside category 'devices'" in message diff --git a/tests/configuration/test_factory.py b/tests/configuration/test_factory.py index 97da045b6..d81a0a16c 100644 --- a/tests/configuration/test_factory.py +++ b/tests/configuration/test_factory.py @@ -46,7 +46,7 @@ def test_error_location(test_file): ) def test_error_cycles(test_file): with pytest.raises(PyAMLException) as exc: - ml: Accelerator = Accelerator.load(test_file) + ml: Accelerator = Accelerator.load(test_file, include_locations=True) assert "Circular file inclusion of " in str(exc.value) if not test_file.endswith(".json"): From 77b204a8255f15f4e7a21ed5464c9e9b38f018cf Mon Sep 17 00:00:00 2001 From: Teresia Olsson Date: Fri, 26 Jun 2026 17:29:39 +0200 Subject: [PATCH 06/14] Change default behaviour of load to use fast loader. --- pyaml/configuration/fileloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index 77a1a6f4c..10413b4a2 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -110,7 +110,7 @@ def loading(self, path: Path): self.include_stack.pop() -def load(filename: str, include_locations: bool = True) -> Union[dict, list]: +def load(filename: str, include_locations: bool = False) -> Union[dict, list]: """Load a configuration file. When include_locations is False, uses the faster C-based YAML loader From acf7252b958d9901692e0c820a8fd5d225d4e291 Mon Sep 17 00:00:00 2001 From: Teresia Olsson Date: Fri, 26 Jun 2026 17:33:28 +0200 Subject: [PATCH 07/14] Cleanup fileloader and remove classes that are not used. --- pyaml/configuration/fileloader.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index 10413b4a2..a12cdc69c 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -264,22 +264,6 @@ def load(self) -> Union[dict, list]: raise PyAMLException(f"{self.path}: {exc}") from exc -@dataclass(frozen=True) -class Location: - file: str - line: int - column: int - - -@dataclass -class LoadedConfig: - data: dict | list - locations: dict[tuple[Any, ...], Location] - - def location_for(self, path: tuple[Any, ...]) -> Location | None: - return self.locations.get(path) - - class SafeLineLoader(SafeLoader): """YAML loader that preserves line and column information for mappings.""" From 85a25c50e1a292fbb51045c68494750d0541f2c3 Mon Sep 17 00:00:00 2001 From: Teresia Olsson Date: Fri, 26 Jun 2026 17:37:46 +0200 Subject: [PATCH 08/14] Update so default load behaviour is to use fast loader everywhere. --- pyaml/configuration/fileloader.py | 2 +- tests/configuration/test_factory.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index a12cdc69c..ab48372c5 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -83,7 +83,7 @@ def __init__(self, error_filename: str, path_stack: list[Path]): class LoadContext: """Track state for one recursive configuration-loading session.""" - include_locations: bool = True + include_locations: bool = False include_stack: list[Path] = field(default_factory=list) @contextmanager diff --git a/tests/configuration/test_factory.py b/tests/configuration/test_factory.py index d81a0a16c..e584bcde2 100644 --- a/tests/configuration/test_factory.py +++ b/tests/configuration/test_factory.py @@ -27,7 +27,7 @@ def test_error_location(test_file): try: with pytest.raises(PyAMLConfigException) as exc: ROOT.set("tests/config") - cfg = load("bad_conf.yml") + cfg = load("bad_conf.yml", include_locations=True) Factory.build(cfg, False) finally: ROOT.set(previous_root) From 2ae733ed3155957eacd213f41aa303c9b7d5c662 Mon Sep 17 00:00:00 2001 From: Teresia Olsson Date: Fri, 26 Jun 2026 17:39:03 +0200 Subject: [PATCH 09/14] Add tests for fileloader. --- tests/configuration/test_fileloader.py | 217 +++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 tests/configuration/test_fileloader.py diff --git a/tests/configuration/test_fileloader.py b/tests/configuration/test_fileloader.py new file mode 100644 index 000000000..b76273fc7 --- /dev/null +++ b/tests/configuration/test_fileloader.py @@ -0,0 +1,217 @@ +import json + +import pytest + +from pyaml import PyAMLException +from pyaml.configuration.fileloader import ( + FIELD_LOCATIONS_KEY, + FILE_PREFIX, + LOCATION_KEY, + ROOT, + LoadContext, + RootFolder, + _is_supported_file, + load, +) + + +def test_rootfolder_uses_current_working_directory_when_no_path(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + + root = RootFolder() + + assert root.get() == tmp_path.resolve() + + +def test_rootfolder_sets_explicit_path(tmp_path): + root = RootFolder(tmp_path) + + assert root.get() == tmp_path.resolve() + + +def test_rootfolder_expands_relative_paths(tmp_path): + root = RootFolder(tmp_path) + + assert root.expand_path("config.yaml") == (tmp_path / "config.yaml").resolve() + + +def test_rootfolder_expands_absolute_paths(tmp_path): + root = RootFolder(tmp_path) + absolute = (tmp_path / "nested" / "config.yaml").resolve() + + assert root.expand_path(absolute) == absolute + + +def test_rootfolder_set_updates_root(tmp_path): + root = RootFolder(tmp_path) + new_root = tmp_path / "other" + + before = root.expand_path("config.yaml") + root.set(new_root) + after = root.expand_path("config.yaml") + + assert root.get() == new_root.resolve() + assert before == (tmp_path / "config.yaml").resolve() + assert after == (new_root / "config.yaml").resolve() + + +def test_is_supported_file(): + assert _is_supported_file("config.yaml") + assert _is_supported_file("config.yml") + assert _is_supported_file("config.json") + + assert not _is_supported_file("config.txt") + assert not _is_supported_file(1) + assert not _is_supported_file(None) + + +def test_load_context_adds_and_removes_paths(tmp_path): + ctx = LoadContext() + path = tmp_path / "config.yaml" + + with ctx.loading(path): + assert ctx.include_stack == [path] + + assert ctx.include_stack == [] + + +def test_load_context_cleans_up_after_exception(tmp_path): + ctx = LoadContext() + path = tmp_path / "config.yaml" + + with pytest.raises(RuntimeError): + with ctx.loading(path): + raise RuntimeError + + assert ctx.include_stack == [] + + +def test_load_yaml(tmp_path): + ROOT.set(tmp_path) + + config = tmp_path / "config.yaml" + config.write_text("a: 1\nb: hello\n") + + result = load("config.yaml") + + assert result["a"] == 1 + assert result["b"] == "hello" + + +def test_load_json(tmp_path): + ROOT.set(tmp_path) + + config = tmp_path / "config.json" + config.write_text(json.dumps({"a": 1, "b": "hello"})) + + result = load("config.json") + + assert result == {"a": 1, "b": "hello"} + + +def test_load_nested_yaml(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "child.yaml").write_text("answer: 42\n") + + (tmp_path / "parent.yaml").write_text("child: child.yaml\n") + + result = load("parent.yaml") + + assert result["child"]["answer"] == 42 + + +def test_load_nested_json(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "child.json").write_text(json.dumps({"answer": 42})) + + (tmp_path / "parent.json").write_text(json.dumps({"child": "child.json"})) + + result = load("parent.json") + + assert result["child"]["answer"] == 42 + + +def test_load_list_include_extends_list(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "items.yaml").write_text("- one\n- two\n") + + (tmp_path / "parent.yaml").write_text( + """ + values: + - start + - items.yaml + - end + """.strip() + ) + + result = load("parent.yaml") + + assert result["values"] == [ + "start", + "one", + "two", + "end", + ] + + +def test_load_file_prefix_returns_absolute_path(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "config.yaml").write_text(f'target: "{FILE_PREFIX}subdir/file.yaml"\n') + + result = load("config.yaml") + + assert result["target"] == str((tmp_path / "subdir" / "file.yaml").resolve()) + + +def test_load_yaml_omits_locations_by_default(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "config.yaml").write_text("a: 1\n") + + result = load("config.yaml") + + assert LOCATION_KEY not in result + assert FIELD_LOCATIONS_KEY not in result + + +def test_load_yaml_includes_locations_when_enables(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "config.yaml").write_text("a: 1\n") + + result = load("config.yaml", include_locations=True) + + assert LOCATION_KEY in result + assert FIELD_LOCATIONS_KEY in result + + +def test_load_invalid_yaml_raises_pyaml_exception(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "bad.yaml").write_text("a:\n - 1\n - 2\n") + + with pytest.raises(PyAMLException, match="bad.yaml"): + load("bad.yaml") + + +def test_load_invalid_json_raises_pyaml_exception(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "bad.json").write_text('{"a": }') + + with pytest.raises(PyAMLException, match="bad.json"): + load("bad.json") + + +def test_load_circular_include_raises_pyaml_exception(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "a.yaml").write_text("b: b.yaml\n") + (tmp_path / "b.yaml").write_text("a: a.yaml\n") + + with pytest.raises(PyAMLException, match="Circular file inclusion"): + load("a.yaml") From b2429b372821f86b3f2ef4a192a71f3ce9054300 Mon Sep 17 00:00:00 2001 From: Teresia Olsson Date: Wed, 1 Jul 2026 17:12:17 +0200 Subject: [PATCH 10/14] Change to using resolvers in file loader. --- pyaml/configuration/fileloader.py | 209 ++++++++++++++++++++---- tests/config/EBSOrbit.yaml | 4 +- tests/config/EBSTune-patterns.yaml | 2 +- tests/config/EBSTune.yaml | 2 +- tests/config/sr_serialized_magnets.yaml | 2 +- tests/configuration/test_fileloader.py | 87 ++++++++-- 6 files changed, 256 insertions(+), 50 deletions(-) diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index ab48372c5..9a952dade 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -1,10 +1,12 @@ """PyAML configuration file loader.""" -import collections.abc import io import json import logging +import os +import re from abc import ABC, abstractmethod +from collections.abc import Callable, Hashable from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path @@ -19,11 +21,13 @@ logger = logging.getLogger(__name__) -ACCEPTED_SUFFIXES = (".yaml", ".yml", ".json") -FILE_PREFIX = "file:" LOCATION_KEY = "__location__" FIELD_LOCATIONS_KEY = "__fieldlocations__" +ACCEPTED_SUFFIXES = (".yaml", ".yml", ".json") +RESOLVER_PATTERN = re.compile(r"\$\{([^{}]+)\}") + +FILE_PREFIX = "file:" # Kept for compatibility reasons with other modules class RootFolder: @@ -110,6 +114,64 @@ def loading(self, path: Path): self.include_stack.pop() +Resolver = Callable[[str, LoadContext | None], Any] +RESOLVERS: dict[str, Resolver] = {} + + +def resolver(name: str): + """Register a function as a configuration value resolver. + + Args: + name: Prefix used to invoke the resolver (for example ``"env"`` + or ``"file"``). + + Returns: + A decorator that registers the decorated function in the global + resolver registry. + """ + + def decorate(func: Resolver) -> Resolver: + RESOLVERS[name] = func + return func + + return decorate + + +@resolver("env") +def resolve_env(value: str, _context: LoadContext | None = None) -> str: + """Resolve an environment variable. + + Args: + value: Name of the environment variable. + context: Unused loading context. Present to match the resolver + interface. + + Raises: + PyAMLException: If the environment variable is not set. + """ + try: + return os.environ[value] + except KeyError as exc: + raise PyAMLException(f"Environment variable '{value}' is not set") from exc + + +@resolver("file") +def resolve_file(value: str, context: LoadContext) -> Any: + """Load and return the contents of a configuration file. + + Args: + value: Path to the configuration file. + context: Shared loading context used to track recursive includes + and detect inclusion cycles. + + Raises: + RuntimeError: If no loading context is provided. + """ + if context is None: + raise RuntimeError("File resolver requires LoadContext") + return _load(value, context) + + def load(filename: str, include_locations: bool = False) -> Union[dict, list]: """Load a configuration file. @@ -154,40 +216,103 @@ def __init__(self, path: Path, context: LoadContext): self.context = context def expand(self, obj: Union[dict, list, Any]) -> Union[dict, list, Any]: - """Recursively expand nested dictionaries and lists.""" + """Recursively expand configuration values. + + Dictionaries and lists are traversed recursively, while string values + are resolved using the registered resolvers. All other values are + returned unchanged. + """ if isinstance(obj, dict): return self._expand_dict(obj) if isinstance(obj, list): return self._expand_list(obj) + if isinstance(obj, str): + return self._expand_string(obj) return obj - def _expand_dict(self, d: dict) -> dict: - """Expand any supported file references found inside a dictionary.""" + def _expand_string(self, value: str) -> Any: + """Expand resolver expressions and file references in a string. - for key, value in list(d.items()): - try: - if _is_supported_file(value): - # If it is a reference to another file - resolved = self._resolve_file_reference(value) - if resolved is not None: - d[key] = resolved - else: - d[key] = _load(value, self.context) - else: - d[key] = self.expand(value) - except PyAMLConfigCyclingException as exc: - self._raise_cycle_error(exc, d, key) + If the entire string is a resolver expression (for example + ``"${env:HOME}"`` or ``"${file:config.yaml}"``), the resolved value is + returned directly and may be of any type. + + Resolver expressions embedded inside a larger string are interpolated + into the surrounding text. Only scalar values may be interpolated; + attempting to embed a dictionary or list raises a ``PyAMLException``. + + After resolver expansion, plain strings that refer to supported + configuration files are loaded automatically. + + Args: + value: The string value to expand. + + Returns: + The expanded value, which may be a string or another object if the + input consists solely of a resolver expression. + """ + + full_match = RESOLVER_PATTERN.fullmatch(value) + if full_match: + return self._resolve_resolver_expression(full_match.group(1).strip()) + + # Handle embedded case + def replace(match: re.Match[str]) -> str: + resolved = self._resolve_resolver_expression(match.group(1).strip()) + + if isinstance(resolved, (dict, list)): + raise PyAMLException( + f"Resolver '{match.group(1)}' returned a {type(resolved).__name__}, " + "which cannot be interpolated into a string." + ) + + return str(resolved) + + value = RESOLVER_PATTERN.sub(replace, value) + + if _is_supported_file(value): + return RESOLVERS["file"](value, self.context) + + return value + + def _resolve_resolver_expression(self, expr: str) -> Any: + """Resolve a single resolver expression. + + The expression must have the form ``":"``, for + example ``"env:HOME"`` or ``"file:config.yaml"``. The resolver is + looked up in the global resolver registry and invoked with the + supplied payload. + + Args: + expr: Resolver expression without the surrounding ``"${...}"``. - return d + Returns: + The value returned by the matching resolver. - def _resolve_file_reference(self, value: str) -> str | None: - """Return an absolute path for FILE_PREFIX references, otherwise None.""" + Raises: + PyAMLException: If the expression is malformed or references an + unknown resolver. + """ + if ":" not in expr: + raise PyAMLException(f"Invalid resolver expression '{expr}'") + + prefix, payload = expr.split(":", 1) + resolver = RESOLVERS.get(prefix.strip()) + if resolver is None: + raise PyAMLException(f"Unknown resolver '{prefix.strip()}'") + + return resolver(payload.strip(), self.context) - if value.startswith(FILE_PREFIX): - stripped_value = value[len(FILE_PREFIX) :] - return str(ROOT.get() / Path(stripped_value)) - return None + def _expand_dict(self, data: dict) -> dict: + """Recursively expand dictionary values and enrich cycle errors with location information.""" + + for key, value in list(data.items()): + try: + data[key] = self.expand(value) + except PyAMLConfigCyclingException as exc: + self._raise_cycle_error(exc, data, key) + return data def _raise_cycle_error(self, exc: PyAMLConfigCyclingException, obj: dict, key: Any) -> None: """Re-raise a cycle error with file location information, if available.""" @@ -205,18 +330,34 @@ def _raise_cycle_error(self, exc: PyAMLConfigCyclingException, obj: dict, key: A raise PyAMLException(f"Circular file inclusion of {exc.error_filename}{location_str}") from exc def _expand_list(self, items: list) -> list: - """Expand supported file references inside a list.""" + """Recursively expand the elements of a list. + + Plain string values that refer to supported configuration files are + treated as list includes. If the referenced file loads to a list, its + elements are spliced into the current list. Otherwise, the loaded + object is appended as a single element. - expanded = [] - for value in items: - if _is_supported_file(value): - loaded = _load(value, self.context) + All other items are expanded recursively using :meth:`expand`. + + Args: + items: The list to expand. + + Returns: + The expanded list. + """ + expanded: list[Any] = [] + + for item in items: + if isinstance(item, str) and _is_supported_file(item): + loaded = RESOLVERS["file"](item, self.context) if isinstance(loaded, list): expanded.extend(loaded) else: expanded.append(loaded) - else: - expanded.append(self.expand(value)) + continue + + expanded.append(self.expand(item)) + return expanded @abstractmethod @@ -281,7 +422,7 @@ def construct_mapping(self, node, deep=False): for key_node, value_node in node.value: key = self.construct_object(key_node, deep=deep) - if not isinstance(key, collections.abc.Hashable): + if not isinstance(key, Hashable): raise ConstructorError( "while constructing a mapping", node.start_mark, diff --git a/tests/config/EBSOrbit.yaml b/tests/config/EBSOrbit.yaml index a3c66132e..82ea191db 100644 --- a/tests/config/EBSOrbit.yaml +++ b/tests/config/EBSOrbit.yaml @@ -55,7 +55,7 @@ devices: name: DEFAULT_CHROMATICITY_CORRECTION chromaticty_monitor_name: CHROMATICITY_MONITOR sextu_array_name: Sext - response_matrix: file:ideal_chroma_resp.json + response_matrix: ${file:ideal_chroma_resp.json} - type: pyaml.tuning_tools.orbit bpm_array_name: BPM hcorr_array_name: HCorr @@ -63,7 +63,7 @@ devices: rf_plant_name: RF name: DEFAULT_ORBIT_CORRECTION singular_values: 162 - response_matrix: file:ideal_orm_disp.json + response_matrix: ${file:ideal_orm_disp.json} - type: pyaml.tuning_tools.orbit_response_matrix bpm_array_name: BPM hcorr_array_name: HCorr diff --git a/tests/config/EBSTune-patterns.yaml b/tests/config/EBSTune-patterns.yaml index abab79d85..86463b0b2 100644 --- a/tests/config/EBSTune-patterns.yaml +++ b/tests/config/EBSTune-patterns.yaml @@ -36,7 +36,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: file:tune_response.json + response_matrix: ${file:tune_response.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune diff --git a/tests/config/EBSTune.yaml b/tests/config/EBSTune.yaml index 373e83c31..70fdf1efa 100644 --- a/tests/config/EBSTune.yaml +++ b/tests/config/EBSTune.yaml @@ -29,7 +29,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: file:tune_response.json + response_matrix: ${file:tune_response.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune diff --git a/tests/config/sr_serialized_magnets.yaml b/tests/config/sr_serialized_magnets.yaml index e3609e8ab..9ce5926ff 100644 --- a/tests/config/sr_serialized_magnets.yaml +++ b/tests/config/sr_serialized_magnets.yaml @@ -225,7 +225,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: file:tune_response.json + response_matrix: ${file:tune_response.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune diff --git a/tests/configuration/test_fileloader.py b/tests/configuration/test_fileloader.py index b76273fc7..71d56031a 100644 --- a/tests/configuration/test_fileloader.py +++ b/tests/configuration/test_fileloader.py @@ -5,7 +5,6 @@ from pyaml import PyAMLException from pyaml.configuration.fileloader import ( FIELD_LOCATIONS_KEY, - FILE_PREFIX, LOCATION_KEY, ROOT, LoadContext, @@ -133,6 +132,82 @@ def test_load_nested_json(tmp_path): assert result["child"]["answer"] == 42 +def test_load_file_resolver_loads_nested_file(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "subdir").mkdir() + (tmp_path / "subdir" / "child.yaml").write_text("answer: 42\n") + (tmp_path / "parent.yaml").write_text('target: "${file:subdir/child.yaml}"\n') + + result = load("parent.yaml") + + assert result["target"]["answer"] == 42 + + +def test_load_env_resolver_resolves_environment_variable(tmp_path, monkeypatch): + ROOT.set(tmp_path) + + monkeypatch.setenv("TANGO_HOST", "localhost:10000") + (tmp_path / "config.yaml").write_text("host: ${env:TANGO_HOST}\n") + + result = load("config.yaml") + + assert result["host"] == "localhost:10000" + + +def test_load_env_resolver_missing_variable_raises(tmp_path, monkeypatch): + ROOT.set(tmp_path) + + monkeypatch.delenv("TANGO_HOST", raising=False) + (tmp_path / "config.yaml").write_text("host: ${env:TANGO_HOST}\n") + + with pytest.raises(PyAMLException, match="Environment variable 'TANGO_HOST' is not set"): + load("config.yaml") + + +def test_interpolates_env_inside_string(tmp_path, monkeypatch): + ROOT.set(tmp_path) + monkeypatch.setenv("HOST", "localhost") + monkeypatch.setenv("PORT", "5432") + + (tmp_path / "config.yaml").write_text("url: http://${env:HOST}:${env:PORT}\n") + + result = load("config.yaml") + + assert result["url"] == "http://localhost:5432" + + +def test_load_interpolates_env_multiple_times_in_one_string(tmp_path, monkeypatch): + ROOT.set(tmp_path) + + monkeypatch.setenv("USER", "alice") + + (tmp_path / "config.yaml").write_text("message: hello ${env:USER}, ${env:USER}!\n") + + result = load("config.yaml") + + assert result["message"] == "hello alice, alice!" + + +def test_load_interpolated_file_resolver_inside_string_raises(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "child.yaml").write_text("answer: 42\n") + (tmp_path / "config.yaml").write_text('value: "prefix-${file:child.yaml}-suffix"\n') + + with pytest.raises(PyAMLException, match="cannot be interpolated into a string"): + load("config.yaml") + + +def test_load_interpolation_unknown_resolver_raises(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "config.yaml").write_text("value: prefix-${missing:VALUE}-suffix\n") + + with pytest.raises(PyAMLException, match="Unknown resolver 'missing'"): + load("config.yaml") + + def test_load_list_include_extends_list(tmp_path): ROOT.set(tmp_path) @@ -157,16 +232,6 @@ def test_load_list_include_extends_list(tmp_path): ] -def test_load_file_prefix_returns_absolute_path(tmp_path): - ROOT.set(tmp_path) - - (tmp_path / "config.yaml").write_text(f'target: "{FILE_PREFIX}subdir/file.yaml"\n') - - result = load("config.yaml") - - assert result["target"] == str((tmp_path / "subdir" / "file.yaml").resolve()) - - def test_load_yaml_omits_locations_by_default(tmp_path): ROOT.set(tmp_path) From 0e745e4d57d0225d887965c294d3e0e244d82c24 Mon Sep 17 00:00:00 2001 From: Teresia Olsson Date: Mon, 6 Jul 2026 11:46:25 +0200 Subject: [PATCH 11/14] Add resolver for path. --- pyaml/configuration/fileloader.py | 17 +++++++++++++++++ tests/config/EBSOrbit.yaml | 4 ++-- tests/config/EBSTune-patterns.yaml | 2 +- tests/config/EBSTune.yaml | 2 +- tests/config/sr_serialized_magnets.yaml | 2 +- tests/configuration/test_fileloader.py | 10 ++++++++++ 6 files changed, 32 insertions(+), 5 deletions(-) diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index 9a952dade..317fb29cc 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -155,6 +155,23 @@ def resolve_env(value: str, _context: LoadContext | None = None) -> str: raise PyAMLException(f"Environment variable '{value}' is not set") from exc +@resolver("path") +def resolve_path(value: str, _context: LoadContext | None = None) -> str: + """Resolve a configuration path without loading the file. + + Relative paths are expanded using the configured root folder. + + Args: + value: Path to resolve. + context: Unused loading context. Present to match the resolver + interface. + + Returns: + The absolute, normalized path as a string. + """ + return str(ROOT.expand_path(value)) + + @resolver("file") def resolve_file(value: str, context: LoadContext) -> Any: """Load and return the contents of a configuration file. diff --git a/tests/config/EBSOrbit.yaml b/tests/config/EBSOrbit.yaml index 82ea191db..9b66318ad 100644 --- a/tests/config/EBSOrbit.yaml +++ b/tests/config/EBSOrbit.yaml @@ -55,7 +55,7 @@ devices: name: DEFAULT_CHROMATICITY_CORRECTION chromaticty_monitor_name: CHROMATICITY_MONITOR sextu_array_name: Sext - response_matrix: ${file:ideal_chroma_resp.json} + response_matrix: ${path:ideal_chroma_resp.json} - type: pyaml.tuning_tools.orbit bpm_array_name: BPM hcorr_array_name: HCorr @@ -63,7 +63,7 @@ devices: rf_plant_name: RF name: DEFAULT_ORBIT_CORRECTION singular_values: 162 - response_matrix: ${file:ideal_orm_disp.json} + response_matrix: ${path:ideal_orm_disp.json} - type: pyaml.tuning_tools.orbit_response_matrix bpm_array_name: BPM hcorr_array_name: HCorr diff --git a/tests/config/EBSTune-patterns.yaml b/tests/config/EBSTune-patterns.yaml index 86463b0b2..0c258bdea 100644 --- a/tests/config/EBSTune-patterns.yaml +++ b/tests/config/EBSTune-patterns.yaml @@ -36,7 +36,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: ${file:tune_response.json} + response_matrix: ${path:tune_response.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune diff --git a/tests/config/EBSTune.yaml b/tests/config/EBSTune.yaml index 70fdf1efa..f5a8d5c08 100644 --- a/tests/config/EBSTune.yaml +++ b/tests/config/EBSTune.yaml @@ -29,7 +29,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: ${file:tune_response.json} + response_matrix: ${path:tune_response.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune diff --git a/tests/config/sr_serialized_magnets.yaml b/tests/config/sr_serialized_magnets.yaml index 9ce5926ff..a8a3c486d 100644 --- a/tests/config/sr_serialized_magnets.yaml +++ b/tests/config/sr_serialized_magnets.yaml @@ -225,7 +225,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: ${file:tune_response.json} + response_matrix: ${path:tune_response.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune diff --git a/tests/configuration/test_fileloader.py b/tests/configuration/test_fileloader.py index 71d56031a..53b234a66 100644 --- a/tests/configuration/test_fileloader.py +++ b/tests/configuration/test_fileloader.py @@ -189,6 +189,16 @@ def test_load_interpolates_env_multiple_times_in_one_string(tmp_path, monkeypatc assert result["message"] == "hello alice, alice!" +def test_load_path_resolver_resolves_without_loading_file(tmp_path): + ROOT.set(tmp_path) + + (tmp_path / "config.yaml").write_text("target: ${path:subdir/missing.json}\n") + + result = load("config.yaml") + + assert result["target"] == str((tmp_path / "subdir" / "missing.json").resolve()) + + def test_load_interpolated_file_resolver_inside_string_raises(tmp_path): ROOT.set(tmp_path) From 6b5b3e5d82c4d32969b1597312290d20fb9b99a1 Mon Sep 17 00:00:00 2001 From: Teresia Olsson Date: Mon, 6 Jul 2026 11:51:12 +0200 Subject: [PATCH 12/14] Updated BESSY II examples. --- examples/BESSY2_example/BESSY2Orbit.yaml | 2 +- examples/BESSY2_example/BESSY2Tune.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/BESSY2_example/BESSY2Orbit.yaml b/examples/BESSY2_example/BESSY2Orbit.yaml index e4e5947ec..4d9fabed5 100644 --- a/examples/BESSY2_example/BESSY2Orbit.yaml +++ b/examples/BESSY2_example/BESSY2Orbit.yaml @@ -277,7 +277,7 @@ devices: vcorr_array_name: VCorr name: DEFAULT_ORBIT_CORRECTION singular_values: 16 - response_matrix: file:orm.json + response_matrix: ${path:orm.json} - type: pyaml.rf.rf_plant name: RF masterclock: (MCLKHX251C:freq)[KHz] diff --git a/examples/BESSY2_example/BESSY2Tune.yaml b/examples/BESSY2_example/BESSY2Tune.yaml index 23a5bfd4c..754ab9cd8 100644 --- a/examples/BESSY2_example/BESSY2Tune.yaml +++ b/examples/BESSY2_example/BESSY2Tune.yaml @@ -668,7 +668,7 @@ devices: name: DEFAULT_TUNE_CORRECTION quad_array_name: QForTune betatron_tune_name: BETATRON_TUNE - response_matrix: file:trm.json + response_matrix: ${path:trm.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QForTune From caae6d5bf5dc037278d3a95563676fcad361c3db Mon Sep 17 00:00:00 2001 From: Teresia Olsson Date: Mon, 6 Jul 2026 11:53:09 +0200 Subject: [PATCH 13/14] Update SOLEIL examples. --- examples/SOLEIL_examples/tuning_tools.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/SOLEIL_examples/tuning_tools.yaml b/examples/SOLEIL_examples/tuning_tools.yaml index 4b5eefbdb..3d531b0ba 100644 --- a/examples/SOLEIL_examples/tuning_tools.yaml +++ b/examples/SOLEIL_examples/tuning_tools.yaml @@ -6,7 +6,7 @@ name: DEFAULT_TUNE_CORRECTION quad_array_name: QCORR betatron_tune_name: BETATRON_TUNE - response_matrix: file:trm.json + response_matrix: ${path:trm.json} - type: pyaml.tuning_tools.tune_response_matrix name: DEFAULT_TUNE_RESPONSE_MATRIX quad_array_name: QCORR @@ -40,4 +40,4 @@ vcorr_array_name: VCORR name: DEFAULT_ORBIT_CORRECTION singular_values: 16 - response_matrix: file:orm.json + response_matrix: ${path:file:orm.json} From c55958dfd93d369d402db17178027a19ad6fad1a Mon Sep 17 00:00:00 2001 From: Teresia Olsson Date: Mon, 6 Jul 2026 12:00:14 +0200 Subject: [PATCH 14/14] Remove FILE_PREFIX from fileloader since not used anymore. --- pyaml/configuration/fileloader.py | 2 -- pyaml/configuration/restfetcher.py | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyaml/configuration/fileloader.py b/pyaml/configuration/fileloader.py index 317fb29cc..65f66a105 100644 --- a/pyaml/configuration/fileloader.py +++ b/pyaml/configuration/fileloader.py @@ -27,8 +27,6 @@ ACCEPTED_SUFFIXES = (".yaml", ".yml", ".json") RESOLVER_PATTERN = re.compile(r"\$\{([^{}]+)\}") -FILE_PREFIX = "file:" # Kept for compatibility reasons with other modules - class RootFolder: """Manage the root directory used to resolve configuration paths.""" diff --git a/pyaml/configuration/restfetcher.py b/pyaml/configuration/restfetcher.py index 380da6176..78d4aca7e 100644 --- a/pyaml/configuration/restfetcher.py +++ b/pyaml/configuration/restfetcher.py @@ -15,12 +15,14 @@ from yaml import CLoader from ..common.exception import PyAMLConfigException -from .fileloader import ACCEPTED_SUFFIXES, FILE_PREFIX, SafeLineLoader +from .fileloader import ACCEPTED_SUFFIXES, SafeLineLoader REMOTE_BASE_URL_KEY = "__baseurl__" SourceRoot = Path | str | None _REMOTE_SCHEMES = {"http", "https"} +FILE_PREFIX = "${path:" + class _NamedStringIO(io.StringIO): def __init__(self, value: str, name: str):