Add include resolver and remove automatic inclusion of files with specific file extensions.

examples/use_cases/config/esrf/catalogs/ebs_catalogs.yaml

@@ -1,8 +1,8 @@
 type: tango.pyaml.static_catalog
 entries:
-  - catalogs/ebs_bpm_catalogs.yaml
-  - catalogs/ebs_rf_catalogs.yaml
-  - catalogs/ebs_diag_catalogs.yaml
-  - catalogs/ebs_magnets_catalogs.yaml
-  - catalogs/ebs_sh_catalogs.yaml
-  - catalogs/ebs_sext_catalogs.yaml
+  - ${include:catalogs/ebs_bpm_catalogs.yaml}
+  - ${include:catalogs/ebs_rf_catalogs.yaml}
+  - ${include:catalogs/ebs_diag_catalogs.yaml}
+  - ${include:catalogs/ebs_magnets_catalogs.yaml}
+  - ${include:catalogs/ebs_sh_catalogs.yaml}
+  - ${include:catalogs/ebs_sext_catalogs.yaml}

examples/use_cases/config/esrf/esrf.yaml

@@ -11,7 +11,7 @@ controls:
   - type: tango.pyaml.controlsystem
     tango_host: ebs-simu-3:10000
     name: live
-    catalog: catalogs/ebs_catalogs.yaml
+    catalog: ${include:catalogs/ebs_catalogs.yaml}
 data_folder: /data/store
 arrays:
   - type: pyaml.arrays.magnet

examples/use_cases/config/soleil_ii/p.yaml

@@ -14,8 +14,8 @@ controls:
 - type: tango.pyaml.controlsystem
   name: live
   tango_host: localhost:11000
-  catalog: catalogs.yaml
-arrays: arrays.yaml
+  catalog: ${include:catalogs.yaml}
+arrays: ${include:arrays.yaml}
 devices:
-- devices.yaml
-- tuning_tools.yaml
+- ${include:devices.yaml}
+- ${include:tuning_tools.yaml}

pyaml/configuration/fileloader.py

@@ -29,7 +29,6 @@
 
 LOCATION_KEY = "__location__"
 FIELD_LOCATIONS_KEY = "__fieldlocations__"
-ACCEPTED_SUFFIXES = (".yaml", ".yml", ".json")
 RESOLVER_PATTERN = re.compile(r"\$\{([^{}]+)\}")
 
 
@@ -189,7 +188,7 @@ def resolver(name: str):
 
     Args:
         name: Prefix used to invoke the resolver (for example ``"env"``
-            or ``"file"``).
+            or ``"include"``).
 
     Returns:
         A decorator that registers the decorated function in the global
@@ -266,10 +265,10 @@ def resolve_path(value: str, _context: LoadContext | None = None) -> str:
     return str(ROOT.expand_path(value))
 
 
-@resolver("file")
-def resolve_file(value: str, context: LoadContext | None = None) -> Any:
+@resolver("include")
+def resolve_include(value: str, context: LoadContext | None = None) -> Any:
     """
-    Load and return the contents of a configuration file.
+    Load and include the contents of a configuration file.
 
     Parameters
     ----------
@@ -289,7 +288,7 @@ def resolve_file(value: str, context: LoadContext | None = None) -> Any:
         If no loading context is provided.
     """
     if context is None:
-        raise RuntimeError("File resolver requires LoadContext")
+        raise RuntimeError("Include resolver requires LoadContext")
     return _load(value, context)
 
 
@@ -323,11 +322,6 @@ def _load(filename: str, context: LoadContext) -> Union[dict, list]:
         return loader.load()
 
 
-def _is_supported_file(value: Any) -> bool:
-    """Return True if the value looks like a supported configuration file name."""
-    return isinstance(value, str) and value.endswith(ACCEPTED_SUFFIXES)
-
-
 class ConfigLoader(ABC):
     """
     Base class for parsers that expand nested configuration references.
@@ -375,7 +369,7 @@ def _expand_string(self, value: str) -> Any:
         Expand resolver expressions and file references in a string.
 
         If the entire string is a resolver expression (for example
-        ``"${env:HOME}"`` or ``"${file:config.yaml}"``), the resolved value is
+        ``"${env:HOME}"`` or ``"${include:config.yaml}"``), the resolved value is
         returned directly and may be of any type.
 
         Resolver expressions embedded inside a larger string are interpolated
@@ -424,17 +418,14 @@ def replace(match: re.Match[str]) -> str:
 
         value = RESOLVER_PATTERN.sub(replace, value)
 
-        if _is_supported_file(value):
-            return RESOLVERS["file"](value, self.context)
-
         return value
 
     def _resolve_resolver_expression(self, expr: str) -> Any:
         """
         Resolve a single resolver expression.
 
         The expression must have the form ``"<resolver>:<payload>"``, for
-        example ``"env:HOME"`` or ``"file:config.yaml"``. The resolver is
+        example ``"env:HOME"`` or ``"include:config.yaml"``. The resolver is
         looked up in the global resolver registry and invoked with the
         supplied payload.
 
@@ -487,31 +478,24 @@ def _expand_list(self, items: list) -> list:
         """
         Recursively expand the elements of a list.
 
-        Plain string values that refer to supported configuration files are
-        treated as list includes. If the referenced file loads to a list, its
-        elements are spliced into the current list. Otherwise, the loaded
-        object is appended as a single element.
-
-        All other items are expanded recursively using :meth:`expand`.
-
-        Args:
-            items: The list to expand.
-
-        Returns:
-            The expanded list.
+        Lists returned by an explicit include resolver are spliced into the
+        current list. Other nested lists are preserved.
         """
         expanded: list[Any] = []
 
         for item in items:
-            if isinstance(item, str) and _is_supported_file(item):
-                loaded = RESOLVERS["file"](item, self.context)
-                if isinstance(loaded, list):
-                    expanded.extend(loaded)
-                else:
-                    expanded.append(loaded)
-                continue
-
-            expanded.append(self.expand(item))
+            is_include = (
+                isinstance(item, str)
+                and (match := RESOLVER_PATTERN.fullmatch(item)) is not None
+                and match.group(1).strip().startswith("include:")
+            )
+
+            value = self.expand(item)
+
+            if is_include and isinstance(value, list):
+                expanded.extend(value)
+            else:
+                expanded.append(value)
 
         return expanded
 

pyaml/configuration/restfetcher.py

@@ -18,12 +18,12 @@
 from yaml import CLoader
 
 from ..common.exception import PyAMLConfigException
-from .fileloader import ACCEPTED_SUFFIXES, SafeLineLoader
+from .fileloader import SafeLineLoader
 
 REMOTE_BASE_URL_KEY = "__baseurl__"
 SourceRoot = Path | str | None
 _REMOTE_SCHEMES = {"http", "https"}
-
+INCLUDE_PREFIX = "${include:"
 FILE_PREFIX = "${path:"
 
 
@@ -285,9 +285,10 @@ def _expand_remote_dict(
     """
     values.setdefault(REMOTE_BASE_URL_KEY, base_url)
     for key, value in list(values.items()):
-        if _is_config_reference(value):
+        if _is_include(value):
+            reference = _include_reference(value)
             values[key] = _load_remote_document(
-                _resolve_remote_config_reference(value, base_url),
+                _resolve_remote_config_reference(reference, base_url),
                 include_locations=include_locations,
                 stack=stack,
             )
@@ -324,9 +325,10 @@ def _expand_remote_list(values: list[Any], base_url: str, stack: list[str], *, i
     index = 0
     while index < len(values):
         value = values[index]
-        if _is_config_reference(value):
+        if _is_include(value):
+            reference = _include_reference(value)
             expanded = _load_remote_document(
-                _resolve_remote_config_reference(value, base_url),
+                _resolve_remote_config_reference(reference, base_url),
                 include_locations=include_locations,
                 stack=stack,
             )
@@ -348,29 +350,14 @@ def _expand_remote_list(values: list[Any], base_url: str, stack: list[str], *, i
     return values
 
 
-def _is_config_reference(value: Any) -> bool:
-    """
-    Return whether a value names a supported configuration document.
-
-    Parameters
-    ----------
-    value : Any
-        Value to inspect.
-
-    Returns
-    -------
-    bool
-        ``True`` for a YAML, JSON, or supported remote document reference.
-    """
-    if not isinstance(value, str):
-        return False
+def _is_include(value: Any) -> bool:
+    """Return whether a value is an explicit include expression."""
+    return isinstance(value, str) and value.startswith(INCLUDE_PREFIX) and value.endswith("}")
 
-    if value.startswith(FILE_PREFIX):
-        return False
 
-    parsed = urlparse(value)
-    path = parsed.path if parsed.scheme in _REMOTE_SCHEMES else value
-    return any(path.endswith(suffix) for suffix in ACCEPTED_SUFFIXES)
+def _include_reference(value: str) -> str:
+    """Return the reference contained in an include expression."""
+    return value[len(INCLUDE_PREFIX) : -1].strip()
 
 
 def _resolve_remote_config_reference(reference: str, base_url: str) -> str:

tests/common/test_errors.py

@@ -60,7 +60,7 @@ def test_duplicate_error_reports_source_line_and_column_across_files(tmp_path):
     )
     root = tmp_path / "root.yaml"
     root.write_text(
-        "type: pyaml.accelerator\ndevices:\n  - devices_a.yaml\n  - devices_b.yaml\n",
+        "type: pyaml.accelerator\ndevices:\n  - ${include:devices_a.yaml}\n  - ${include:devices_b.yaml}\n",
         encoding="utf-8",
     )
 
@@ -86,7 +86,7 @@ def test_malformed_local_included_yaml_reports_source_line_and_column(tmp_path):
         "machine: sr\n"
         "energy: 6e9\n"
         "data_folder: /data/store\n"
-        "devices: broken_devices.yaml\n",
+        "devices: ${include:broken_devices.yaml}\n",
         encoding="utf-8",
     )
 
@@ -111,7 +111,7 @@ def test_truncated_local_included_json_reports_source_and_position(tmp_path):
         "machine: sr\n"
         "energy: 6e9\n"
         "data_folder: /data/store\n"
-        "devices: broken_devices.json\n",
+        "devices: ${include:broken_devices.json}\n",
         encoding="utf-8",
     )
 
@@ -131,7 +131,7 @@ def test_malformed_remote_included_yaml_reports_source_line_and_column(http_conf
             "machine: sr\n"
             "energy: 6000000000.0\n"
             "data_folder: /data/store\n"
-            "devices: fragments/broken_devices.yaml\n"
+            "devices: ${include:fragments/broken_devices.yaml}\n"
         ),
         "/configs/fragments/broken_devices.yaml": ("- type: pyaml.bpm.bpm\n  name: BPM_BROKEN\n  model:\n    type: [oops\n"),
     }
@@ -153,7 +153,7 @@ def test_truncated_remote_included_json_reports_source_and_position(http_config_
             "machine: sr\n"
             "energy: 6000000000.0\n"
             "data_folder: /data/store\n"
-            "devices: fragments/broken_devices.json\n"
+            "devices: ${include:fragments/broken_devices.json}\n"
         ),
         "/configs/fragments/broken_devices.json": (
             '{"type": "pyaml.bpm.bpm", "name": "BPM_BROKEN", "model": ',

tests/config/EBSNames-linker.yaml

@@ -13,7 +13,7 @@ controls:
   - type: tango.pyaml.controlsystem
     tango_host: ebs-simu-3:10000
     name: live
-    catalog: catalogs/ebs_catalogs.yaml
+    catalog: ${include:catalogs/ebs_catalogs.yaml}
 data_folder: /data/store
 devices:
 - type: pyaml.magnet.quadrupole

tests/config/EBSNames.yaml

@@ -10,7 +10,7 @@ controls:
   - type: tango.pyaml.controlsystem
     tango_host: ebs-simu-3:10000
     name: live
-    catalog: catalogs/ebs_catalogs.yaml
+    catalog: ${include:catalogs/ebs_catalogs.yaml}
 data_folder: /data/store
 devices:
 - type: pyaml.magnet.quadrupole

tests/config/EBSOrbit.yaml

@@ -11,7 +11,7 @@ controls:
   - type: tango.pyaml.controlsystem
     tango_host: ebs-simu-3:10000
     name: live
-    catalog: catalogs/ebs_catalogs.yaml
+    catalog: ${include:catalogs/ebs_catalogs.yaml}
 data_folder: /data/store
 arrays:
   - type: pyaml.arrays.magnet

tests/config/EBSTune-patterns.yaml

@@ -12,7 +12,7 @@ controls:
   - type: tango.pyaml.controlsystem
     tango_host: ebs-simu-3:10000
     name: live
-    catalog: catalogs/ebs_catalogs.yaml
+    catalog: ${include:catalogs/ebs_catalogs.yaml}
 data_folder: /data/store
 arrays:
   - type: pyaml.arrays.magnet

tests/config/EBSTune.yaml

@@ -12,7 +12,7 @@ controls:
   - type: tango.pyaml.controlsystem
     tango_host: ebs-simu-3:10000
     name: live
-    catalog: catalogs/ebs_catalogs.yaml
+    catalog: ${include:catalogs/ebs_catalogs.yaml}
 data_folder: /data/store
 arrays:
   - type: pyaml.arrays.magnet

tests/config/EBS_rf_multi.yaml

@@ -10,7 +10,7 @@ controls:
   - type: tango.pyaml.controlsystem
     tango_host: ebs-simu-3:10000
     name: live
-    catalog: catalogs/ebs_catalogs.yaml
+    catalog: ${include:catalogs/ebs_catalogs.yaml}
 data_folder: /data/store
 devices:
   - type: pyaml.rf.rf_plant

tests/config/EBS_rf_notrans.yaml

@@ -10,7 +10,7 @@ controls:
   - type: tango.pyaml.controlsystem
     tango_host: ebs-simu-3:10000
     name: live
-    catalog: catalogs/ebs_catalogs.yaml
+    catalog: ${include:catalogs/ebs_catalogs.yaml}
 data_folder: /data/store
 devices:
   - type: pyaml.rf.rf_plant

tests/config/bad_conf.yml

@@ -19,6 +19,6 @@ arrays:
       - SH1A-C01-V
       - SH1A-C02-V
 devices:
-  - sr/quadrupoles/QF1AC01.yaml
-  - sr/correctors/SH1AC01.yaml
-  - sr/correctors/SH1AC02.yaml
+  - ${include:sr/quadrupoles/QF1AC01.yaml}
+  - ${include:sr/correctors/SH1AC01.yaml}
+  - ${include:sr/correctors/SH1AC02.yaml}

tests/config/bad_conf_cycles.json

@@ -3,7 +3,7 @@
     "facility": "ESRF",
     "machine": "sr",
     "energy": 6e9,
-    "simulators": ["../config/bad_conf_cycles.json"],
+    "simulators": ["${include:../config/bad_conf_cycles.json}"],
     "data_folder": "/data/store",
     "arrays": [
       {
@@ -21,8 +21,8 @@
       }
     ],
     "devices": [
-      "sr/quadrupoles/QF1AC01.yaml",
-      "sr/correctors/SH1AC01.yaml",
-      "sr/correctors/SH1AC02.yaml"
+      "${include:sr/quadrupoles/QF1AC01.yaml}",
+      "${include:sr/correctors/SH1AC01.yaml}",
+      "${include:sr/correctors/SH1AC02.yaml}"
     ]
 }

tests/config/bad_conf_cycles.yml

@@ -19,7 +19,7 @@ arrays:
       - SH1A-C01-V
       - SH1A-C02-V
 devices:
-  - ../config/bad_conf_cycles.yml # Cycle here
-  - sr/quadrupoles/QF1AC01.yaml
-  - sr/correctors/SH1AC01.yaml
-  - sr/correctors/SH1AC02.yaml
+  - ${include:../config/bad_conf_cycles.yml} # Cycle here
+  - ${include:sr/quadrupoles/QF1AC01.yaml}
+  - ${include:sr/correctors/SH1AC01.yaml}
+  - ${include:sr/correctors/SH1AC02.yaml}