Merge pull request #1462 from lytedev/fix/include-relative-path-resolution

Fix/include relative path resolution

Author: p12tic

newsfragments/resolve-include-relative-paths.bugfix

@@ -0,0 +1 @@
+Resolve relative paths in included compose files against the included file's directory rather than the project root, matching the Compose Spec. Affects volumes, env_file, and build.context inside files referenced by include:. Fixes #1301.

podman_compose.py

@@ -2150,16 +2150,33 @@ def normalize_service(service: dict[str, Any], sub_dir: str = "") -> dict[str, A
 
             new_volumes.append(v)
         service["volumes"] = new_volumes
+    if "env_file" in service and sub_dir:
+        new_env_file = []
+        for ef in service["env_file"]:
+            if isinstance(ef, str):
+                if is_relative_ref(ef):
+                    ef = os.path.join(sub_dir, ef)
+            elif isinstance(ef, dict):
+                path = ef.get("path")
+                if isinstance(path, str) and is_relative_ref(path):
+                    ef["path"] = os.path.join(sub_dir, path)
+            new_env_file.append(ef)
+        service["env_file"] = new_env_file
     return service
 
 
-def normalize(compose: dict[str, Any]) -> dict[str, Any]:
+def normalize(compose: dict[str, Any], sub_dir: str = "") -> dict[str, Any]:
     """
     convert compose dict of some keys from string or dicts into arrays
+
+    If ``sub_dir`` is provided, relative paths in ``volumes``, ``env_file`` and
+    ``build.context`` are rewritten to be relative to ``sub_dir`` (used when an
+    included file lives in a different directory than the project root, per
+    Compose Spec resolution of paths in ``include:``d files).
     """
     services = compose.get("services", {}) or {}
     for service in services.values():
-        normalize_service(service)
+        normalize_service(service, sub_dir)
     return compose
 
 
@@ -2721,6 +2738,11 @@ def _parse_compose_file(self) -> None:
         compose: dict[str, Any] = {}
         # Iterate over files primitively to allow appending to files in-loop
         files_iter = iter(files)
+        # Track files appended by ``include:`` so we can resolve their
+        # relative paths against the included file's directory per the
+        # Compose Spec, without changing the legacy merge behavior of
+        # files passed directly via ``-f``.
+        include_origin_files: set[str] = set()
 
         while True:
             try:
@@ -2737,7 +2759,22 @@ def _parse_compose_file(self) -> None:
             if not isinstance(content, dict):
                 log.fatal("Compose file does not contain a top level object: %s", filename)
                 sys.exit(1)
-            content = normalize(content)
+            # For files arriving via ``include:``, paths inside the file must
+            # resolve against the included file's directory rather than the
+            # project root (Compose Spec, ``include`` section). Pass that as
+            # sub_dir so volumes / env_file / build.context get rewritten.
+            file_sub_dir = ""
+            if filename in include_origin_files:
+                file_dir = os.path.dirname(os.path.abspath(filename))
+                file_sub_dir = os.path.relpath(file_dir, self.dirname)
+                if file_sub_dir == ".":
+                    file_sub_dir = ""
+                elif not file_sub_dir.startswith((".", "/")):
+                    # Prefix with "./" so rewritten paths remain recognizable
+                    # as relative refs (is_relative_ref checks for "./"/".."
+                    # prefixes).
+                    file_sub_dir = "./" + file_sub_dir
+            content = normalize(content, file_sub_dir)
             # log(filename, json.dumps(content, indent = 2))
 
             # See also https://docs.docker.com/compose/how-tos/project-name/#set-a-project-name
@@ -2780,23 +2817,28 @@ def _parse_compose_file(self) -> None:
                 if not isinstance(include, list):
                     raise RuntimeError("`include` must be a list")
 
+                new_includes: list[str] = []
                 for item in include:
                     if isinstance(item, str):
-                        files.append(os.path.join(os.path.dirname(filename), item))
+                        new_includes.append(os.path.join(os.path.dirname(filename), item))
                     elif isinstance(item, dict):
                         if "path" not in item:
                             raise RuntimeError("Missing required 'path' key in `include` block")
                         path = item["path"]
                         if isinstance(path, str):
-                            files.append(os.path.join(os.path.dirname(filename), path))
+                            new_includes.append(os.path.join(os.path.dirname(filename), path))
                         elif isinstance(path, list):
-                            files.extend([os.path.join(os.path.dirname(filename), p) for p in path])
+                            new_includes.extend(
+                                os.path.join(os.path.dirname(filename), p) for p in path
+                            )
                         else:
                             raise RuntimeError("'path' must be a string or a list of strings")
                     else:
                         raise RuntimeError(
                             "Items in `include` must be strings or dictionaries with a 'path' key"
                         )
+                files.extend(new_includes)
+                include_origin_files.update(new_includes)
                 # As compose obj is updated and tested with every loop, not deleting `include`
                 # from it, results in it being tested again and again, original values for
                 # `include` be appended to `files`, and, included files be processed for ever.

tests/integration/include_relative_paths/__init__.py

@@ -0,0 +1,6 @@
+version: "3.6"
+
+name: include-relative-paths
+
+include:
+  - sub/included.yaml

tests/integration/include_relative_paths/docker-compose.yaml

@@ -0,0 +1,15 @@
+services:
+  # All relative paths below should resolve against this file's directory
+  # (sub/), not the project root. So:
+  #   ./local.env       -> sub/local.env
+  #   ../shared.env     -> shared.env (in project root)
+  #   ./data:/data:ro   -> sub/data
+  #   ../assets:/assets:ro -> assets (in project root)
+  web:
+    image: nopush/podman-compose-test
+    env_file:
+      - ./local.env
+      - ../shared.env
+    volumes:
+      - ./data:/data:ro
+      - ../assets:/assets:ro

tests/integration/include_relative_paths/sub/included.yaml

@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: GPL-2.0
+
+"""
+Tests that relative paths (volumes, env_file) inside an included compose
+file are resolved against the included file's directory, not the project
+root. See https://github.com/containers/podman-compose/issues/1301 and the
+Compose Spec include resolution rules.
+"""
+
+import os
+import textwrap
+import unittest
+
+from tests.integration.test_utils import RunSubprocessMixin
+from tests.integration.test_utils import podman_compose_path
+from tests.integration.test_utils import test_path
+
+
+def compose_file() -> str:
+    return os.path.join(test_path(), "include_relative_paths", "docker-compose.yaml")
+
+
+class TestIncludeRelativePaths(unittest.TestCase, RunSubprocessMixin):
+    def test_relative_paths_in_included_file(self) -> None:
+        """
+        The included file at ``sub/included.yaml`` references
+        ``./local.env``, ``../shared.env``, ``./data:/data:ro`` and
+        ``../assets:/assets:ro``. After include resolution these must point
+        at paths under ``sub/`` for the ``./`` forms and at the project
+        root for the ``../`` forms.
+        """
+        out, _ = self.run_subprocess_assert_returncode([
+            "coverage",
+            "run",
+            podman_compose_path(),
+            "-f",
+            compose_file(),
+            "config",
+        ])
+
+        expected = textwrap.dedent("""\
+            name: include-relative-paths
+            services:
+              web:
+                env_file:
+                - ./sub/./local.env
+                - ./sub/../shared.env
+                image: nopush/podman-compose-test
+                volumes:
+                - ./sub/./data:/data:ro
+                - ./sub/../assets:/assets:ro
+            version: '3.6'
+
+            """)
+        self.assertEqual(out.decode("utf-8"), expected)

tests/integration/include_relative_paths/test_podman_compose_include_relative_paths.py

@@ -78,6 +78,18 @@ def test_simple(self, input: dict[str, Any], expected: dict[str, Any]) -> None:
                 ]
             },
         ),
+        (
+            {"env_file": "./.env"},
+            {"env_file": ["./sub_dir/./.env"]},
+        ),
+        (
+            {"env_file": ["./.env", "../shared.env"]},
+            {"env_file": ["./sub_dir/./.env", "./sub_dir/../shared.env"]},
+        ),
+        (
+            {"env_file": [{"path": "./.env", "required": False}]},
+            {"env_file": [{"path": "./sub_dir/./.env", "required": False}]},
+        ),
     ])
     def test_normalize_service_with_sub_dir(
         self, input: dict[str, Any], expected: dict[str, Any]