Wire exclude-newer policy into libmambapy Database

.github/workflows/tests.yml

@@ -44,7 +44,7 @@ env:
   # we can't break classic from here; no need to test it
   # those tests will still be available for local debugging if necessary
   CONDA_TEST_SOLVERS: libmamba
-  CONDA_REF: main # pin conda version to latest for more stable tests
+  CONDA_REF: feature/repodata-filter-cooldown # temporarily test against exclude-newer branch
 
 jobs:
   # detect whether any code changes are included in this PR
@@ -112,7 +112,7 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
-          repository: conda/conda  # CONDA-LIBMAMBA-SOLVER CHANGE
+          repository: jezdez/conda  # CONDA-LIBMAMBA-SOLVER CHANGE
           path: conda  # CONDA-LIBMAMBA-SOLVER CHANGE
           ref: ${{ env.CONDA_REF }}
 
@@ -278,7 +278,7 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
-          repository: conda/conda  # CONDA-LIBMAMBA-SOLVER CHANGE
+          repository: jezdez/conda  # CONDA-LIBMAMBA-SOLVER CHANGE
           path: conda  # CONDA-LIBMAMBA-SOLVER CHANGE
           ref: ${{ env.CONDA_REF }}
 
@@ -550,7 +550,7 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
-          repository: conda/conda  # CONDA-LIBMAMBA-SOLVER CHANGE
+          repository: jezdez/conda  # CONDA-LIBMAMBA-SOLVER CHANGE
           path: conda  # CONDA-LIBMAMBA-SOLVER CHANGE
           ref: ${{ env.CONDA_REF }}
 

conda_libmamba_solver/index.py

@@ -73,6 +73,7 @@
 
 from __future__ import annotations
 
+import json
 import logging
 import os
 from dataclasses import dataclass
@@ -85,6 +86,7 @@
 from conda.common.compat import on_win
 from conda.common.io import DummyExecutor, ThreadLimitedThreadPoolExecutor, time_recorder
 from conda.common.url import path_to_url, remove_auth, split_anaconda_token
+from conda.core.exclude_newer import ExcludeNewerPolicy
 from conda.core.package_cache_data import PackageCacheData
 from conda.core.subdir_data import SubdirData
 from conda.models.channel import Channel
@@ -253,6 +255,7 @@ def __init__(
         installed_records: Iterable[PackageRecord] = (),
         pkgs_dirs: PathsType = (),
         in_state: SolverInputState | None = None,
+        exclude_newer_policy: ExcludeNewerPolicy | None = None,
     ):
         platform_less_channels: list[Channel] = []
         for channel in channels:
@@ -272,6 +275,7 @@ def __init__(
         self.repodata_fn = repodata_fn
         self.in_state = in_state
         self._add_pip_as_python_dependency = context.add_pip_as_python_dependency
+        self.exclude_newer_policy = exclude_newer_policy or ExcludeNewerPolicy.disabled()
         self.db = self._init_db()
 
         self.repos: list[_ChannelRepoInfo] = self._load_channels()
@@ -362,10 +366,26 @@ def _init_db(self) -> Database:
             home_dir=str(Path.home()),
             current_working_dir=os.getcwd(),
         )
-        db = Database(params)
+        db_kwargs = {}
+        if (exclude_newer_timestamp := self._exclude_newer_timestamp()) is not None:
+            db_kwargs["exclude_newer_timestamp"] = exclude_newer_timestamp
+        db = Database(params, **db_kwargs)
         db.set_logger(logger_callback)
         return db
 
+    def _exclude_newer_timestamp(self) -> int | None:
+        """Return the resolved global cutoff as a Unix timestamp for libmambapy."""
+        if (
+            self.exclude_newer_policy.global_cutoff is None
+            or self.exclude_newer_policy.has_channel_overrides
+            or self.exclude_newer_policy.has_package_overrides
+        ):
+            return None
+        return int(self.exclude_newer_policy.global_cutoff)
+
+    def _uses_python_exclude_newer_filter(self) -> bool:
+        return self.exclude_newer_policy.active and self._exclude_newer_timestamp() is None
+
     def _load_channels(
         self,
         urls_to_channel: dict[str, Channel] | None = None,
@@ -535,6 +555,13 @@ def _load_repo_info_from_json_path(
             repodata_origin = None
         channel = Channel(channel_url)
         channel_id = self._channel_to_id(channel)
+        if self._uses_python_exclude_newer_filter():
+            return self._load_repo_info_from_filtered_json_path(
+                json_path,
+                channel_url,
+                channel_id,
+            )
+
         if try_solv and repodata_origin:
             try:
                 log.debug(
@@ -588,6 +615,64 @@ def _load_repo_info_from_json_path(
                 log.debug("Ignored SOLV writing error for %s", channel_id, exc_info=exc)
         return repo
 
+    def _load_repo_info_from_filtered_json_path(
+        self,
+        json_path: Path,
+        channel_url: str,
+        channel_id: str,
+    ) -> RepoInfo | None:
+        try:
+            repodata = json.loads(json_path.read_text())
+        except FileNotFoundError:
+            if context.offline:
+                log.warning("Could not load repodata for %s.", channel_id)
+                return None
+            raise
+
+        base_url = f"{channel_url.rstrip('/')}/"
+        packages = []
+        for package_group in ("packages", "packages.conda"):
+            for filename, record in repodata.get(package_group, {}).items():
+                package_url = record.get("url") or f"{base_url}{filename}"
+                if not self._record_allowed(record, filename, channel_url, package_url):
+                    continue
+                packages.append(
+                    _package_info_from_package_dict(
+                        record,
+                        filename,
+                        url=package_url,
+                        channel_id=channel_id,
+                        add_pip_as_python_dependency=self._add_pip_as_python_dependency,
+                    )
+                )
+
+        return self.db.add_repo_from_packages(
+            packages=packages,
+            name=channel_url,
+            add_pip_as_python_dependency=PipAsPythonDependency(
+                context.add_pip_as_python_dependency
+            ),
+        )
+
+    def _record_allowed(
+        self,
+        record: PackageRecordDict,
+        filename: str,
+        channel_url: str,
+        package_url: str,
+    ) -> bool:
+        if not self._uses_python_exclude_newer_filter():
+            return True
+
+        return self.exclude_newer_policy.should_include(
+            {
+                **record,
+                "channel": channel_url,
+                "fn": record.get("fn") or filename,
+                "url": package_url,
+            }
+        )
+
     def _channel_to_id(self, channel: Channel):
         channel_id = channel.canonical_name
         if channel_id in context.custom_multichannels:
@@ -651,10 +736,18 @@ def _load_repo_info_from_repodata_dict(
             packages = []
             for package_group in ("packages", "packages.conda"):
                 for filename, record in repodata.get(package_group, {}).items():
+                    package_url = f"{base_url}{filename}"
+                    if not self._record_allowed(
+                        record,
+                        filename,
+                        channel_url,
+                        package_url,
+                    ):
+                        continue
                     package = _package_info_from_package_dict(
                         record,
                         filename,
-                        url=f"{base_url}{filename}",
+                        url=package_url,
                         channel_id=channel_id,
                         add_pip_as_python_dependency=self._add_pip_as_python_dependency,
                     )

conda_libmamba_solver/solver.py

@@ -79,6 +79,9 @@
 class LibMambaSolver(Solver):
     MAX_SOLVER_ATTEMPTS_CAP = 10
     _uses_ssc = False
+    supports_exclude_newer_global = True
+    supports_exclude_newer_channel = True
+    supports_exclude_newer_package = True
 
     @staticmethod
     @cache
@@ -273,6 +276,7 @@ def _collect_all_metadata(
             ),
             pkgs_dirs=context.pkgs_dirs if context.offline else (),
             in_state=in_state,
+            exclude_newer_policy=self.exclude_newer_policy,
         )
         for channel in conda_build_channels:
             index.reload_channel(channel)

news/exclude-newer

@@ -0,0 +1,22 @@
+### Enhancements
+
+* Pass conda's resolved `exclude_newer` global cutoff to libmambapy's
+  `Database` as `exclude_newer_timestamp`, enabling native
+  `--exclude-newer` support when the upstream mamba PR lands. (#905,
+  conda/conda#15759)
+
+### Bug fixes
+
+* <news item>
+
+### Deprecations
+
+* <news item>
+
+### Docs
+
+* <news item>
+
+### Other
+
+* <news item>

tests/test_index.py

@@ -12,6 +12,7 @@
 import pytest
 from conda.base.context import context, reset_context
 from conda.common.compat import on_win
+from conda.core.exclude_newer import ExcludeNewerPolicy
 from conda.core.subdir_data import SubdirData
 from conda.gateways.logging import initialize_logging
 from conda.models.channel import Channel
@@ -36,6 +37,8 @@
 
 initialize_logging()
 DATA = Path(__file__).parent / "data"
+NOW = 1_700_000_000.0
+DAY = 86400
 
 
 def test_given_channels(monkeypatch: pytest.MonkeyPatch, tmp_path: os.PathLike):
@@ -434,3 +437,55 @@ def test_package_info_from_package_dict_add_pip_invalid_version():
     # pip should NOT be appended for invalid Python versions
     assert "pip" not in package_info.dependencies
     assert len(package_info.dependencies) == 0
+
+
+def test_exclude_newer_timestamp_unset():
+    index = object.__new__(LibMambaIndexHelper)
+    index.exclude_newer_policy = ExcludeNewerPolicy.disabled()
+
+    assert index._exclude_newer_timestamp() is None
+
+
+def test_exclude_newer_timestamp_uses_resolved_global_cutoff():
+    index = object.__new__(LibMambaIndexHelper)
+    index.exclude_newer_policy = ExcludeNewerPolicy(global_cutoff=1234.56)
+
+    assert index._exclude_newer_timestamp() == 1234
+
+
+def test_exclude_newer_timestamp_is_disabled_for_policy_overrides():
+    index = object.__new__(LibMambaIndexHelper)
+    index.exclude_newer_policy = ExcludeNewerPolicy.from_values(
+        "1d",
+        {"openssl": False},
+        channel_settings=({"channel": "https://example.test/conda", "exclude_newer": "3d"},),
+        now=NOW,
+    )
+
+    assert index._exclude_newer_timestamp() is None
+    assert index._uses_python_exclude_newer_filter()
+
+
+def test_exclude_newer_record_filter_honors_package_and_channel_overrides():
+    index = object.__new__(LibMambaIndexHelper)
+    index.exclude_newer_policy = ExcludeNewerPolicy.from_values(
+        "1d",
+        {"openssl": "false", "numpy": "1d"},
+        channel_settings=({"channel": "https://example.test/conda", "exclude_newer": "3d"},),
+        now=NOW,
+    )
+
+    def allowed(name: str, channel_url: str, timestamp: float) -> bool:
+        filename = f"{name}-1.0-0.tar.bz2"
+        package_url = f"{channel_url}/{filename}"
+        return index._record_allowed(
+            {"name": name, "timestamp": timestamp},
+            filename,
+            channel_url,
+            package_url,
+        )
+
+    assert allowed("openssl", "https://example.test/conda/linux-64", NOW - 60)
+    assert allowed("numpy", "https://example.test/conda/linux-64", NOW - 2 * DAY)
+    assert not allowed("scipy", "https://example.test/conda/linux-64", NOW - 2 * DAY)
+    assert allowed("scipy", "https://other.example.test/conda/linux-64", NOW - 2 * DAY)

tests/test_solver.py

@@ -35,6 +35,12 @@
     from pytest import MonkeyPatch
 
 
+def test_solver_declares_exclude_newer_support() -> None:
+    assert Solver.supports_exclude_newer_global is True
+    assert Solver.supports_exclude_newer_channel is True
+    assert Solver.supports_exclude_newer_package is True
+
+
 def test_python_downgrade_reinstalls_noarch_packages(
     tmp_env: TmpEnvFixture,
     conda_cli: CondaCLIFixture,