diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 170b8115..f35cf21e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,7 +42,7 @@ env: # we can't break classic from here; no need to test it # those tests will still be available for local debugging if necessary CONDA_TEST_SOLVERS: libmamba - CONDA_REF: main # pin conda version to latest for more stable tests + CONDA_REF: feature/repodata-filter-cooldown # temporarily test against exclude-newer branch jobs: # detect whether any code changes are included in this PR @@ -110,7 +110,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 - repository: conda/conda # CONDA-LIBMAMBA-SOLVER CHANGE + repository: jezdez/conda # CONDA-LIBMAMBA-SOLVER CHANGE path: conda # CONDA-LIBMAMBA-SOLVER CHANGE ref: ${{ env.CONDA_REF }} @@ -275,7 +275,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 - repository: conda/conda # CONDA-LIBMAMBA-SOLVER CHANGE + repository: jezdez/conda # CONDA-LIBMAMBA-SOLVER CHANGE path: conda # CONDA-LIBMAMBA-SOLVER CHANGE ref: ${{ env.CONDA_REF }} @@ -572,7 +572,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 - repository: conda/conda # CONDA-LIBMAMBA-SOLVER CHANGE + repository: jezdez/conda # CONDA-LIBMAMBA-SOLVER CHANGE path: conda # CONDA-LIBMAMBA-SOLVER CHANGE ref: ${{ env.CONDA_REF }} diff --git a/conda_libmamba_solver/index.py b/conda_libmamba_solver/index.py index 7398b6d5..9cf1ba2b 100644 --- a/conda_libmamba_solver/index.py +++ b/conda_libmamba_solver/index.py @@ -73,6 +73,7 @@ from __future__ import annotations +import json import logging import os from dataclasses import dataclass @@ -85,6 +86,7 @@ from conda.common.compat import on_win from conda.common.io import DummyExecutor, ThreadLimitedThreadPoolExecutor, time_recorder from conda.common.url import path_to_url, remove_auth, split_anaconda_token +from conda.core.exclude_newer import ExcludeNewerPolicy from conda.core.package_cache_data import PackageCacheData from conda.core.subdir_data import SubdirData from conda.models.channel import Channel @@ -254,6 +256,7 @@ def __init__( in_state: SolverInputState | None = None, *, build_repodata_subset: BuildRepodataSubset | None = None, + exclude_newer_policy: ExcludeNewerPolicy | None = None, ): platform_less_channels: list[Channel] = [] for channel in channels: @@ -274,6 +277,7 @@ def __init__( self.in_state = in_state self._add_pip_as_python_dependency = context.add_pip_as_python_dependency self.build_repodata_subset = build_repodata_subset + self.exclude_newer_policy = exclude_newer_policy or ExcludeNewerPolicy.disabled() self.db = self._init_db() self.repos: list[_ChannelRepoInfo] = self._load_channels() @@ -364,10 +368,26 @@ def _init_db(self) -> Database: home_dir=str(Path.home()), current_working_dir=os.getcwd(), ) - db = Database(params) + db_kwargs = {} + if (exclude_newer_timestamp := self._exclude_newer_timestamp()) is not None: + db_kwargs["exclude_newer_timestamp"] = exclude_newer_timestamp + db = Database(params, **db_kwargs) db.set_logger(logger_callback) return db + def _exclude_newer_timestamp(self) -> int | None: + """Return the resolved global cutoff as a Unix timestamp for libmambapy.""" + if ( + self.exclude_newer_policy.global_cutoff is None + or self.exclude_newer_policy.has_channel_overrides + or self.exclude_newer_policy.has_package_overrides + ): + return None + return int(self.exclude_newer_policy.global_cutoff) + + def _uses_python_exclude_newer_filter(self) -> bool: + return self.exclude_newer_policy.active and self._exclude_newer_timestamp() is None + def _load_channels( self, urls_to_channel: dict[str, Channel] | None = None, @@ -539,6 +559,13 @@ def _load_repo_info_from_json_path( repodata_origin = None channel = Channel(channel_url) channel_id = self._channel_to_id(channel) + if self._uses_python_exclude_newer_filter(): + return self._load_repo_info_from_filtered_json_path( + json_path, + channel_url, + channel_id, + ) + if try_solv and repodata_origin: try: log.debug( @@ -592,6 +619,64 @@ def _load_repo_info_from_json_path( log.debug("Ignored SOLV writing error for %s", channel_id, exc_info=exc) return repo + def _load_repo_info_from_filtered_json_path( + self, + json_path: Path, + channel_url: str, + channel_id: str, + ) -> RepoInfo | None: + try: + repodata = json.loads(json_path.read_text()) + except FileNotFoundError: + if context.offline: + log.warning("Could not load repodata for %s.", channel_id) + return None + raise + + base_url = f"{channel_url.rstrip('/')}/" + packages = [] + for package_group in ("packages", "packages.conda"): + for filename, record in repodata.get(package_group, {}).items(): + package_url = record.get("url") or f"{base_url}{filename}" + if not self._record_allowed(record, filename, channel_url, package_url): + continue + packages.append( + _package_info_from_package_dict( + record, + filename, + url=package_url, + channel_id=channel_id, + add_pip_as_python_dependency=self._add_pip_as_python_dependency, + ) + ) + + return self.db.add_repo_from_packages( + packages=packages, + name=channel_url, + add_pip_as_python_dependency=PipAsPythonDependency( + context.add_pip_as_python_dependency + ), + ) + + def _record_allowed( + self, + record: PackageRecordDict, + filename: str, + channel_url: str, + package_url: str, + ) -> bool: + if not self._uses_python_exclude_newer_filter(): + return True + + return self.exclude_newer_policy.should_include( + { + **record, + "channel": channel_url, + "fn": record.get("fn") or filename, + "url": package_url, + } + ) + def _channel_to_id(self, channel: Channel): channel_id = channel.canonical_name if channel_id in context.custom_multichannels: @@ -652,10 +737,13 @@ def _load_repo_info_from_repodata_dict( packages = [] for filename, record in shardlike.iter_records(): + package_url = f"{base_url}{filename}" + if not self._record_allowed(record, filename, channel_url, package_url): + continue package = _package_info_from_package_dict( record, filename, - url=f"{base_url}{filename}", + url=package_url, channel_id=channel_id, add_pip_as_python_dependency=self._add_pip_as_python_dependency, ) diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index d8d8c90b..eaecbc30 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -80,6 +80,9 @@ class LibMambaSolver(Solver): MAX_SOLVER_ATTEMPTS_CAP = 10 _uses_ssc = False + supports_exclude_newer_global = True + supports_exclude_newer_channel = True + supports_exclude_newer_package = True @staticmethod @cache @@ -278,6 +281,7 @@ def _collect_all_metadata( pkgs_dirs=context.pkgs_dirs if context.offline else (), in_state=in_state, build_repodata_subset=self._build_repodata_subset, + exclude_newer_policy=self.exclude_newer_policy, ) for channel in conda_build_channels: index.reload_channel(channel) diff --git a/news/exclude-newer b/news/exclude-newer new file mode 100644 index 00000000..3ed19d9f --- /dev/null +++ b/news/exclude-newer @@ -0,0 +1,6 @@ +### Enhancements + +* Pass conda's resolved `exclude_newer` global cutoff to libmambapy's + `Database` as `exclude_newer_timestamp`, enabling native + `--exclude-newer` support when the upstream mamba PR lands. (#905, + conda/conda#15759) diff --git a/tests/test_index.py b/tests/test_index.py index fb5d49da..7d07bc7d 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -12,6 +12,7 @@ import pytest from conda.base.context import reset_context from conda.common.compat import on_win +from conda.core.exclude_newer import ExcludeNewerPolicy from conda.core.subdir_data import SubdirData from conda.gateways.logging import initialize_logging from conda.models.channel import Channel @@ -27,6 +28,8 @@ initialize_logging() DATA = Path(__file__).parent / "data" +NOW = 1_700_000_000.0 +DAY = 86400 def test_given_channels(monkeypatch: pytest.MonkeyPatch, tmp_path: os.PathLike): @@ -257,3 +260,55 @@ def test_package_info_from_package_dict_add_pip_invalid_version(): # pip should NOT be appended for invalid Python versions assert "pip" not in package_info.dependencies assert len(package_info.dependencies) == 0 + + +def test_exclude_newer_timestamp_unset(): + index = object.__new__(LibMambaIndexHelper) + index.exclude_newer_policy = ExcludeNewerPolicy.disabled() + + assert index._exclude_newer_timestamp() is None + + +def test_exclude_newer_timestamp_uses_resolved_global_cutoff(): + index = object.__new__(LibMambaIndexHelper) + index.exclude_newer_policy = ExcludeNewerPolicy(global_cutoff=1234.56) + + assert index._exclude_newer_timestamp() == 1234 + + +def test_exclude_newer_timestamp_is_disabled_for_policy_overrides(): + index = object.__new__(LibMambaIndexHelper) + index.exclude_newer_policy = ExcludeNewerPolicy.from_values( + "1d", + {"openssl": False}, + channel_settings=({"channel": "https://example.test/conda", "exclude_newer": "3d"},), + now=NOW, + ) + + assert index._exclude_newer_timestamp() is None + assert index._uses_python_exclude_newer_filter() + + +def test_exclude_newer_record_filter_honors_package_and_channel_overrides(): + index = object.__new__(LibMambaIndexHelper) + index.exclude_newer_policy = ExcludeNewerPolicy.from_values( + "1d", + {"openssl": "false", "numpy": "1d"}, + channel_settings=({"channel": "https://example.test/conda", "exclude_newer": "3d"},), + now=NOW, + ) + + def allowed(name: str, channel_url: str, timestamp: float) -> bool: + filename = f"{name}-1.0-0.tar.bz2" + package_url = f"{channel_url}/{filename}" + return index._record_allowed( + {"name": name, "timestamp": timestamp}, + filename, + channel_url, + package_url, + ) + + assert allowed("openssl", "https://example.test/conda/linux-64", NOW - 60) + assert allowed("numpy", "https://example.test/conda/linux-64", NOW - 2 * DAY) + assert not allowed("scipy", "https://example.test/conda/linux-64", NOW - 2 * DAY) + assert allowed("scipy", "https://other.example.test/conda/linux-64", NOW - 2 * DAY) diff --git a/tests/test_solver.py b/tests/test_solver.py index 8d9a01b2..e4974353 100644 --- a/tests/test_solver.py +++ b/tests/test_solver.py @@ -35,6 +35,12 @@ from pytest import MonkeyPatch +def test_solver_declares_exclude_newer_support() -> None: + assert Solver.supports_exclude_newer_global is True + assert Solver.supports_exclude_newer_channel is True + assert Solver.supports_exclude_newer_package is True + + def test_python_downgrade_reinstalls_noarch_packages( tmp_env: TmpEnvFixture, conda_cli: CondaCLIFixture,