diff --git a/libmamba/include/mamba/solver/libsolv/database.hpp b/libmamba/include/mamba/solver/libsolv/database.hpp index 968f02444c..3c06b18f7e 100644 --- a/libmamba/include/mamba/solver/libsolv/database.hpp +++ b/libmamba/include/mamba/solver/libsolv/database.hpp @@ -8,6 +8,7 @@ #define MAMBA_SOLVER_LIBSOLV_DATABASE_HPP #include +#include #include #include #include @@ -64,6 +65,7 @@ namespace mamba::solver::libsolv struct Settings { MatchSpecParser matchspec_parser = MatchSpecParser::Libsolv; + std::optional exclude_newer_timestamp = std::nullopt; }; using logger_type = std::function; diff --git a/libmamba/src/solver/libsolv/database.cpp b/libmamba/src/solver/libsolv/database.cpp index 3cd6e0b3f4..9822a0e7ef 100644 --- a/libmamba/src/solver/libsolv/database.cpp +++ b/libmamba/src/solver/libsolv/database.cpp @@ -183,7 +183,8 @@ namespace mamba::solver::libsolv channel_id, package_types, settings().matchspec_parser, - verify_artifacts + verify_artifacts, + settings().exclude_newer_timestamp ); } @@ -261,6 +262,13 @@ namespace mamba::solver::libsolv void Database::add_repo_from_packages_impl_loop(const RepoInfo& repo, const specs::PackageInfo& pkg) { + if (const auto cutoff = settings().exclude_newer_timestamp) + { + if (normalize_conda_timestamp(pkg.timestamp) > *cutoff) + { + return; + } + } auto s_repo = solv::ObjRepoView(*repo.m_ptr); auto [id, solv] = s_repo.add_solvable(); set_solvable(pool(), solv, pkg, settings().matchspec_parser); diff --git a/libmamba/src/solver/libsolv/helpers.cpp b/libmamba/src/solver/libsolv/helpers.cpp index 5289e24401..f5b16d94e3 100644 --- a/libmamba/src/solver/libsolv/helpers.cpp +++ b/libmamba/src/solver/libsolv/helpers.cpp @@ -40,10 +40,6 @@ namespace mamba::solver::libsolv { - // Beyond this value, the timestamp would be in milliseconds and therefore should be converted - // to seconds. - inline constexpr auto MAX_CONDA_TIMESTAMP = 253402300799ULL; - void set_solvable( solv::ObjPool& pool, solv::ObjSolvableView solv, @@ -72,9 +68,7 @@ namespace mamba::solver::libsolv // TODO conda timestamp are not Unix timestamp. // Libsolv normalize them this way, we need to do the same here otherwise the current // package may get arbitrary priority. - solv.set_timestamp( - (pkg.timestamp > MAX_CONDA_TIMESTAMP) ? (pkg.timestamp / 1000) : pkg.timestamp - ); + solv.set_timestamp(normalize_conda_timestamp(pkg.timestamp)); solv.set_md5(pkg.md5); solv.set_sha256(pkg.sha256); solv.set_python_site_packages_path(pkg.python_site_packages_path); @@ -215,16 +209,15 @@ namespace mamba::solver::libsolv template [[nodiscard]] auto set_solvable( solv::ObjPool& pool, - // const std::string& repo_url_str, const specs::CondaURL& repo_url, const std::string& channel_id, solv::ObjSolvableView solv, - const std::string& filename, JSONObject&& pkg, const std::optional& signatures, const std::string& default_subdir, - MatchSpecParser parser + MatchSpecParser parser, + std::uint64_t* out_timestamp = nullptr ) -> bool { // Not available from RepoDataPackage @@ -336,10 +329,26 @@ namespace mamba::solver::libsolv // TODO conda timestamp are not Unix timestamp. // Libsolv normalize them this way, we need to do the same here otherwise the current // package may get arbitrary priority. + std::optional policy_timestamp; + if (auto indexed_timestamp = pkg["indexed_timestamp"]; !indexed_timestamp.error()) + { + policy_timestamp = normalize_conda_timestamp( + indexed_timestamp.get_uint64().value_unsafe() + ); + } + if (auto timestamp = pkg["timestamp"]; !timestamp.error()) { - const auto time = timestamp.get_uint64().value_unsafe(); - solv.set_timestamp((time > MAX_CONDA_TIMESTAMP) ? (time / 1000) : time); + const auto normalized = normalize_conda_timestamp( + timestamp.get_uint64().value_unsafe() + ); + solv.set_timestamp(normalized); + policy_timestamp = policy_timestamp.value_or(normalized); + } + + if (out_timestamp && policy_timestamp) + { + *out_timestamp = *policy_timestamp; } if (auto depends = pkg["depends"].get_array(); !depends.error()) @@ -443,7 +452,8 @@ namespace mamba::solver::libsolv const std::optional& signatures, Filter&& filter, OnParsed&& on_parsed, - MatchSpecParser parser + MatchSpecParser parser, + std::optional exclude_newer_timestamp = std::nullopt ) { auto packages_as_object = packages.get_object(); @@ -453,6 +463,7 @@ namespace mamba::solver::libsolv if (filter(filename)) { auto [id, solv] = repo.add_solvable(); + std::uint64_t pkg_timestamp = 0; const bool parsed = set_solvable( pool, repo_url, @@ -462,11 +473,19 @@ namespace mamba::solver::libsolv pkg_field.value(), signatures, default_subdir, - parser + parser, + &pkg_timestamp ); if (parsed) { - on_parsed(filename); + if (exclude_newer_timestamp && pkg_timestamp > *exclude_newer_timestamp) + { + repo.remove_solvable(id, /* reuse_id= */ true); + } + else + { + on_parsed(filename); + } } else { @@ -486,7 +505,8 @@ namespace mamba::solver::libsolv const std::string& default_subdir, JSONObject& packages, const std::optional& signatures, - MatchSpecParser parser + MatchSpecParser parser, + std::optional exclude_newer_timestamp = std::nullopt ) { return set_repo_solvables_impl( @@ -499,7 +519,8 @@ namespace mamba::solver::libsolv signatures, /* filter= */ [](const auto&) { return true; }, /* on_parsed= */ [](const auto&) {}, - parser + parser, + exclude_newer_timestamp ); } @@ -512,7 +533,8 @@ namespace mamba::solver::libsolv const std::string& default_subdir, JSONObject& packages, const std::optional& signatures, - MatchSpecParser parser + MatchSpecParser parser, + std::optional exclude_newer_timestamp = std::nullopt ) -> util::flat_set { auto filenames = util::flat_set(); @@ -528,7 +550,8 @@ namespace mamba::solver::libsolv /* on_parsed= */ [&](const auto& fn) { filenames.insert(std::string(specs::strip_archive_extension(fn))); }, - parser + parser, + exclude_newer_timestamp ); // Sort only once return filenames; @@ -544,7 +567,8 @@ namespace mamba::solver::libsolv JSONObject& packages, const std::optional& signatures, const SortedStringRange& added, - MatchSpecParser parser + MatchSpecParser parser, + std::optional exclude_newer_timestamp = std::nullopt ) { return set_repo_solvables_impl( @@ -558,7 +582,8 @@ namespace mamba::solver::libsolv /* filter= */ [&](const auto& fn) { return !added.contains(specs::strip_archive_extension(fn)); }, /* on_parsed= */ [&](const auto&) {}, - parser + parser, + exclude_newer_timestamp ); } } @@ -623,7 +648,8 @@ namespace mamba::solver::libsolv const std::string& channel_id, PackageTypes package_types, MatchSpecParser ms_parser, - bool verify_artifacts + bool verify_artifacts, + std::optional exclude_newer_timestamp ) -> expected_t { LOG_INFO << "Reading repodata.json file " << filename << " for repo " << repo.name() @@ -739,7 +765,8 @@ namespace mamba::solver::libsolv default_subdir, pkgs, json_signatures, - ms_parser + ms_parser, + exclude_newer_timestamp ); } if (auto pkgs = repodata_doc["packages"]; !pkgs.error()) @@ -753,7 +780,8 @@ namespace mamba::solver::libsolv pkgs, json_signatures, added, - ms_parser + ms_parser, + exclude_newer_timestamp ); } } @@ -770,7 +798,8 @@ namespace mamba::solver::libsolv default_subdir, pkgs, json_signatures, - ms_parser + ms_parser, + exclude_newer_timestamp ); } @@ -785,7 +814,8 @@ namespace mamba::solver::libsolv default_subdir, pkgs, json_signatures, - ms_parser + ms_parser, + exclude_newer_timestamp ); } } diff --git a/libmamba/src/solver/libsolv/helpers.hpp b/libmamba/src/solver/libsolv/helpers.hpp index 2054625d18..878c688d4a 100644 --- a/libmamba/src/solver/libsolv/helpers.hpp +++ b/libmamba/src/solver/libsolv/helpers.hpp @@ -7,6 +7,7 @@ #ifndef MAMBA_SOLVER_LIBSOLV_HELPERS #define MAMBA_SOLVER_LIBSOLV_HELPERS +#include #include #include #include @@ -37,6 +38,15 @@ namespace mamba::fs namespace mamba::solver::libsolv { + // Beyond this value, the timestamp would be in milliseconds and therefore should be + // converted to seconds. + inline constexpr std::uint64_t MAX_CONDA_TIMESTAMP = 253402300799ULL; + + [[nodiscard]] constexpr auto normalize_conda_timestamp(std::uint64_t timestamp) -> std::uint64_t + { + return (timestamp > MAX_CONDA_TIMESTAMP) ? (timestamp / 1000) : timestamp; + } + void set_solvable( solv::ObjPool& pool, solv::ObjSolvableView solv, @@ -62,7 +72,8 @@ namespace mamba::solver::libsolv const std::string& channel_id, PackageTypes types, MatchSpecParser parser, - bool verify_artifacts + bool verify_artifacts, + std::optional exclude_newer_timestamp = std::nullopt ) -> expected_t; [[nodiscard]] auto read_solv( diff --git a/libmamba/tests/src/solver/libsolv/test_database.cpp b/libmamba/tests/src/solver/libsolv/test_database.cpp index 062e07109d..0a52d2a7dc 100644 --- a/libmamba/tests/src/solver/libsolv/test_database.cpp +++ b/libmamba/tests/src/solver/libsolv/test_database.cpp @@ -5,6 +5,8 @@ // The full license is in the file LICENSE, distributed with this software. #include +#include +#include #include #include @@ -189,6 +191,51 @@ namespace } } + SECTION("exclude_newer_timestamp filters packages from add_repo_from_packages") + { + const std::uint64_t cutoff = 2000; + auto db_filtered = libsolv::Database( + {}, + { matchspec_parser, /* exclude_newer_timestamp= */ cutoff } + ); + + auto old_pkg = specs::PackageInfo(); + old_pkg.name = "old-pkg"; + old_pkg.version = "1.0"; + old_pkg.timestamp = 1000; + + auto new_pkg = specs::PackageInfo(); + new_pkg.name = "new-pkg"; + new_pkg.version = "1.0"; + new_pkg.timestamp = 3000; + + auto pkgs = std::array{ old_pkg, new_pkg }; + auto repo1 = db_filtered.add_repo_from_packages(pkgs, "repo1"); + REQUIRE(repo1.package_count() == 1); + + db_filtered.for_each_package_in_repo( + repo1, + [](const auto& p) { REQUIRE(p.name == "old-pkg"); } + ); + } + + SECTION("exclude_newer_timestamp normalizes millisecond timestamps") + { + const std::uint64_t cutoff = 2000000000; + auto db_filtered = libsolv::Database( + {}, + { matchspec_parser, /* exclude_newer_timestamp= */ cutoff } + ); + + auto ms_pkg = specs::PackageInfo(); + ms_pkg.name = "ms-pkg"; + ms_pkg.version = "1.0"; + ms_pkg.timestamp = 1500000000000; // 1.5e12 ms → 1.5e9 seconds, below cutoff + + auto repo1 = db_filtered.add_repo_from_packages(std::array{ ms_pkg }, "repo1"); + REQUIRE(repo1.package_count() == 1); + } + SECTION("Add repo from repodata with no extra pip") { const auto repodata = mambatests::test_data_dir @@ -218,6 +265,86 @@ namespace REQUIRE(found_python); } + SECTION("exclude_newer_timestamp filters packages from repodata JSON") + { + const auto repodata = mambatests::test_data_dir + / "repodata/conda-forge-numpy-linux-64.json"; + auto db_filtered = libsolv::Database( + {}, + { matchspec_parser, /* exclude_newer_timestamp= */ std::uint64_t(1700000000) } + ); + auto repo1 = db_filtered.add_repo_from_repodata_json( + repodata, + "https://conda.anaconda.org/conda-forge/linux-64", + "conda-forge", + libsolv::PipAsPythonDependency::No + ); + REQUIRE(repo1.has_value()); + REQUIRE(repo1->package_count() < 33); + REQUIRE(repo1->package_count() > 0); + + auto db_unfiltered = libsolv::Database({}, { matchspec_parser }); + auto unfiltered_repo = db_unfiltered.add_repo_from_repodata_json( + repodata, + "https://conda.anaconda.org/conda-forge/linux-64", + "conda-forge", + libsolv::PipAsPythonDependency::No + ); + REQUIRE(unfiltered_repo.has_value()); + REQUIRE(unfiltered_repo->package_count() > repo1->package_count()); + } + + SECTION("exclude_newer_timestamp prefers indexed_timestamp from repodata JSON") + { + auto tmp_dir = TemporaryDirectory(); + const auto repodata = tmp_dir.path() / "repodata.json"; + std::ofstream out_file(repodata.std_path()); + out_file << R"({ + "packages": { + "excluded-pkg-1.0-bld.tar.bz2": { + "name": "excluded-pkg", + "version": "1.0", + "build": "bld", + "build_number": 0, + "subdir": "linux-64", + "depends": [], + "timestamp": 1000, + "indexed_timestamp": 3000 + }, + "included-pkg-1.0-bld.tar.bz2": { + "name": "included-pkg", + "version": "1.0", + "build": "bld", + "build_number": 0, + "subdir": "linux-64", + "depends": [], + "timestamp": 3000, + "indexed_timestamp": 1000 + } + }, + "packages.conda": {} + })"; + out_file.close(); + + auto db_filtered = libsolv::Database( + {}, + { matchspec_parser, /* exclude_newer_timestamp= */ std::uint64_t(2000) } + ); + auto repo1 = db_filtered.add_repo_from_repodata_json( + repodata, + "https://conda.anaconda.org/conda-forge/linux-64", + "conda-forge", + libsolv::PipAsPythonDependency::No + ); + REQUIRE(repo1.has_value()); + REQUIRE(repo1->package_count() == 1); + + db_filtered.for_each_package_in_repo( + *repo1, + [](const auto& p) { REQUIRE(p.name == "included-pkg"); } + ); + } + SECTION("Add repo from repodata with extra pip") { const auto repodata = mambatests::test_data_dir diff --git a/libmambapy/bindings/solver_libsolv.cpp b/libmambapy/bindings/solver_libsolv.cpp index 50f1c68d93..f1f8a01ed1 100644 --- a/libmambapy/bindings/solver_libsolv.cpp +++ b/libmambapy/bindings/solver_libsolv.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include "mamba/solver/libsolv/database.hpp" #include "mamba/solver/libsolv/parameters.hpp" @@ -142,18 +143,22 @@ namespace mambapy py::class_(m, "Database") .def( py::init( - [](specs::ChannelResolveParams channel_params, MatchSpecParser matchspec_parser) + [](specs::ChannelResolveParams channel_params, + MatchSpecParser matchspec_parser, + std::optional exclude_newer_timestamp) { return Database( channel_params, Database::Settings{ matchspec_parser, + exclude_newer_timestamp, } ); } ), py::arg("channel_params"), - py::arg("matchspec_parser") = MatchSpecParser::Libsolv + py::arg("matchspec_parser") = MatchSpecParser::Libsolv, + py::arg("exclude_newer_timestamp") = py::none() ) .def("set_logger", &Database::set_logger, py::call_guard()) .def( diff --git a/libmambapy/tests/test_solver_libsolv.py b/libmambapy/tests/test_solver_libsolv.py index f7e65dcf1a..8c006d2edf 100644 --- a/libmambapy/tests/test_solver_libsolv.py +++ b/libmambapy/tests/test_solver_libsolv.py @@ -182,6 +182,90 @@ def test_Database_RepoInfo_from_packages(add_pip_as_python_dependency, matchspec assert db.installed_repo() is None +def test_Database_exclude_newer_timestamp_filters_packages(): + cutoff = 2000 + db = libsolv.Database( + libmambapy.specs.ChannelResolveParams(), + exclude_newer_timestamp=cutoff, + ) + + old_pkg = libmambapy.specs.PackageInfo(name="old-pkg") + old_pkg.version = "1.0" + old_pkg.timestamp = 1000 + + new_pkg = libmambapy.specs.PackageInfo(name="new-pkg") + new_pkg.version = "1.0" + new_pkg.timestamp = 3000 + + repo = db.add_repo_from_packages([old_pkg, new_pkg], name="test") + assert repo.package_count() == 1 + + pkgs = db.packages_in_repo(repo) + assert len(pkgs) == 1 + assert pkgs[0].name == "old-pkg" + + +def test_Database_exclude_newer_timestamp_none_keeps_all(): + db = libsolv.Database( + libmambapy.specs.ChannelResolveParams(), + exclude_newer_timestamp=None, + ) + + pkg = libmambapy.specs.PackageInfo(name="pkg") + pkg.version = "1.0" + pkg.timestamp = 9999 + + repo = db.add_repo_from_packages([pkg], name="test") + assert repo.package_count() == 1 + + +def test_Database_exclude_newer_timestamp_repodata(tmp_path): + repodata_file = tmp_path / "repodata.json" + with open(repodata_file, "w+") as f: + json.dump( + { + "packages": { + "excluded-pkg-1.0-bld.tar.bz2": { + "name": "excluded-pkg", + "version": "1.0", + "build": "bld", + "build_number": 0, + "subdir": "linux-64", + "depends": [], + "timestamp": 1000, + "indexed_timestamp": 3000, + }, + "included-pkg-1.0-bld.tar.bz2": { + "name": "included-pkg", + "version": "1.0", + "build": "bld", + "build_number": 0, + "subdir": "linux-64", + "depends": [], + "timestamp": 3000, + "indexed_timestamp": 1000, + }, + }, + "packages.conda": {}, + }, + f, + ) + + db = libsolv.Database( + libmambapy.specs.ChannelResolveParams(), + exclude_newer_timestamp=2000, + ) + repo = db.add_repo_from_repodata_json( + repodata_file, + "https://example.com/linux-64", + "test-channel", + ) + assert repo is not None + assert repo.package_count() == 1 + pkgs = db.packages_in_repo(repo) + assert pkgs[0].name == "included-pkg" + + @pytest.fixture def tmp_repodata_json(tmp_path): file = tmp_path / "repodata.json"