Skip to content

feat: Support excluding recent builds using timestamp (--exclude-newer) #4228

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jezdez wants to merge 5 commits into
base: main
Choose a base branch
from
+297 −30
Open

feat: Support excluding recent builds using timestamp (--exclude-newer) #4228

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
32645dd
Add exclude_newer_timestamp to Database::Settings
jezdez Apr 10, 2026
f446a4e
Add tests for exclude_newer_timestamp filtering
jezdez Apr 10, 2026
94020ac
Fix exclude_newer filtering for repodata JSON path
jezdez Apr 10, 2026
891ee17
Fix clang-format violations
jezdez Apr 10, 2026
96d2ca7
Address exclude-newer review feedback
jezdez Jun 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions libmamba/include/mamba/solver/libsolv/database.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#define MAMBA_SOLVER_LIBSOLV_DATABASE_HPP

#include <concepts>
#include <cstdint>
#include <functional>
#include <iterator>
#include <memory>
Expand Down Expand Up @@ -64,6 +65,7 @@ namespace mamba::solver::libsolv
struct Settings
{
MatchSpecParser matchspec_parser = MatchSpecParser::Libsolv;
std::optional<std::uint64_t> exclude_newer_timestamp = std::nullopt;
};

using logger_type = std::function<void(LogLevel, std::string_view)>;
Expand Down
10 changes: 9 additions & 1 deletion libmamba/src/solver/libsolv/database.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,8 @@ namespace mamba::solver::libsolv
channel_id,
package_types,
settings().matchspec_parser,
verify_artifacts
verify_artifacts,
settings().exclude_newer_timestamp
);
}

Expand Down Expand Up @@ -261,6 +262,13 @@ namespace mamba::solver::libsolv
void
Database::add_repo_from_packages_impl_loop(const RepoInfo& repo, const specs::PackageInfo& pkg)
{
if (const auto cutoff = settings().exclude_newer_timestamp)
{
if (normalize_conda_timestamp(pkg.timestamp) > *cutoff)
{
return;
}
}
auto s_repo = solv::ObjRepoView(*repo.m_ptr);
auto [id, solv] = s_repo.add_solvable();
set_solvable(pool(), solv, pkg, settings().matchspec_parser);
Expand Down
82 changes: 56 additions & 26 deletions libmamba/src/solver/libsolv/helpers.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,6 @@

namespace mamba::solver::libsolv
{
// Beyond this value, the timestamp would be in milliseconds and therefore should be converted
// to seconds.
inline constexpr auto MAX_CONDA_TIMESTAMP = 253402300799ULL;

void set_solvable(
solv::ObjPool& pool,
solv::ObjSolvableView solv,
Expand Down Expand Up @@ -72,9 +68,7 @@ namespace mamba::solver::libsolv
// TODO conda timestamp are not Unix timestamp.
// Libsolv normalize them this way, we need to do the same here otherwise the current
// package may get arbitrary priority.
solv.set_timestamp(
(pkg.timestamp > MAX_CONDA_TIMESTAMP) ? (pkg.timestamp / 1000) : pkg.timestamp
);
solv.set_timestamp(normalize_conda_timestamp(pkg.timestamp));
solv.set_md5(pkg.md5);
solv.set_sha256(pkg.sha256);
solv.set_python_site_packages_path(pkg.python_site_packages_path);
Expand Down Expand Up @@ -215,16 +209,15 @@ namespace mamba::solver::libsolv
template <class JSONObject>
[[nodiscard]] auto set_solvable(
solv::ObjPool& pool,
// const std::string& repo_url_str,
const specs::CondaURL& repo_url,
const std::string& channel_id,
solv::ObjSolvableView solv,

const std::string& filename,
JSONObject&& pkg,
const std::optional<nlohmann::json>& signatures,
const std::string& default_subdir,
MatchSpecParser parser
MatchSpecParser parser,
std::uint64_t* out_timestamp = nullptr
) -> bool
{
// Not available from RepoDataPackage
Expand Down Expand Up @@ -336,10 +329,26 @@ namespace mamba::solver::libsolv
// TODO conda timestamp are not Unix timestamp.
// Libsolv normalize them this way, we need to do the same here otherwise the current
// package may get arbitrary priority.
std::optional<std::uint64_t> policy_timestamp;
if (auto indexed_timestamp = pkg["indexed_timestamp"]; !indexed_timestamp.error())
{
policy_timestamp = normalize_conda_timestamp(
indexed_timestamp.get_uint64().value_unsafe()
);
}

if (auto timestamp = pkg["timestamp"]; !timestamp.error())
{
const auto time = timestamp.get_uint64().value_unsafe();
solv.set_timestamp((time > MAX_CONDA_TIMESTAMP) ? (time / 1000) : time);
const auto normalized = normalize_conda_timestamp(
timestamp.get_uint64().value_unsafe()
);
solv.set_timestamp(normalized);
policy_timestamp = policy_timestamp.value_or(normalized);
}

if (out_timestamp && policy_timestamp)
{
*out_timestamp = *policy_timestamp;
}

if (auto depends = pkg["depends"].get_array(); !depends.error())
Expand Down Expand Up @@ -443,7 +452,8 @@ namespace mamba::solver::libsolv
const std::optional<nlohmann::json>& signatures,
Filter&& filter,
OnParsed&& on_parsed,
MatchSpecParser parser
MatchSpecParser parser,
std::optional<std::uint64_t> exclude_newer_timestamp = std::nullopt
)
{
auto packages_as_object = packages.get_object();
Expand All @@ -453,6 +463,7 @@ namespace mamba::solver::libsolv
if (filter(filename))
{
auto [id, solv] = repo.add_solvable();
std::uint64_t pkg_timestamp = 0;
const bool parsed = set_solvable(
pool,
repo_url,
Expand All @@ -462,11 +473,19 @@ namespace mamba::solver::libsolv
pkg_field.value(),
signatures,
default_subdir,
parser
parser,
&pkg_timestamp
);
if (parsed)
{
on_parsed(filename);
if (exclude_newer_timestamp && pkg_timestamp > *exclude_newer_timestamp)
{
repo.remove_solvable(id, /* reuse_id= */ true);

@jjerphan jjerphan Apr 22, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Side-note: Removing solvable here is the most appropriate to me. I think that we might even be able (in the case of sharded repodata usage) to filter builds on timestamp at the earliest (see #4214); yet I am not sure that there will be massive benefits (more maintenance and complexity for probably not the most reduced overhead).

@jezdez jezdez Jun 8, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed. I kept the native libmamba implementation at the solvable insertion/removal point for this PR. The conda-side scoped policy uses Python repodata filtering before libmamba; earlier shard-level filtering can be revisited separately once the shard APIs settle.

}
else
{
on_parsed(filename);
}
}
else
{
Expand All @@ -486,7 +505,8 @@ namespace mamba::solver::libsolv
const std::string& default_subdir,
JSONObject& packages,
const std::optional<nlohmann::json>& signatures,
MatchSpecParser parser
MatchSpecParser parser,
std::optional<std::uint64_t> exclude_newer_timestamp = std::nullopt
)
{
return set_repo_solvables_impl(
Expand All @@ -499,7 +519,8 @@ namespace mamba::solver::libsolv
signatures,
/* filter= */ [](const auto&) { return true; },
/* on_parsed= */ [](const auto&) {},
parser
parser,
exclude_newer_timestamp
);
}

Expand All @@ -512,7 +533,8 @@ namespace mamba::solver::libsolv
const std::string& default_subdir,
JSONObject& packages,
const std::optional<nlohmann::json>& signatures,
MatchSpecParser parser
MatchSpecParser parser,
std::optional<std::uint64_t> exclude_newer_timestamp = std::nullopt
) -> util::flat_set<std::string>
{
auto filenames = util::flat_set<std::string>();
Expand All @@ -528,7 +550,8 @@ namespace mamba::solver::libsolv
/* on_parsed= */
[&](const auto& fn)
{ filenames.insert(std::string(specs::strip_archive_extension(fn))); },
parser
parser,
exclude_newer_timestamp
);
// Sort only once
return filenames;
Expand All @@ -544,7 +567,8 @@ namespace mamba::solver::libsolv
JSONObject& packages,
const std::optional<nlohmann::json>& signatures,
const SortedStringRange& added,
MatchSpecParser parser
MatchSpecParser parser,
std::optional<std::uint64_t> exclude_newer_timestamp = std::nullopt
)
{
return set_repo_solvables_impl(
Expand All @@ -558,7 +582,8 @@ namespace mamba::solver::libsolv
/* filter= */
[&](const auto& fn) { return !added.contains(specs::strip_archive_extension(fn)); },
/* on_parsed= */ [&](const auto&) {},
parser
parser,
exclude_newer_timestamp
);
}
}
Expand Down Expand Up @@ -623,7 +648,8 @@ namespace mamba::solver::libsolv
const std::string& channel_id,
PackageTypes package_types,
MatchSpecParser ms_parser,
bool verify_artifacts
bool verify_artifacts,
std::optional<std::uint64_t> exclude_newer_timestamp
) -> expected_t<solv::ObjRepoView>
{
LOG_INFO << "Reading repodata.json file " << filename << " for repo " << repo.name()
Expand Down Expand Up @@ -739,7 +765,8 @@ namespace mamba::solver::libsolv
default_subdir,
pkgs,
json_signatures,
ms_parser
ms_parser,
exclude_newer_timestamp
);
}
if (auto pkgs = repodata_doc["packages"]; !pkgs.error())
Expand All @@ -753,7 +780,8 @@ namespace mamba::solver::libsolv
pkgs,
json_signatures,
added,
ms_parser
ms_parser,
exclude_newer_timestamp
);
}
}
Expand All @@ -770,7 +798,8 @@ namespace mamba::solver::libsolv
default_subdir,
pkgs,
json_signatures,
ms_parser
ms_parser,
exclude_newer_timestamp
);
}

Expand All @@ -785,7 +814,8 @@ namespace mamba::solver::libsolv
default_subdir,
pkgs,
json_signatures,
ms_parser
ms_parser,
exclude_newer_timestamp
);
}
}
Expand Down
13 changes: 12 additions & 1 deletion libmamba/src/solver/libsolv/helpers.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#ifndef MAMBA_SOLVER_LIBSOLV_HELPERS
#define MAMBA_SOLVER_LIBSOLV_HELPERS

#include <cstdint>
#include <optional>
#include <string>
#include <string_view>
Expand Down Expand Up @@ -37,6 +38,15 @@ namespace mamba::fs

namespace mamba::solver::libsolv
{
// Beyond this value, the timestamp would be in milliseconds and therefore should be
// converted to seconds.
inline constexpr std::uint64_t MAX_CONDA_TIMESTAMP = 253402300799ULL;

[[nodiscard]] constexpr auto normalize_conda_timestamp(std::uint64_t timestamp) -> std::uint64_t
{
return (timestamp > MAX_CONDA_TIMESTAMP) ? (timestamp / 1000) : timestamp;
}

void set_solvable(
solv::ObjPool& pool,
solv::ObjSolvableView solv,
Expand All @@ -62,7 +72,8 @@ namespace mamba::solver::libsolv
const std::string& channel_id,
PackageTypes types,
MatchSpecParser parser,
bool verify_artifacts
bool verify_artifacts,
std::optional<std::uint64_t> exclude_newer_timestamp = std::nullopt
) -> expected_t<solv::ObjRepoView>;

[[nodiscard]] auto read_solv(
Expand Down
Loading
Loading