fix: Purge invalid hierarchical package cache before re-extract

Signed-off-by: Julien Jerphanion <git@jjerphan.xyz>

Author: jjerphan

libmamba/include/mamba/core/package_cache.hpp

@@ -108,6 +108,7 @@ namespace mamba
         Writable is_writable();
         fs::u8path path() const;
         void clear_query_cache(const specs::PackageInfo& s);
+        void remove_extracted_package(const specs::PackageInfo& s);
 
         bool has_valid_tarball(const specs::PackageInfo& s, const ValidationParams& params);
         bool has_valid_extracted_dir(const specs::PackageInfo& s, const ValidationParams& params);
@@ -138,6 +139,7 @@ namespace mamba
         std::vector<PackageCacheData*> writable_caches();
 
         void clear_query_cache(const specs::PackageInfo& s);
+        void remove_extracted_package(const specs::PackageInfo& s);
 
     private:
 

libmamba/src/core/package_cache.cpp

@@ -148,6 +148,15 @@ namespace mamba
         m_valid_extracted_dir.erase(s.long_str());
     }
 
+    void PackageCacheData::remove_extracted_package(const specs::PackageInfo& s)
+    {
+        const auto folder = package_cache_folder_relative_path(s);
+        const fs::u8path pkg_name = specs::strip_archive_extension(s.filename);
+        fs::remove_all(m_path / folder / pkg_name);
+        fs::remove_all(m_path / pkg_name);
+        clear_query_cache(s);
+    }
+
     void PackageCacheData::check_writable()
     {
         fs::u8path magic_file = m_path / PACKAGE_CACHE_MAGIC_FILE;
@@ -664,4 +673,15 @@ namespace mamba
             c.clear_query_cache(s);
         }
     }
+
+    void MultiPackageCache::remove_extracted_package(const specs::PackageInfo& s)
+    {
+        for (auto& c : m_caches)
+        {
+            c.remove_extracted_package(s);
+        }
+        const std::string pkg = s.long_str();
+        m_cached_tarballs.erase(pkg);
+        m_cached_extracted_dirs.erase(pkg);
+    }
 }  // namespace mamba

libmamba/src/core/package_fetcher.cpp

@@ -388,9 +388,23 @@ namespace mamba
 
     void PackageFetcher::clear_cache() const
     {
+        const auto remove_extracted_at = [&](const fs::u8path& cache_path)
+        { fs::remove_all(get_extract_path(filename(), cache_path)); };
+
         fs::remove_all(m_tarball_path);
-        const fs::u8path dest_dir = specs::strip_archive_extension(m_tarball_path.string());
-        fs::remove_all(dest_dir);
+        remove_extracted_at(m_cache_path);
+
+        // Tarballs may use the flat layout while extraction uses the hierarchical layout.
+        const fs::u8path tarball_parent = m_tarball_path.parent_path();
+        if (tarball_parent != m_cache_path)
+        {
+            remove_extracted_at(tarball_parent);
+        }
+
+        if (m_caches)
+        {
+            m_caches->clear_query_cache(m_package_info);
+        }
     }
 
     /*******************

libmamba/src/core/transaction.cpp

@@ -83,6 +83,10 @@ namespace mamba
                 return path;
             }
 
+            // Drop invalid or partial extracted directories so a cached tarball can be
+            // re-extracted (e.g. after a failed extraction left stale files behind).
+            caches.remove_extracted_package(pkg);
+
             PackageFetcher fetcher(pkg, caches);
             if (fetcher.needs_download())
             {

libmamba/tests/src/core/test_package_cache.cpp

@@ -199,5 +199,42 @@ namespace mamba
             MultiPackageCache cache({ pkgs_dir }, warn_params);
             REQUIRE(cache.get_extracted_dir_path(pkg_info).empty());
         }
+
+        // Non-regression for https://github.com/mamba-org/mamba/issues/4331#issuecomment-4771693736
+        SECTION("remove_extracted_package clears invalid hierarchical cache #4331")
+        {
+            auto warn_params = ctx.validation_params;
+            warn_params.safety_checks = VerificationLevel::Warn;
+
+            const fs::u8path invalid_hierarchical_dir = pkgs_dir / rel_path
+                                                        / "test-pkg-1.0.0-h123456_0";
+            fs::create_directories(invalid_hierarchical_dir / "info");
+            write_repodata_record(invalid_hierarchical_dir / "info" / "repodata_record.json", pkg_info);
+
+            nlohmann::json paths_json;
+            paths_json["paths_version"] = 1;
+            paths_json["paths"] = nlohmann::json::array(
+                {
+                    nlohmann::json{
+                        { "_path", "etc/conda/test-files/missing-file.txt" },
+                        { "path_type", "hardlink" },
+                        { "size_in_bytes", 42 },
+                    },
+                }
+            );
+            {
+                std::ofstream paths_out((invalid_hierarchical_dir / "info" / "paths.json").std_path());
+                paths_out << paths_json.dump();
+            }
+
+            MultiPackageCache cache({ pkgs_dir }, warn_params);
+            REQUIRE(cache.get_extracted_dir_path(pkg_info).empty());
+            REQUIRE(fs::exists(invalid_hierarchical_dir));
+
+            cache.remove_extracted_package(pkg_info);
+
+            REQUIRE_FALSE(fs::exists(invalid_hierarchical_dir));
+            REQUIRE(cache.get_extracted_dir_path(pkg_info).empty());
+        }
     }
 }  // namespace mamba

libmamba/tests/src/core/test_package_fetcher.cpp

@@ -15,6 +15,7 @@
 #include "mamba/core/package_handling.hpp"
 #include "mamba/core/util.hpp"
 #include "mamba/fs/filesystem.hpp"
+#include "mamba/specs/archive.hpp"
 #include "mamba/util/url_manip.hpp"
 
 #include "mambatests.hpp"
@@ -1300,4 +1301,79 @@ namespace
         REQUIRE(repodata_record.contains("size"));
         CHECK(repodata_record["size"] == tarball_size);
     }
+
+    // Non-regression for https://github.com/mamba-org/mamba/issues/4322 and
+    // https://github.com/mamba-org/mamba/issues/4331#issuecomment-4771693736
+    // When the tarball uses the flat layout but extraction targets the hierarchical layout,
+    // clear_cache() must remove the hierarchical extracted directory.
+    TEST_CASE("PackageFetcher::clear_cache removes hierarchical extracted directory", "[regression][4322][4331]")
+    {
+        auto& ctx = mambatests::context();
+        TemporaryDirectory temp_dir;
+        const fs::u8path pkgs_dir = temp_dir.path() / "pkgs";
+        fs::create_directories(pkgs_dir);
+        MultiPackageCache package_caches({ pkgs_dir }, ctx.validation_params);
+
+        specs::PackageInfo pkg;
+        pkg.name = "cuda-nvvp";
+        pkg.version = "12.9.79";
+        pkg.build_string = "he0c23c2_0";
+        pkg.platform = "win-64";
+        pkg.channel = "conda-forge";
+        pkg.package_url = "https://conda.anaconda.org/conda-forge/win-64/cuda-nvvp-12.9.79-he0c23c2_0.conda";
+        pkg.filename = "cuda-nvvp-12.9.79-he0c23c2_0.conda";
+
+        const auto cache_subdir = package_cache_folder_relative_path(pkg);
+        const std::string pkg_basename = std::string(specs::strip_archive_extension(pkg.filename));
+        const fs::u8path hierarchical_extract = pkgs_dir / cache_subdir / pkg_basename;
+        const fs::u8path flat_tarball = pkgs_dir / pkg.filename;
+
+        // Build a valid tarball at the flat cache location.
+        const fs::u8path build_dir = temp_dir.path() / "build";
+        const fs::u8path info_dir = build_dir / "info";
+        fs::create_directories(info_dir);
+        {
+            std::ofstream index_file((info_dir / "index.json").std_path());
+            index_file << R"({"name":"cuda-nvvp","version":"12.9.79","build":"he0c23c2_0"})";
+        }
+        {
+            std::ofstream paths_file((info_dir / "paths.json").std_path());
+            paths_file << R"({"paths": [], "paths_version": 1})";
+        }
+        create_archive(
+            build_dir,
+            flat_tarball,
+            compression_algorithm::bzip2,
+            /* compression_level= */ 1,
+            /* compression_threads= */ 1,
+            /* filter= */ nullptr
+        );
+        REQUIRE(fs::exists(flat_tarball));
+
+        // Simulate a stale partial extraction at the hierarchical location.
+        fs::create_directories(hierarchical_extract / "info");
+        {
+            auto out = open_ofstream(hierarchical_extract / "info" / "repodata_record.json");
+            out << R"({"name":"cuda-nvvp","version":"12.9.79","build":"he0c23c2_0","url":")"
+                << pkg.package_url << R"(","channel":")" << pkg.channel << R"("})";
+        }
+        {
+            std::ofstream paths_file((hierarchical_extract / "info" / "paths.json").std_path());
+            paths_file << R"({
+  "paths_version": 1,
+  "paths": [
+    {"_path": "Library/missing-file.txt", "path_type": "hardlink", "size_in_bytes": 42}
+  ]
+})";
+        }
+
+        PackageFetcher fetcher(pkg, package_caches);
+        REQUIRE(fetcher.needs_extract());
+
+        fetcher.clear_cache();
+
+        REQUIRE_FALSE(fs::exists(hierarchical_extract));
+        REQUIRE_FALSE(fs::exists(flat_tarball));
+        REQUIRE(package_caches.get_extracted_dir_path(pkg).empty());
+    }
 }