feat(bootstrap): parallelize RESOLVE and PREPARE_SOURCE I/O with background threads

.gitignore

@@ -220,6 +220,7 @@ __marimo__/
 /work-dir*/
 /build-logs/
 /e2e-output/
+/e2e-failed*
 /artifacts
 /overrides/
 

AGENTS.md

@@ -122,6 +122,20 @@ Closes: #123
 
 This produces a commit message with both the `Signed-off-by` trailer (from `--signoff`) and the `Co-Authored-By` trailer.
 
+## End-to-end tests
+
+There is a set of end-to-end tests in `./e2e`. Files named like
+`test_*.sh` are test scripts and files named like `ci_*.sh` are CI
+test suites that run the individual test scripts.
+
+The end-to-end tests are expensive so **ONLY** run them when asked.
+
+The CI test suites produce a lot of output. The most important
+information is in the summary at the end, where it shows which test
+scripts fail. Always run CI jobs in a way that only adds the last 25
+lines to the context window, then run an individual failing test
+script to debug the problem.
+
 ## Workflow for Complex Tasks
 
 1. **Search codebase** for similar patterns first

e2e/test_bootstrap_build_tags.sh

@@ -55,7 +55,7 @@ fromager \
   --settings-file="$SCRIPTDIR/bootstrap_settings.yaml" \
   bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0'
 
-if ! grep -q "stevedore: found built wheel on cache server" "$LOGFILE"; then
+if ! grep -q "stevedore-5.2.0: found built wheel on cache server" "$LOGFILE"; then
   echo "FAIL: Did not find log message found built wheel on cache server in $LOGFILE" 1>&2
   pass=false
 fi
@@ -77,7 +77,7 @@ fromager \
   --settings-file="$SCRIPTDIR/bootstrap_settings.yaml" \
   bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0'
 
-if ! grep -q "stevedore: found existing wheel " "$LOGFILE"; then
+if ! grep -q "stevedore-5.2.0: found existing wheel " "$LOGFILE"; then
   echo "FAIL: Did not find log message found existing wheel in $LOGFILE" 1>&2
   pass=false
 fi

e2e/test_bootstrap_cache.sh

@@ -42,23 +42,23 @@ fromager \
   bootstrap --cache-wheel-server-url="https://pypi.org/simple" "$DIST==$VER"
 
 EXPECTED_LOG_MESSAGES=(
-"$DIST: looking for existing wheel for version $VER with build tag () in"
-"$DIST: found existing wheel"
+"$DIST-$VER: looking for existing wheel for version $VER with build tag () in"
+"$DIST-$VER: found existing wheel"
 )
 for pattern in "${EXPECTED_LOG_MESSAGES[@]}"; do
-  if ! grep -q "$pattern" "$OUTDIR/bootstrap.log"; then
+  if ! grep -qF "$pattern" "$OUTDIR/bootstrap.log"; then
     echo "FAIL: Did not find log message $pattern in $OUTDIR/bootstrap.log" 1>&2
     pass=false
   fi
 done
 $pass
 
 UNEXPECTED_LOG_MESSAGES=(
-"$DIST: checking if wheel was already uploaded to https://pypi.org/simple"
+"$DIST-$VER: checking if wheel was already uploaded to https://pypi.org/simple"
 )
 
 for pattern in "${UNEXPECTED_LOG_MESSAGES[@]}"; do
-  if grep -q "$pattern" "$OUTDIR/bootstrap.log"; then
+  if grep -qF "$pattern" "$OUTDIR/bootstrap.log"; then
     echo "FAIL: Found log message $pattern in $OUTDIR/bootstrap.log" 1>&2
     pass=false
   fi
@@ -105,14 +105,14 @@ done
 $pass
 
 UNEXPECTED_LOG_MESSAGES=(
-"$DIST: loading build sdist dependencies from build-sdist-requirements.txt"
-"$DIST: loading build backend dependencies from build-backend-requirements.txt"
-"$DIST: loading build system dependencies from build-system-requirements.txt"
+"$DIST-$VER: loading build sdist dependencies from build-sdist-requirements.txt"
+"$DIST-$VER: loading build backend dependencies from build-backend-requirements.txt"
+"$DIST-$VER: loading build system dependencies from build-system-requirements.txt"
 )
 
 for pattern in "${UNEXPECTED_LOG_MESSAGES[@]}"; do
   echo $pattern
-  if grep -q "$pattern" "$OUTDIR/bootstrap.log"; then
+  if grep -qF "$pattern" "$OUTDIR/bootstrap.log"; then
     echo "FAIL: found log message $pattern in $OUTDIR/bootstrap.log" 1>&2
     pass=false
   fi

e2e/test_bootstrap_cooldown_prebuilt.sh

@@ -45,7 +45,7 @@ if ! grep -q "new toplevel dependency stevedore resolves to 5.3.0" "$OUTDIR/boot
 fi
 
 # The wheel must have been downloaded as a pre-built (not built from source).
-if ! grep -q "uses a pre-built wheel" "$OUTDIR/bootstrap.log"; then
+if ! grep -q "using pre-built wheel" "$OUTDIR/bootstrap.log"; then
   echo "FAIL: stevedore was not downloaded as a pre-built wheel" 1>&2
   pass=false
 fi

e2e/test_bootstrap_prerelease.sh

@@ -20,7 +20,7 @@ fromager \
 pass=true
 
 # Check for log message that the override is loaded
-if ! grep -q "flit_core: new toplevel dependency flit_core<2.0.1 resolves to 2.0" "$OUTDIR/bootstrap.log"; then
+if ! grep -q "flit_core-2.0: new toplevel dependency flit_core<2.0.1 resolves to 2.0" "$OUTDIR/bootstrap.log"; then
   echo "FAIL: flit_core did not resolve to 2.0 $OUTDIR/bootstrap.log" 1>&2
   pass=false
 fi
@@ -41,7 +41,7 @@ DEBUG_RESOLVER=true fromager \
 
 
 # Check for log message that the override is loaded
-if ! grep -q "flit_core: new toplevel dependency flit_core<2.0.1 resolves to 2.0rc3" "$OUTDIR/bootstrap.log"; then
+if ! grep -q "flit_core-2.0rc3: new toplevel dependency flit_core<2.0.1 resolves to 2.0rc3" "$OUTDIR/bootstrap.log"; then
   echo "FAIL: flit_core did not resolve to 2.0rc3 $OUTDIR/bootstrap.log" 1>&2
   pass=false
 fi

pyproject.toml

@@ -297,6 +297,7 @@ build = "sphinx-build -M html docs docs/_build -j auto --keep-going {args:--fail
 [tool.pytest.ini_options]
 # run tests in parallel with 3 pytest-xdist workers
 addopts = "-n 3"
+testpaths = ["tests"]
 markers = [
     "network: test that need network access",
 ]

src/fromager/bootstrap_requirement_resolver.py

@@ -118,6 +118,15 @@ def resolve(
             pbi = self.ctx.package_build_info(req)
             pre_built = pbi.pre_built
 
+        # Check session cache BEFORE the git URL guard so that background
+        # threads can retrieve pre-cached git URL resolutions (populated by
+        # Bootstrapper.resolve_versions() on the main thread before bootstrap()
+        # is called) without hitting the ValueError.
+        cached_result = self.get_cached_resolution(req, pre_built)
+        if cached_result is not None:
+            logger.debug(f"resolved {req} from cache")
+            return list(cached_result) if return_all_versions else [cached_result[0]]
+
         # Git URL source resolution must be handled by Bootstrapper.
         # But git URL prebuilt resolution is allowed - we look for wheels on PyPI
         # (test mode fallback uses this path).
@@ -146,6 +155,32 @@ def resolve(
             return matching
         return [matching[0]]
 
+    def get_cached_resolution(
+        self,
+        req: Requirement,
+        pre_built: bool,
+    ) -> list[tuple[str, Version]] | None:
+        """Return cached matching versions if this requirement was already resolved.
+
+        Returns ``None`` if the requirement has not been resolved yet, allowing
+        callers to distinguish between "no matching versions" and "not yet resolved".
+
+        Used by background threads to retrieve pre-cached git URL resolutions
+        populated by the main thread before entering the parallel section.
+
+        Args:
+            req: Package requirement
+            pre_built: Whether looking for prebuilt or source resolution
+
+        Returns:
+            List of (url, version) tuples if previously resolved, None otherwise.
+        """
+        rule_key = (str(req), pre_built)
+        with self._lock:
+            if rule_key in self._resolved_rules:
+                return self._get_matching_versions(req, pre_built)
+        return None
+
     def _resolve_and_extend(
         self,
         req: Requirement,