test(e2e): add test-mode failure handling tests

e2e/ci_bootstrap_suite.sh

@@ -26,6 +26,12 @@ run_test "bootstrap_prerelease"
 run_test "bootstrap_cache"
 run_test "bootstrap_sdist_only"
 
+test_section "bootstrap test-mode tests"
+run_test "mode_resolution"
+run_test "mode_deps"
+run_test "mode_build"
+run_test "mode_fallback"
+
 test_section "bootstrap git URL tests"
 run_test "bootstrap_git_url"
 run_test "bootstrap_git_url_tag"

e2e/test_build_failure/pyproject.toml

@@ -0,0 +1,8 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "test_build_failure"
+version = "1.0.0"
+description = "Test fixture that intentionally fails to build"

e2e/test_build_failure/setup.py

@@ -0,0 +1,23 @@
+# mypy: ignore-errors
+"""Setup script that intentionally fails during wheel build.
+
+This fixture is designed to pass metadata extraction but fail during
+actual wheel building, producing a 'bootstrap' failure in test-mode.
+The failure is triggered by a custom build_ext command that always fails.
+"""
+
+from setuptools import Extension, setup
+from setuptools.command.build_ext import build_ext
+
+
+class FailingBuildExt(build_ext):
+    """Custom build_ext that always fails."""
+
+    def run(self) -> None:
+        raise RuntimeError("Intentional build failure for e2e testing")
+
+
+setup(
+    ext_modules=[Extension("test_build_failure._dummy", sources=["missing.c"])],
+    cmdclass={"build_ext": FailingBuildExt},
+)

e2e/test_build_failure/test_build_failure/__init__.py

@@ -0,0 +1 @@
+"""Test fixture package for e2e build failure tests."""

e2e/test_mode_build.sh

@@ -0,0 +1,113 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# Test --test-mode: build failure without prebuilt fallback
+#
+# Verifies that when a package fails to build and no prebuilt wheel is available
+# (because the package is not on PyPI), test-mode records the failure.
+# Uses a local git repo fixture with a broken build backend.
+#
+# See: https://github.com/python-wheel-build/fromager/issues/895
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+
+# Use the test_build_failure fixture (local git repo)
+# Initialize git repo at runtime (fixture files are committed without .git)
+FIXTURE_DIR="$SCRIPTDIR/test_build_failure"
+CREATED_FIXTURE_GIT=false
+if [ ! -d "$FIXTURE_DIR/.git" ]; then
+  CREATED_FIXTURE_GIT=true
+  (cd "$FIXTURE_DIR" && git init -q && \
+   git config user.email "test@example.com" && \
+   git config user.name "Test User" && \
+   git add -A && git commit -q -m "init")
+fi
+FIXTURE_URL="git+file://${FIXTURE_DIR}"
+
+# Cleanup .git on exit if we created it (prevents flaky reruns)
+cleanup_fixture_git() {
+  if [ "$CREATED_FIXTURE_GIT" = true ] && [ -d "$FIXTURE_DIR/.git" ]; then
+    rm -rf "$FIXTURE_DIR/.git"
+  fi
+}
+trap cleanup_fixture_git EXIT
+
+# Create a requirements file pointing to the local fixture
+REQUIREMENTS_FILE="$OUTDIR/test-requirements.txt"
+echo "test_build_failure @ ${FIXTURE_URL}" > "$REQUIREMENTS_FILE"
+
+# Run bootstrap in test mode
+# - Package resolves from local git repo
+# - Build fails (broken build backend)
+# - Prebuilt fallback fails (package not on PyPI)
+# - Failure should be recorded
+set +e
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  bootstrap --test-mode -r "$REQUIREMENTS_FILE"
+EXIT_CODE=$?
+set -e
+
+pass=true
+
+# Check 1: Exit code should be 1 (failures recorded)
+if [ "$EXIT_CODE" -ne 1 ]; then
+  echo "FAIL: Expected exit code 1, got $EXIT_CODE" 1>&2
+  pass=false
+fi
+
+# Check 2: The test-mode-failures JSON file should exist
+FAILURES_FILE=$(find "$OUTDIR/work-dir" -name "test-mode-failures-*.json" 2>/dev/null | head -1)
+if [ -z "$FAILURES_FILE" ] || [ ! -f "$FAILURES_FILE" ]; then
+  echo "FAIL: test-mode-failures-*.json file not found" 1>&2
+  pass=false
+else
+  echo "Found failures file: $FAILURES_FILE"
+
+  # Check 3: test_build_failure should be in failed packages
+  # Note: package name uses underscore as recorded by fromager
+  if ! jq -e '.failures[] | select(.package == "test_build_failure")' "$FAILURES_FILE" > /dev/null 2>&1; then
+    echo "FAIL: Expected 'test_build_failure' in failed packages" 1>&2
+    jq '.' "$FAILURES_FILE" 1>&2
+    pass=false
+  fi
+
+  # Check 4: failure_type MUST be 'bootstrap' (actual build failure, not resolution)
+  # Pinning to 'bootstrap' catches regressions if fromager misclassifies failures
+  FAILURE_TYPE=$(jq -r '[.failures[] | select(.package == "test_build_failure")][0].failure_type' "$FAILURES_FILE")
+  if [ "$FAILURE_TYPE" != "bootstrap" ]; then
+    echo "FAIL: Expected failure_type 'bootstrap', got '$FAILURE_TYPE'" 1>&2
+    pass=false
+  else
+    echo "Failure type: $FAILURE_TYPE"
+  fi
+
+  # Check 5: exception_message should indicate a build-related error
+  EXCEPTION_MSG=$(jq -r '[.failures[] | select(.package == "test_build_failure")][0].exception_message' "$FAILURES_FILE")
+  if [[ "$EXCEPTION_MSG" != *"nonexistent_file"* ]] && [[ "$EXCEPTION_MSG" != *"MANIFEST"* ]] && [[ "$EXCEPTION_MSG" != *"build"* ]] && [[ "$EXCEPTION_MSG" != *"CalledProcessError"* ]]; then
+    echo "FAIL: Expected exception message about build failure, got: $EXCEPTION_MSG" 1>&2
+    pass=false
+  fi
+fi
+
+# Check 6: Log should show test mode enabled
+if ! grep -q "test mode enabled" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: Log should contain 'test mode enabled'" 1>&2
+  pass=false
+fi
+
+# Check 7: Log may show fallback attempt (depends on where failure occurs)
+# Note: Fallback is only attempted when build fails after resolution succeeds.
+# Our fixture fails during metadata extraction, so fallback may not be triggered.
+if grep -q "pre-built fallback" "$OUTDIR/bootstrap.log"; then
+  echo "INFO: Fallback was attempted (package not on PyPI, so it failed)"
+else
+  echo "INFO: No fallback attempt (failure occurred before build phase)"
+fi
+
+$pass

e2e/test_mode_deps.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# Test --test-mode: secondary dependency resolution failure
+#
+# Verifies that when a top-level package resolves but one of its dependencies
+# cannot be resolved, test-mode records the failure and continues processing.
+#
+# See: https://github.com/python-wheel-build/fromager/issues/895
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+
+# Use stevedore which depends on pbr
+# Constrain pbr to a version that doesn't exist to trigger secondary dep failure
+TOPLEVEL_PKG="stevedore==5.2.0"
+NONEXISTENT_VERSION="99999.0.0"
+
+# Create a constraints file that forces pbr to a non-existent version
+CONSTRAINTS_FILE="$OUTDIR/test-constraints.txt"
+echo "pbr==${NONEXISTENT_VERSION}" > "$CONSTRAINTS_FILE"
+
+# Run bootstrap in test mode
+# The top-level stevedore should resolve, but pbr should fail
+set +e
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --constraints-file="$CONSTRAINTS_FILE" \
+  bootstrap --test-mode "${TOPLEVEL_PKG}"
+EXIT_CODE=$?
+set -e
+
+pass=true
+
+# Check 1: Exit code should be 1 (indicating failures in test mode)
+if [ "$EXIT_CODE" -ne 1 ]; then
+  echo "FAIL: Expected exit code 1, got $EXIT_CODE" 1>&2
+  pass=false
+fi
+
+# Check 2: The test-mode-failures JSON file should exist
+FAILURES_FILE=$(find "$OUTDIR/work-dir" -name "test-mode-failures-*.json" 2>/dev/null | head -1)
+if [ -z "$FAILURES_FILE" ] || [ ! -f "$FAILURES_FILE" ]; then
+  echo "FAIL: test-mode-failures-*.json file not found in $OUTDIR/work-dir" 1>&2
+  ls -la "$OUTDIR/work-dir" 1>&2
+  pass=false
+else
+  echo "Found failures file: $FAILURES_FILE"
+
+  # Check 3: JSON file should contain at least one failure
+  FAILURE_COUNT=$(jq '.failures | length' "$FAILURES_FILE")
+  if [ "$FAILURE_COUNT" -lt 1 ]; then
+    echo "FAIL: Expected at least 1 failure in JSON, got $FAILURE_COUNT" 1>&2
+    jq '.' "$FAILURES_FILE" 1>&2
+    pass=false
+  fi
+
+  # Check 4: pbr should be in the failed packages (secondary dependency)
+  if ! jq -e '.failures[] | select(.package == "pbr")' "$FAILURES_FILE" > /dev/null 2>&1; then
+    echo "FAIL: Expected 'pbr' to be in failed packages" 1>&2
+    jq '.' "$FAILURES_FILE" 1>&2
+    pass=false
+  fi
+
+  # Check 5: All pbr failures should be "resolution" type
+  # Use first match since pbr may fail multiple times (as build dep of multiple packages)
+  PBR_FAILURE_TYPE=$(jq -r '[.failures[] | select(.package == "pbr")][0].failure_type' "$FAILURES_FILE")
+  if [ "$PBR_FAILURE_TYPE" != "resolution" ]; then
+    echo "FAIL: Expected failure_type 'resolution' for pbr, got '$PBR_FAILURE_TYPE'" 1>&2
+    jq '.' "$FAILURES_FILE" 1>&2
+    pass=false
+  fi
+fi
+
+# Check 6: Log should contain test mode messages
+if ! grep -q "test mode enabled" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: Log should contain 'test mode enabled' message" 1>&2
+  pass=false
+fi
+
+# Check 7: stevedore should have been resolved (top-level success)
+if ! grep -q "stevedore.*resolves to" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: stevedore should have been resolved" 1>&2
+  pass=false
+fi
+
+$pass

e2e/test_mode_fallback.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# Test --test-mode: build failure with prebuilt fallback
+#
+# Verifies that when a source build fails but a prebuilt wheel is available,
+# test-mode uses the prebuilt wheel as fallback and continues without failure.
+# Uses a broken patch to trigger the build failure, then falls back to PyPI wheel.
+#
+# See: https://github.com/python-wheel-build/fromager/issues/895
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+
+# Use setuptools - it's on PyPI with prebuilt wheels
+DIST="setuptools"
+VERSION="75.8.0"
+
+# Step 1: Configure settings to mark setuptools as NOT prebuilt
+# This forces fromager to try building from source
+SETTINGS_DIR="$OUTDIR/test-settings"
+mkdir -p "$SETTINGS_DIR"
+cat > "$SETTINGS_DIR/${DIST}.yaml" << EOF
+variants:
+  cpu:
+    pre_built: false
+EOF
+
+# Step 2: Create a broken patches dir that will cause build to fail
+# We create a patch targeting setup.py with wrong content - patch will fail
+# without prompting for input (unlike targeting a non-existent file)
+PATCHES_DIR="$OUTDIR/test-patches"
+mkdir -p "$PATCHES_DIR/${DIST}"
+cat > "$PATCHES_DIR/${DIST}/break-build.patch" << 'PATCHEOF'
+--- a/setup.py
++++ b/setup.py
+@@ -1,3 +1,3 @@
+-this content does not match
+-the actual setup.py file
+-so patch will fail
++replaced content
++that will never
++be applied
+PATCHEOF
+
+# Step 3: Run bootstrap in test mode
+# - Package will resolve from PyPI
+# - Source preparation will fail (bad patch)
+# - Prebuilt fallback should succeed (wheel on PyPI)
+echo "Running test-mode bootstrap with broken patch..."
+set +e
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --settings-dir="$SETTINGS_DIR" \
+  --patches-dir="$PATCHES_DIR" \
+  bootstrap --test-mode "${DIST}==${VERSION}"
+EXIT_CODE=$?
+set -e
+
+pass=true
+
+# Check 1: Exit code should be 0 (fallback succeeded, no failures recorded)
+echo "Exit code: $EXIT_CODE"
+if [ "$EXIT_CODE" -ne 0 ]; then
+  echo "FAIL: Expected exit code 0 (fallback success), got $EXIT_CODE" 1>&2
+  pass=false
+fi
+
+# Check 2: Log should show test mode was enabled
+if ! grep -q "test mode enabled" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: Log should contain 'test mode enabled' message" 1>&2
+  pass=false
+fi
+
+# Check 3: Patch application must be attempted
+if ! grep -q "applying patch file.*break-build.patch" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: Expected patch 'break-build.patch' to be applied" 1>&2
+  pass=false
+else
+  echo "Patch application was attempted"
+fi
+
+# Check 4: Prebuilt fallback MUST be triggered and succeed
+if ! grep -q "pre-built fallback" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: Expected prebuilt fallback to be triggered" 1>&2
+  pass=false
+elif ! grep -q "successfully used pre-built wheel" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: Prebuilt fallback was triggered but did not succeed" 1>&2
+  pass=false
+else
+  echo "SUCCESS: Prebuilt fallback triggered and succeeded"
+fi
+
+# Check 5: No failures should be recorded (fallback succeeded)
+FAILURES_FILE=$(find "$OUTDIR/work-dir" -name "test-mode-failures-*.json" 2>/dev/null | head -1)
+if [ -n "$FAILURES_FILE" ] && [ -f "$FAILURES_FILE" ]; then
+  FAILURE_COUNT=$(jq '.failures | length' "$FAILURES_FILE")
+  if [ "$FAILURE_COUNT" -gt 0 ]; then
+    echo "FAIL: Expected no failures (fallback should succeed), got $FAILURE_COUNT" 1>&2
+    jq '.failures[] | {package, failure_type, exception_type}' "$FAILURES_FILE" 1>&2
+    pass=false
+  fi
+fi
+
+# Check 6: Verify test mode completed
+if grep -q "test mode:" "$OUTDIR/bootstrap.log"; then
+  echo "Test mode processing completed"
+else
+  echo "NOTE: Test mode summary not found in log" 1>&2
+fi
+
+$pass