diff --git a/e2e/ci_bootstrap_suite.sh b/e2e/ci_bootstrap_suite.sh index 3d4db445..217d361c 100755 --- a/e2e/ci_bootstrap_suite.sh +++ b/e2e/ci_bootstrap_suite.sh @@ -26,6 +26,12 @@ run_test "bootstrap_prerelease" run_test "bootstrap_cache" run_test "bootstrap_sdist_only" +test_section "bootstrap test-mode tests" +run_test "mode_resolution" +run_test "mode_deps" +run_test "mode_build" +run_test "mode_fallback" + test_section "bootstrap git URL tests" run_test "bootstrap_git_url" run_test "bootstrap_git_url_tag" diff --git a/e2e/test_build_failure/pyproject.toml b/e2e/test_build_failure/pyproject.toml new file mode 100644 index 00000000..9335b99d --- /dev/null +++ b/e2e/test_build_failure/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "test_build_failure" +version = "1.0.0" +description = "Test fixture that intentionally fails to build" diff --git a/e2e/test_build_failure/setup.py b/e2e/test_build_failure/setup.py new file mode 100644 index 00000000..d13a3a60 --- /dev/null +++ b/e2e/test_build_failure/setup.py @@ -0,0 +1,23 @@ +# mypy: ignore-errors +"""Setup script that intentionally fails during wheel build. + +This fixture is designed to pass metadata extraction but fail during +actual wheel building, producing a 'bootstrap' failure in test-mode. +The failure is triggered by a custom build_ext command that always fails. +""" + +from setuptools import Extension, setup +from setuptools.command.build_ext import build_ext + + +class FailingBuildExt(build_ext): + """Custom build_ext that always fails.""" + + def run(self) -> None: + raise RuntimeError("Intentional build failure for e2e testing") + + +setup( + ext_modules=[Extension("test_build_failure._dummy", sources=["missing.c"])], + cmdclass={"build_ext": FailingBuildExt}, +) diff --git a/e2e/test_build_failure/test_build_failure/__init__.py b/e2e/test_build_failure/test_build_failure/__init__.py new file mode 100644 index 00000000..16707c10 --- /dev/null +++ b/e2e/test_build_failure/test_build_failure/__init__.py @@ -0,0 +1 @@ +"""Test fixture package for e2e build failure tests.""" diff --git a/e2e/test_mode_build.sh b/e2e/test_mode_build.sh new file mode 100755 index 00000000..5a379e4a --- /dev/null +++ b/e2e/test_mode_build.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# Test --test-mode: build failure without prebuilt fallback +# +# Verifies that when a package fails to build and no prebuilt wheel is available +# (because the package is not on PyPI), test-mode records the failure. +# Uses a local git repo fixture with a broken build backend. +# +# See: https://github.com/python-wheel-build/fromager/issues/895 + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/common.sh" + +# Use the test_build_failure fixture (local git repo) +# Initialize git repo at runtime (fixture files are committed without .git) +FIXTURE_DIR="$SCRIPTDIR/test_build_failure" +CREATED_FIXTURE_GIT=false +if [ ! -d "$FIXTURE_DIR/.git" ]; then + CREATED_FIXTURE_GIT=true + (cd "$FIXTURE_DIR" && git init -q && \ + git config user.email "test@example.com" && \ + git config user.name "Test User" && \ + git add -A && git commit -q -m "init") +fi +FIXTURE_URL="git+file://${FIXTURE_DIR}" + +# Cleanup .git on exit if we created it (prevents flaky reruns) +cleanup_fixture_git() { + if [ "$CREATED_FIXTURE_GIT" = true ] && [ -d "$FIXTURE_DIR/.git" ]; then + rm -rf "$FIXTURE_DIR/.git" + fi +} +trap cleanup_fixture_git EXIT + +# Create a requirements file pointing to the local fixture +REQUIREMENTS_FILE="$OUTDIR/test-requirements.txt" +echo "test_build_failure @ ${FIXTURE_URL}" > "$REQUIREMENTS_FILE" + +# Run bootstrap in test mode +# - Package resolves from local git repo +# - Build fails (broken build backend) +# - Prebuilt fallback fails (package not on PyPI) +# - Failure should be recorded +set +e +fromager \ + --log-file="$OUTDIR/bootstrap.log" \ + --error-log-file="$OUTDIR/fromager-errors.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + bootstrap --test-mode -r "$REQUIREMENTS_FILE" +EXIT_CODE=$? +set -e + +pass=true + +# Check 1: Exit code should be 1 (failures recorded) +if [ "$EXIT_CODE" -ne 1 ]; then + echo "FAIL: Expected exit code 1, got $EXIT_CODE" 1>&2 + pass=false +fi + +# Check 2: The test-mode-failures JSON file should exist +FAILURES_FILE=$(find "$OUTDIR/work-dir" -name "test-mode-failures-*.json" 2>/dev/null | head -1) +if [ -z "$FAILURES_FILE" ] || [ ! -f "$FAILURES_FILE" ]; then + echo "FAIL: test-mode-failures-*.json file not found" 1>&2 + pass=false +else + echo "Found failures file: $FAILURES_FILE" + + # Check 3: test_build_failure should be in failed packages + # Note: package name uses underscore as recorded by fromager + if ! jq -e '.failures[] | select(.package == "test_build_failure")' "$FAILURES_FILE" > /dev/null 2>&1; then + echo "FAIL: Expected 'test_build_failure' in failed packages" 1>&2 + jq '.' "$FAILURES_FILE" 1>&2 + pass=false + fi + + # Check 4: failure_type MUST be 'bootstrap' (actual build failure, not resolution) + # Pinning to 'bootstrap' catches regressions if fromager misclassifies failures + FAILURE_TYPE=$(jq -r '[.failures[] | select(.package == "test_build_failure")][0].failure_type' "$FAILURES_FILE") + if [ "$FAILURE_TYPE" != "bootstrap" ]; then + echo "FAIL: Expected failure_type 'bootstrap', got '$FAILURE_TYPE'" 1>&2 + pass=false + else + echo "Failure type: $FAILURE_TYPE" + fi + + # Check 5: exception_message should indicate a build-related error + EXCEPTION_MSG=$(jq -r '[.failures[] | select(.package == "test_build_failure")][0].exception_message' "$FAILURES_FILE") + if [[ "$EXCEPTION_MSG" != *"nonexistent_file"* ]] && [[ "$EXCEPTION_MSG" != *"MANIFEST"* ]] && [[ "$EXCEPTION_MSG" != *"build"* ]] && [[ "$EXCEPTION_MSG" != *"CalledProcessError"* ]]; then + echo "FAIL: Expected exception message about build failure, got: $EXCEPTION_MSG" 1>&2 + pass=false + fi +fi + +# Check 6: Log should show test mode enabled +if ! grep -q "test mode enabled" "$OUTDIR/bootstrap.log"; then + echo "FAIL: Log should contain 'test mode enabled'" 1>&2 + pass=false +fi + +# Check 7: Log may show fallback attempt (depends on where failure occurs) +# Note: Fallback is only attempted when build fails after resolution succeeds. +# Our fixture fails during metadata extraction, so fallback may not be triggered. +if grep -q "pre-built fallback" "$OUTDIR/bootstrap.log"; then + echo "INFO: Fallback was attempted (package not on PyPI, so it failed)" +else + echo "INFO: No fallback attempt (failure occurred before build phase)" +fi + +$pass diff --git a/e2e/test_mode_deps.sh b/e2e/test_mode_deps.sh new file mode 100755 index 00000000..2afab910 --- /dev/null +++ b/e2e/test_mode_deps.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# Test --test-mode: secondary dependency resolution failure +# +# Verifies that when a top-level package resolves but one of its dependencies +# cannot be resolved, test-mode records the failure and continues processing. +# +# See: https://github.com/python-wheel-build/fromager/issues/895 + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/common.sh" + +# Use stevedore which depends on pbr +# Constrain pbr to a version that doesn't exist to trigger secondary dep failure +TOPLEVEL_PKG="stevedore==5.2.0" +NONEXISTENT_VERSION="99999.0.0" + +# Create a constraints file that forces pbr to a non-existent version +CONSTRAINTS_FILE="$OUTDIR/test-constraints.txt" +echo "pbr==${NONEXISTENT_VERSION}" > "$CONSTRAINTS_FILE" + +# Run bootstrap in test mode +# The top-level stevedore should resolve, but pbr should fail +set +e +fromager \ + --log-file="$OUTDIR/bootstrap.log" \ + --error-log-file="$OUTDIR/fromager-errors.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + --constraints-file="$CONSTRAINTS_FILE" \ + bootstrap --test-mode "${TOPLEVEL_PKG}" +EXIT_CODE=$? +set -e + +pass=true + +# Check 1: Exit code should be 1 (indicating failures in test mode) +if [ "$EXIT_CODE" -ne 1 ]; then + echo "FAIL: Expected exit code 1, got $EXIT_CODE" 1>&2 + pass=false +fi + +# Check 2: The test-mode-failures JSON file should exist +FAILURES_FILE=$(find "$OUTDIR/work-dir" -name "test-mode-failures-*.json" 2>/dev/null | head -1) +if [ -z "$FAILURES_FILE" ] || [ ! -f "$FAILURES_FILE" ]; then + echo "FAIL: test-mode-failures-*.json file not found in $OUTDIR/work-dir" 1>&2 + ls -la "$OUTDIR/work-dir" 1>&2 + pass=false +else + echo "Found failures file: $FAILURES_FILE" + + # Check 3: JSON file should contain at least one failure + FAILURE_COUNT=$(jq '.failures | length' "$FAILURES_FILE") + if [ "$FAILURE_COUNT" -lt 1 ]; then + echo "FAIL: Expected at least 1 failure in JSON, got $FAILURE_COUNT" 1>&2 + jq '.' "$FAILURES_FILE" 1>&2 + pass=false + fi + + # Check 4: pbr should be in the failed packages (secondary dependency) + if ! jq -e '.failures[] | select(.package == "pbr")' "$FAILURES_FILE" > /dev/null 2>&1; then + echo "FAIL: Expected 'pbr' to be in failed packages" 1>&2 + jq '.' "$FAILURES_FILE" 1>&2 + pass=false + fi + + # Check 5: All pbr failures should be "resolution" type + # Use first match since pbr may fail multiple times (as build dep of multiple packages) + PBR_FAILURE_TYPE=$(jq -r '[.failures[] | select(.package == "pbr")][0].failure_type' "$FAILURES_FILE") + if [ "$PBR_FAILURE_TYPE" != "resolution" ]; then + echo "FAIL: Expected failure_type 'resolution' for pbr, got '$PBR_FAILURE_TYPE'" 1>&2 + jq '.' "$FAILURES_FILE" 1>&2 + pass=false + fi +fi + +# Check 6: Log should contain test mode messages +if ! grep -q "test mode enabled" "$OUTDIR/bootstrap.log"; then + echo "FAIL: Log should contain 'test mode enabled' message" 1>&2 + pass=false +fi + +# Check 7: stevedore should have been resolved (top-level success) +if ! grep -q "stevedore.*resolves to" "$OUTDIR/bootstrap.log"; then + echo "FAIL: stevedore should have been resolved" 1>&2 + pass=false +fi + +$pass diff --git a/e2e/test_mode_fallback.sh b/e2e/test_mode_fallback.sh new file mode 100755 index 00000000..4d8bc914 --- /dev/null +++ b/e2e/test_mode_fallback.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# Test --test-mode: build failure with prebuilt fallback +# +# Verifies that when a source build fails but a prebuilt wheel is available, +# test-mode uses the prebuilt wheel as fallback and continues without failure. +# Uses a broken patch to trigger the build failure, then falls back to PyPI wheel. +# +# See: https://github.com/python-wheel-build/fromager/issues/895 + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/common.sh" + +# Use setuptools - it's on PyPI with prebuilt wheels +DIST="setuptools" +VERSION="75.8.0" + +# Step 1: Configure settings to mark setuptools as NOT prebuilt +# This forces fromager to try building from source +SETTINGS_DIR="$OUTDIR/test-settings" +mkdir -p "$SETTINGS_DIR" +cat > "$SETTINGS_DIR/${DIST}.yaml" << EOF +variants: + cpu: + pre_built: false +EOF + +# Step 2: Create a broken patches dir that will cause build to fail +# We create a patch targeting setup.py with wrong content - patch will fail +# without prompting for input (unlike targeting a non-existent file) +PATCHES_DIR="$OUTDIR/test-patches" +mkdir -p "$PATCHES_DIR/${DIST}" +cat > "$PATCHES_DIR/${DIST}/break-build.patch" << 'PATCHEOF' +--- a/setup.py ++++ b/setup.py +@@ -1,3 +1,3 @@ +-this content does not match +-the actual setup.py file +-so patch will fail ++replaced content ++that will never ++be applied +PATCHEOF + +# Step 3: Run bootstrap in test mode +# - Package will resolve from PyPI +# - Source preparation will fail (bad patch) +# - Prebuilt fallback should succeed (wheel on PyPI) +echo "Running test-mode bootstrap with broken patch..." +set +e +fromager \ + --log-file="$OUTDIR/bootstrap.log" \ + --error-log-file="$OUTDIR/fromager-errors.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + --settings-dir="$SETTINGS_DIR" \ + --patches-dir="$PATCHES_DIR" \ + bootstrap --test-mode "${DIST}==${VERSION}" +EXIT_CODE=$? +set -e + +pass=true + +# Check 1: Exit code should be 0 (fallback succeeded, no failures recorded) +echo "Exit code: $EXIT_CODE" +if [ "$EXIT_CODE" -ne 0 ]; then + echo "FAIL: Expected exit code 0 (fallback success), got $EXIT_CODE" 1>&2 + pass=false +fi + +# Check 2: Log should show test mode was enabled +if ! grep -q "test mode enabled" "$OUTDIR/bootstrap.log"; then + echo "FAIL: Log should contain 'test mode enabled' message" 1>&2 + pass=false +fi + +# Check 3: Patch application must be attempted +if ! grep -q "applying patch file.*break-build.patch" "$OUTDIR/bootstrap.log"; then + echo "FAIL: Expected patch 'break-build.patch' to be applied" 1>&2 + pass=false +else + echo "Patch application was attempted" +fi + +# Check 4: Prebuilt fallback MUST be triggered and succeed +if ! grep -q "pre-built fallback" "$OUTDIR/bootstrap.log"; then + echo "FAIL: Expected prebuilt fallback to be triggered" 1>&2 + pass=false +elif ! grep -q "successfully used pre-built wheel" "$OUTDIR/bootstrap.log"; then + echo "FAIL: Prebuilt fallback was triggered but did not succeed" 1>&2 + pass=false +else + echo "SUCCESS: Prebuilt fallback triggered and succeeded" +fi + +# Check 5: No failures should be recorded (fallback succeeded) +FAILURES_FILE=$(find "$OUTDIR/work-dir" -name "test-mode-failures-*.json" 2>/dev/null | head -1) +if [ -n "$FAILURES_FILE" ] && [ -f "$FAILURES_FILE" ]; then + FAILURE_COUNT=$(jq '.failures | length' "$FAILURES_FILE") + if [ "$FAILURE_COUNT" -gt 0 ]; then + echo "FAIL: Expected no failures (fallback should succeed), got $FAILURE_COUNT" 1>&2 + jq '.failures[] | {package, failure_type, exception_type}' "$FAILURES_FILE" 1>&2 + pass=false + fi +fi + +# Check 6: Verify test mode completed +if grep -q "test mode:" "$OUTDIR/bootstrap.log"; then + echo "Test mode processing completed" +else + echo "NOTE: Test mode summary not found in log" 1>&2 +fi + +$pass diff --git a/e2e/test_mode_resolution.sh b/e2e/test_mode_resolution.sh new file mode 100755 index 00000000..a45e5315 --- /dev/null +++ b/e2e/test_mode_resolution.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# Test --test-mode: top-level resolution failure +# +# Verifies that when a top-level package cannot be resolved (doesn't exist), +# test-mode records the failure in JSON and exits non-zero instead of crashing. +# +# See: https://github.com/python-wheel-build/fromager/issues/895 + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/common.sh" + +# Use a package name that definitely does not exist on PyPI +# This will trigger a resolution failure +NONEXISTENT_PKG="nonexistent-package-xyz-12345-does-not-exist" + +# Run bootstrap in test mode with the non-existent package +# We expect this to exit with code 1 (failures recorded) +set +e +fromager \ + --log-file="$OUTDIR/bootstrap.log" \ + --error-log-file="$OUTDIR/fromager-errors.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + bootstrap --test-mode "${NONEXISTENT_PKG}==1.0.0" +EXIT_CODE=$? +set -e + +pass=true + +# Check 1: Exit code should be 1 (indicating failures in test mode) +if [ "$EXIT_CODE" -ne 1 ]; then + echo "FAIL: Expected exit code 1, got $EXIT_CODE" 1>&2 + pass=false +fi + +# Check 2: The test-mode-failures JSON file should exist +FAILURES_FILE=$(find "$OUTDIR/work-dir" -name "test-mode-failures-*.json" 2>/dev/null | head -1) +if [ -z "$FAILURES_FILE" ] || [ ! -f "$FAILURES_FILE" ]; then + echo "FAIL: test-mode-failures-*.json file not found in $OUTDIR/work-dir" 1>&2 + ls -la "$OUTDIR/work-dir" 1>&2 + pass=false +else + echo "Found failures file: $FAILURES_FILE" + + # Check 3: JSON file should contain failures array with at least one entry + FAILURE_COUNT=$(jq '.failures | length' "$FAILURES_FILE") + if [ "$FAILURE_COUNT" -lt 1 ]; then + echo "FAIL: Expected at least 1 failure in JSON, got $FAILURE_COUNT" 1>&2 + jq '.' "$FAILURES_FILE" 1>&2 + pass=false + fi + + # Check 4: The failure should be for our non-existent package + FAILED_PKG=$(jq -r '.failures[0].package' "$FAILURES_FILE") + if [ "$FAILED_PKG" != "$NONEXISTENT_PKG" ]; then + echo "FAIL: Expected failed package '$NONEXISTENT_PKG', got '$FAILED_PKG'" 1>&2 + jq '.' "$FAILURES_FILE" 1>&2 + pass=false + fi + + # Check 5: The failure_type should be "resolution" (package doesn't exist) + FAILURE_TYPE=$(jq -r '.failures[0].failure_type' "$FAILURES_FILE") + if [ "$FAILURE_TYPE" != "resolution" ]; then + echo "FAIL: Expected failure_type 'resolution', got '$FAILURE_TYPE'" 1>&2 + jq '.' "$FAILURES_FILE" 1>&2 + pass=false + fi + + # Check 6: The failure should have an exception_type + EXCEPTION_TYPE=$(jq -r '.failures[0].exception_type' "$FAILURES_FILE") + if [ -z "$EXCEPTION_TYPE" ] || [ "$EXCEPTION_TYPE" = "null" ]; then + echo "FAIL: Expected non-empty exception_type" 1>&2 + jq '.' "$FAILURES_FILE" 1>&2 + pass=false + fi +fi + +# Check 7: Log should contain test mode messages +if ! grep -q "test mode enabled" "$OUTDIR/bootstrap.log"; then + echo "FAIL: Log should contain 'test mode enabled' message" 1>&2 + pass=false +fi + +if ! grep -q "test mode:.*failed" "$OUTDIR/bootstrap.log"; then + echo "FAIL: Log should contain test mode failure message" 1>&2 + pass=false +fi + +$pass