From 72aad142fd83fd61919926d640886d38f86bfbc9 Mon Sep 17 00:00:00 2001 From: Rohan Devasthale Date: Tue, 7 Apr 2026 15:23:01 -0400 Subject: [PATCH] feat(bootstrap): add --multiple-versions flag to bootstrap all matching versions Add --multiple-versions flag to bootstrap command that enables bootstrapping all versions matching a requirement specification, rather than only the highest version. This is useful for creating comprehensive build environments with multiple versions of the same package. Key changes: - Add return_all_versions parameter to BootstrapRequirementResolver.resolve() with type-safe overloads using @overload decorators - Modify Bootstrapper.bootstrap() to iterate over all resolved versions when --multiple-versions flag is enabled - Implement continue-on-error behavior: failed versions are logged, tracked, and reported at the end without stopping the bootstrap process - Add DependencyGraph.remove_dependency() to clean up failed nodes from graph - Apply recursively to entire dependency chain (not just top-level) Testing: - Add 4 unit tests for resolver return_all_versions behavior - Add unit test for continue-on-error and graph cleanup behavior - Add e2e test using tomli>=2.0,<2.1 with constraints to verify multiple versions are bootstrapped successfully Co-Authored-By: Claude Sonnet 4.5 Signed-off-by: Rohan Devasthale --- e2e/ci_bootstrap_suite.sh | 1 + e2e/test_bootstrap_multiple_versions.sh | 62 +++++++ .../bootstrap_requirement_resolver.py | 15 +- src/fromager/bootstrapper.py | 157 ++++++++++++---- src/fromager/commands/bootstrap.py | 23 +++ src/fromager/commands/lint_requirements.py | 3 +- src/fromager/dependency_graph.py | 45 +++++ tests/test_bootstrap_requirement_resolver.py | 20 +- ...bootstrap_requirement_resolver_multiple.py | 172 ++++++++++++++++++ tests/test_bootstrap_test_mode.py | 16 +- tests/test_bootstrapper.py | 77 ++++++++ 11 files changed, 537 insertions(+), 54 deletions(-) create mode 100755 e2e/test_bootstrap_multiple_versions.sh create mode 100644 tests/test_bootstrap_requirement_resolver_multiple.py diff --git a/e2e/ci_bootstrap_suite.sh b/e2e/ci_bootstrap_suite.sh index 3d4db445..ea8fa310 100755 --- a/e2e/ci_bootstrap_suite.sh +++ b/e2e/ci_bootstrap_suite.sh @@ -25,6 +25,7 @@ test_section "bootstrap configuration tests" run_test "bootstrap_prerelease" run_test "bootstrap_cache" run_test "bootstrap_sdist_only" +run_test "bootstrap_multiple_versions" test_section "bootstrap git URL tests" run_test "bootstrap_git_url" diff --git a/e2e/test_bootstrap_multiple_versions.sh b/e2e/test_bootstrap_multiple_versions.sh new file mode 100755 index 00000000..2d77c4b2 --- /dev/null +++ b/e2e/test_bootstrap_multiple_versions.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# Test bootstrap with --multiple-versions flag +# Tests that multiple matching versions are bootstrapped + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/common.sh" + +# Create constraints file to pin build dependencies (keeps CI fast) +constraints_file=$(mktemp) +trap "rm -f $constraints_file" EXIT +cat > "$constraints_file" <=2.0,<=2.0.2' || true + +# Check that wheels were built +echo "Checking for wheels..." +find "$OUTDIR/wheels-repo/downloads/" -name 'tomli-*.whl' | sort + +# Verify that all expected versions were bootstrapped +# Note: We don't check the exact count to avoid test fragility if extra versions appear +EXPECTED_VERSIONS="2.0.0 2.0.1 2.0.2" +MISSING_VERSIONS="" + +for version in $EXPECTED_VERSIONS; do + if find "$OUTDIR/wheels-repo/downloads/" -name "tomli-$version-*.whl" | grep -q .; then + echo "✓ Found wheel for tomli $version" + else + echo "✗ Missing wheel for tomli $version" + MISSING_VERSIONS="$MISSING_VERSIONS $version" + fi +done + +if [ -n "$MISSING_VERSIONS" ]; then + echo "" + echo "ERROR: Missing expected versions:$MISSING_VERSIONS" + echo "The --multiple-versions flag should have bootstrapped all matching versions" + echo "" + echo "Found wheels:" + find "$OUTDIR/wheels-repo/downloads/" -name 'tomli-*.whl' + exit 1 +fi + +echo "" +echo "SUCCESS: All expected tomli versions (2.0.0, 2.0.1, 2.0.2) were bootstrapped" diff --git a/src/fromager/bootstrap_requirement_resolver.py b/src/fromager/bootstrap_requirement_resolver.py index 1c5a6146..9e1fefa3 100644 --- a/src/fromager/bootstrap_requirement_resolver.py +++ b/src/fromager/bootstrap_requirement_resolver.py @@ -62,8 +62,9 @@ def resolve( req_type: RequirementType, parent_req: Requirement | None = None, pre_built: bool | None = None, - ) -> tuple[str, Version]: - """Resolve package requirement to the best matching version. + return_all_versions: bool = False, + ) -> list[tuple[str, Version]]: + """Resolve package requirement to matching version(s). Tries resolution strategies in order: 1. Session cache (if previously resolved) @@ -76,9 +77,13 @@ def resolve( parent_req: Parent requirement from dependency chain pre_built: Optional override to force prebuilt (True) or source (False). If None (default), uses package build info to determine. + return_all_versions: If True, return all matching versions. If False, + return only the highest matching version. Returns: - (url, version) tuple for the highest matching version + List of (url, version) tuples sorted by version (highest first). + Contains one item when return_all_versions=False, or all matching + versions when return_all_versions=True. Raises: ValueError: If req contains a git URL and pre_built is False @@ -101,14 +106,14 @@ def resolve( cached_result = self.get_cached_resolution(req, pre_built) if cached_result is not None: logger.debug(f"resolved {req} from cache") - return cached_result[0] + return cached_result if return_all_versions else [cached_result[0]] # Resolve using strategies results = self._resolve(req, req_type, parent_req, pre_built) # Cache the result self.cache_resolution(req, pre_built, results) - return results[0] + return results if return_all_versions else [results[0]] def _resolve( self, diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index e18137f0..1dff6de0 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -89,6 +89,7 @@ def __init__( cache_wheel_server_url: str | None = None, sdist_only: bool = False, test_mode: bool = False, + multiple_versions: bool = False, ) -> None: if test_mode and sdist_only: raise ValueError( @@ -101,6 +102,7 @@ def __init__( self.cache_wheel_server_url = cache_wheel_server_url or ctx.wheel_server_url self.sdist_only = sdist_only self.test_mode = test_mode + self.multiple_versions = multiple_versions self.why: list[tuple[RequirementType, Requirement, Version]] = [] # Delegate resolution to BootstrapRequirementResolver @@ -126,6 +128,10 @@ def __init__( # Track failed packages in test mode (list of typed dicts for JSON export) self.failed_packages: list[FailureRecord] = [] + # Track failed versions in multiple_versions mode + # Maps (package_name, version) -> exception info + self._failed_versions: list[tuple[str, str, Exception]] = [] + def resolve_and_add_top_level( self, req: Requirement, @@ -135,6 +141,10 @@ def resolve_and_add_top_level( This is the pre-resolution phase before recursive bootstrapping begins. In test mode, catches resolution errors and records them as failures. + When multiple_versions is enabled, resolves and adds all matching versions + to the graph, but still returns only the first (highest) version for + backward compatibility. + Args: req: The top-level requirement to resolve. @@ -147,40 +157,59 @@ def resolve_and_add_top_level( """ try: pbi = self.ctx.package_build_info(req) - source_url, version = self.resolve_version( + results = self.resolve_versions( req=req, req_type=RequirementType.TOP_LEVEL, + return_all_versions=self.multiple_versions, ) - logger.info("%s resolves to %s", req, version) - self.ctx.dependency_graph.add_dependency( - parent_name=None, - parent_version=None, - req_type=RequirementType.TOP_LEVEL, - req=req, - req_version=version, - download_url=source_url, - pre_built=pbi.pre_built, - constraint=self.ctx.constraints.get_constraint(req.name), - ) - return (source_url, version) + if self.multiple_versions: + logger.info(f"resolved {len(results)} version(s) for {req}") + + # Add all resolved versions to the graph + for source_url, version in results: + logger.info("%s resolves to %s", req, version) + self.ctx.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=req, + req_version=version, + download_url=source_url, + pre_built=pbi.pre_built, + constraint=self.ctx.constraints.get_constraint(req.name), + ) + + # Return first result for backward compatibility + return results[0] except Exception as err: if not self.test_mode: raise self._record_test_mode_failure(req, None, err, "resolution") return None - def resolve_version( + def resolve_versions( self, req: Requirement, req_type: RequirementType, - ) -> tuple[str, Version]: - """Resolve the version of a requirement. + return_all_versions: bool = False, + ) -> list[tuple[str, Version]]: + """Resolve version(s) of a requirement. - Returns the source URL and the version of the requirement (highest matching version). + Returns list of (source URL, version) tuples, sorted by version (highest first). Git URL resolution stays in Bootstrapper because it requires build orchestration (BuildEnvironment, build dependencies). Delegates PyPI/graph resolution to BootstrapRequirementResolver. + + Args: + req: Package requirement to resolve + req_type: Type of requirement + return_all_versions: If True, return all matching versions. + If False, return only highest version. + + Returns: + List of (url, version) tuples. Contains one item when + return_all_versions=False, or all matching versions when True. """ if req.url: if req_type != RequirementType.TOP_LEVEL: @@ -193,26 +222,23 @@ def resolve_version( cached_result = self._resolver.get_cached_resolution(req, pre_built=False) if cached_result is not None: logger.debug(f"resolved {req} from cache") - # Pick highest version from cached list - return cached_result[0] + return cached_result if return_all_versions else [cached_result[0]] logger.info("resolving source via URL, ignoring any plugins") source_url, resolved_version = self._resolve_version_from_git_url(req=req) # Cache the git URL resolution (always source, not prebuilt) # Store as list for consistency with cache structure - self._resolver.cache_resolution( - req, pre_built=False, result=[(source_url, resolved_version)] - ) - return source_url, resolved_version + result = [(source_url, resolved_version)] + self._resolver.cache_resolution(req, pre_built=False, result=result) + return result # Git URLs always return single version # Delegate to RequirementResolver parent_req = self.why[-1][1] if self.why else None - - # Returns the highest matching version return self._resolver.resolve( req=req, req_type=req_type, parent_req=parent_req, + return_all_versions=return_all_versions, ) def _processing_build_requirement(self, current_req_type: RequirementType) -> bool: @@ -249,22 +275,61 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> None: In test mode, catches build exceptions, records package name, and continues. In normal mode, raises exceptions immediately (fail-fast). + + When multiple_versions is enabled, bootstraps all matching versions instead + of just the highest version. """ logger.info(f"bootstrapping {req} as {req_type} dependency of {self.why[-1:]}") - # Resolve version first so we have it for error reporting. + # Resolve versions - get all if multiple_versions mode is enabled, else get highest # In test mode, record resolution failures and continue. try: - source_url, resolved_version = self.resolve_version( + resolved_versions = self.resolve_versions( req=req, req_type=req_type, + return_all_versions=self.multiple_versions, ) + if self.multiple_versions: + logger.info(f"resolved {len(resolved_versions)} version(s) for {req}") except Exception as err: if not self.test_mode: raise self._record_test_mode_failure(req, None, err, "resolution") return + # Check if resolution returned no versions + if not resolved_versions: + raise RuntimeError(f"Could not resolve any versions for {req}") + + # Bootstrap each resolved version + for source_url, resolved_version in resolved_versions: + self._bootstrap_single_version(req, req_type, source_url, resolved_version) + + # In multiple versions mode, report any failures for this requirement + if self.multiple_versions and self._failed_versions: + failed_for_req = [ + (name, ver, exc) + for name, ver, exc in self._failed_versions + if name == canonicalize_name(req.name) + ] + if failed_for_req: + logger.warning( + f"{req.name}: {len(failed_for_req)} version(s) failed to bootstrap" + ) + for name, ver, exc in failed_for_req: + logger.warning(f" - {name}=={ver}: {type(exc).__name__}: {exc}") + + def _bootstrap_single_version( + self, + req: Requirement, + req_type: RequirementType, + source_url: str, + resolved_version: Version, + ) -> None: + """Bootstrap a single version of a package. + + Extracted from bootstrap() to handle both single and multiple version modes. + """ # Capture parent before _track_why pushes current package onto the stack parent: tuple[Requirement, Version] | None = None if self.why: @@ -272,7 +337,9 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> None: parent = (parent_req, parent_version) # Update dependency graph unconditionally (before seen check to capture all edges) - self._add_to_graph(req, req_type, resolved_version, source_url, parent) + # Skip for TOP_LEVEL as they were already added in resolve_and_add_top_level() + if req_type != RequirementType.TOP_LEVEL: + self._add_to_graph(req, req_type, resolved_version, source_url, parent) # Build sdist-only (no wheel) if flag is set, unless this is a build # requirement which always needs a full wheel. @@ -298,11 +365,29 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> None: req, req_type, source_url, resolved_version, build_sdist_only ) except Exception as err: - if not self.test_mode: - raise - self._record_test_mode_failure( - req, str(resolved_version), err, "bootstrap" - ) + # In test_mode, record failure and continue + if self.test_mode: + self._record_test_mode_failure( + req, str(resolved_version), err, "bootstrap" + ) + return + + # In multiple_versions mode, record failure and continue to next version + if self.multiple_versions: + pkg_name = canonicalize_name(req.name) + self._failed_versions.append((pkg_name, str(resolved_version), err)) + logger.warning( + f"{req.name}=={resolved_version}: failed to bootstrap: {type(err).__name__}: {err}" + ) + # Remove failed node from graph since bootstrap didn't complete + self.ctx.dependency_graph.remove_dependency( + pkg_name, resolved_version + ) + self.ctx.write_to_graph_to_file() + return + + # Otherwise, raise the exception (fail-fast) + raise def _bootstrap_impl( self, @@ -924,12 +1009,13 @@ def _handle_test_mode_failure( try: parent_req = self.why[-1][1] if self.why else None - wheel_url, fallback_version = self._resolver.resolve( + results = self._resolver.resolve( req=req, req_type=req_type, parent_req=parent_req, pre_built=True, # Force prebuilt for test mode fallback ) + wheel_url, fallback_version = results[0] if fallback_version != resolved_version: logger.warning( @@ -1261,9 +1347,6 @@ def _add_to_graph( download_url: str, parent: tuple[Requirement, Version] | None, ) -> None: - if req_type == RequirementType.TOP_LEVEL: - return - parent_req, parent_version = parent if parent else (None, None) pbi = self.ctx.package_build_info(req) # Update the dependency graph after we determine that this requirement is diff --git a/src/fromager/commands/bootstrap.py b/src/fromager/commands/bootstrap.py index f42cfb71..3abf0644 100644 --- a/src/fromager/commands/bootstrap.py +++ b/src/fromager/commands/bootstrap.py @@ -103,6 +103,13 @@ def _get_requirements_from_args( default=False, help="Test mode: continue processing after failures, report failures at end", ) +@click.option( + "--multiple-versions", + "multiple_versions", + is_flag=True, + default=False, + help="Bootstrap all matching versions instead of only the highest version", +) @click.argument("toplevel", nargs=-1) @click.pass_obj def bootstrap( @@ -113,6 +120,7 @@ def bootstrap( sdist_only: bool, skip_constraints: bool, test_mode: bool, + multiple_versions: bool, toplevel: list[str], ) -> None: """Compute and build the dependencies of a set of requirements recursively @@ -147,6 +155,11 @@ def bootstrap( "test mode enabled: will continue processing after failures and report at end" ) + if multiple_versions: + logger.info( + "multiple versions mode enabled: will bootstrap all matching versions" + ) + pre_built = wkctx.settings.list_pre_built() if pre_built: logger.info("treating %s as pre-built wheels", sorted(pre_built)) @@ -161,6 +174,7 @@ def bootstrap( cache_wheel_server_url, sdist_only=sdist_only, test_mode=test_mode, + multiple_versions=multiple_versions, ) # Pre-resolution phase: Resolve all top-level dependencies before recursive @@ -463,6 +477,13 @@ def write_constraints_file( default=None, help="maximum number of parallel workers to run (default: unlimited)", ) +@click.option( + "--multiple-versions", + "multiple_versions", + is_flag=True, + default=False, + help="Bootstrap all matching versions instead of only the highest version", +) @click.argument("toplevel", nargs=-1) @click.pass_obj @click.pass_context @@ -476,6 +497,7 @@ def bootstrap_parallel( skip_constraints: bool, force: bool, max_workers: int | None, + multiple_versions: bool, toplevel: list[str], ) -> None: """Bootstrap and build-parallel @@ -502,6 +524,7 @@ def bootstrap_parallel( cache_wheel_server_url=cache_wheel_server_url, sdist_only=True, skip_constraints=skip_constraints, + multiple_versions=multiple_versions, toplevel=toplevel, ) diff --git a/src/fromager/commands/lint_requirements.py b/src/fromager/commands/lint_requirements.py index bc78ca11..776f759c 100644 --- a/src/fromager/commands/lint_requirements.py +++ b/src/fromager/commands/lint_requirements.py @@ -89,10 +89,11 @@ def lint_requirements( if resolve_requirements and not is_constraints: token = requirement_ctxvar.set(requirement) try: - _, version = bt.resolve_version( + results = bt.resolve_versions( req=requirement, req_type=RequirementType.TOP_LEVEL, ) + _, version = results[0] logger.info(f"{requirement} resolves to {version}") except Exception as err: logger.error( diff --git a/src/fromager/dependency_graph.py b/src/fromager/dependency_graph.py index 7f1201dc..8f05be99 100644 --- a/src/fromager/dependency_graph.py +++ b/src/fromager/dependency_graph.py @@ -336,6 +336,51 @@ def add_dependency( self.nodes[parent_key].add_child(node, req=req, req_type=req_type) + def remove_dependency( + self, + req_name: NormalizedName, + req_version: Version, + ) -> None: + """Remove a dependency node from the graph. + + Removes the node and all edges pointing to it. This includes: + - Back-references in child nodes' parents lists + - Forward edges in parent nodes' children lists + Child nodes of the removed node are kept if referenced elsewhere. + + Args: + req_name: Canonical name of the package + req_version: Version of the package + """ + key = f"{req_name}=={req_version}" + if key not in self.nodes: + logger.debug(f"Cannot remove {key} - not in graph") + return + + logger.debug(f"Removing failed dependency {key} from graph") + + deleted_node = self.nodes[key] + + # Clean up back-references (parents) in nodes that were children of the removed node + for child_edge in deleted_node.children: + child_node = child_edge.destination_node + filtered_parents = [ + edge for edge in child_node.parents if edge.destination_node.key != key + ] + child_node.parents.clear() + child_node.parents.extend(filtered_parents) + + # Remove the node itself + del self.nodes[key] + + # Remove forward edges from any node whose children pointed to the removed node + for node in self.nodes.values(): + filtered_children = [ + edge for edge in node.children if edge.destination_node.key != key + ] + node.children.clear() + node.children.extend(filtered_children) + def get_dependency_edges( self, match_dep_types: list[RequirementType] | None = None ) -> typing.Iterable[DependencyEdge]: diff --git a/tests/test_bootstrap_requirement_resolver.py b/tests/test_bootstrap_requirement_resolver.py index d8604f66..f6c9377e 100644 --- a/tests/test_bootstrap_requirement_resolver.py +++ b/tests/test_bootstrap_requirement_resolver.py @@ -463,7 +463,7 @@ def test_resolve_allows_git_urls_for_prebuilt( ] # Should NOT raise - git URLs are allowed when explicitly requesting prebuilt - url, version = resolver.resolve( + results = resolver.resolve( req=req, req_type=RequirementType.INSTALL, pre_built=True, @@ -472,6 +472,8 @@ def test_resolve_allows_git_urls_for_prebuilt( # Verify resolution was called mock_resolve.assert_called_once() + assert len(results) == 1 + url, version = results[0] assert url == "https://files.pythonhosted.org/mypkg-1.0-py3-none-any.whl" assert version == Version("1.0") @@ -501,7 +503,7 @@ def test_resolve_auto_routes_to_prebuilt( ] # Call resolve with pre_built=None (should auto-detect) - url, version = resolver.resolve( + results = resolver.resolve( req=req, req_type=RequirementType.INSTALL, parent_req=None, @@ -510,6 +512,8 @@ def test_resolve_auto_routes_to_prebuilt( # Verify resolution was called mock_resolve.assert_called_once() + assert len(results) == 1 + url, version = results[0] assert url == "https://files.pythonhosted.org/setuptools-1.0-py3-none-any.whl" assert version == Version("1.0") @@ -541,7 +545,7 @@ def test_resolve_auto_routes_to_source( ] # Call resolve with pre_built=None (should auto-detect) - url, version = resolver.resolve( + results = resolver.resolve( req=req, req_type=RequirementType.INSTALL, parent_req=None, @@ -550,6 +554,8 @@ def test_resolve_auto_routes_to_source( # Verify resolution was called mock_resolve.assert_called_once() + assert len(results) == 1 + url, version = results[0] assert url == "https://files.pythonhosted.org/mypackage-2.0.tar.gz" assert version == Version("2.0") @@ -578,20 +584,22 @@ def test_resolve_prebuilt_after_source_uses_separate_cache( resolver = BootstrapRequirementResolver(tmp_context) # First call: resolve as source (explicit pre_built=False) - url1, version1 = resolver.resolve( + results1 = resolver.resolve( req=req, req_type=RequirementType.INSTALL, parent_req=None, pre_built=False, ) + assert len(results1) == 1 + url1, version1 = results1[0] assert url1 == "https://files.pythonhosted.org/testpkg-1.5.tar.gz" assert version1 == Version("1.5") assert mock_resolve.call_count == 1 # Second call: resolve same req as prebuilt (explicit pre_built=True) # This should NOT return the cached source result - url2, version2 = resolver.resolve( + results2 = resolver.resolve( req=req, req_type=RequirementType.INSTALL, parent_req=None, @@ -600,5 +608,7 @@ def test_resolve_prebuilt_after_source_uses_separate_cache( # Verify it called resolution again (not cached) because cache keys differ assert mock_resolve.call_count == 2 + assert len(results2) == 1 + url2, version2 = results2[0] assert url2 == "https://files.pythonhosted.org/testpkg-1.5-py3-none-any.whl" assert version2 == Version("1.5") diff --git a/tests/test_bootstrap_requirement_resolver_multiple.py b/tests/test_bootstrap_requirement_resolver_multiple.py new file mode 100644 index 00000000..59745d05 --- /dev/null +++ b/tests/test_bootstrap_requirement_resolver_multiple.py @@ -0,0 +1,172 @@ +"""Tests for multiple versions feature in bootstrap_requirement_resolver.""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from packaging.requirements import Requirement +from packaging.version import Version + +from fromager.bootstrap_requirement_resolver import BootstrapRequirementResolver +from fromager.context import WorkContext +from fromager.dependency_graph import DependencyGraph +from fromager.requirements_file import RequirementType + + +@pytest.fixture +def tmp_context(tmp_path: Path) -> WorkContext: + """Create a minimal WorkContext for testing.""" + ctx = MagicMock(spec=WorkContext) + ctx.work_dir = tmp_path + ctx.constraints = MagicMock() + ctx.constraints.get_constraint.return_value = None + ctx.settings = MagicMock() + ctx.settings.list_pre_built.return_value = set() + ctx.package_build_info = MagicMock() + pbi = MagicMock() + pbi.pre_built = False + pbi.resolver_include_sdists = True + pbi.resolver_include_wheels = False + pbi.resolver_ignore_platform = False + pbi.resolver_sdist_server_url.return_value = "https://pypi.org/simple/" + ctx.package_build_info.return_value = pbi + return ctx + + +def test_resolve_return_all_versions_true(tmp_context: WorkContext) -> None: + """resolve() with return_all_versions=True returns all matching versions.""" + resolver = BootstrapRequirementResolver(tmp_context) + + # Mock the _resolve method to return multiple versions + with patch.object( + resolver, + "_resolve", + return_value=[ + ("https://pypi.org/testpkg-2.0.tar.gz", Version("2.0")), + ("https://pypi.org/testpkg-1.5.tar.gz", Version("1.5")), + ("https://pypi.org/testpkg-1.0.tar.gz", Version("1.0")), + ], + ): + req = Requirement("testpkg>=1.0") + results = resolver.resolve( + req=req, + req_type=RequirementType.INSTALL, + parent_req=None, + return_all_versions=True, + ) + + # Should return all 3 versions + assert len(results) == 3 + assert results[0] == ("https://pypi.org/testpkg-2.0.tar.gz", Version("2.0")) + assert results[1] == ("https://pypi.org/testpkg-1.5.tar.gz", Version("1.5")) + assert results[2] == ("https://pypi.org/testpkg-1.0.tar.gz", Version("1.0")) + + +def test_resolve_return_all_versions_false_default(tmp_context: WorkContext) -> None: + """resolve() with return_all_versions=False (default) returns list with only highest version.""" + resolver = BootstrapRequirementResolver(tmp_context) + + # Mock the _resolve method to return multiple versions + with patch.object( + resolver, + "_resolve", + return_value=[ + ("https://pypi.org/testpkg-2.0.tar.gz", Version("2.0")), + ("https://pypi.org/testpkg-1.5.tar.gz", Version("1.5")), + ("https://pypi.org/testpkg-1.0.tar.gz", Version("1.0")), + ], + ): + req = Requirement("testpkg>=1.0") + + # Call without return_all_versions (default False) + results = resolver.resolve( + req=req, + req_type=RequirementType.INSTALL, + parent_req=None, + ) + + # Should return list with only the highest version + assert len(results) == 1 + assert results[0] == ("https://pypi.org/testpkg-2.0.tar.gz", Version("2.0")) + + +def test_resolve_return_all_versions_uses_cache(tmp_context: WorkContext) -> None: + """resolve() with return_all_versions=True uses cache correctly.""" + resolver = BootstrapRequirementResolver(tmp_context) + + # First call - will populate cache + with patch.object( + resolver, + "_resolve", + return_value=[ + ("https://pypi.org/testpkg-2.0.tar.gz", Version("2.0")), + ("https://pypi.org/testpkg-1.0.tar.gz", Version("1.0")), + ], + ) as mock_resolve: + req = Requirement("testpkg>=1.0") + + # First call + results1 = resolver.resolve( + req=req, + req_type=RequirementType.INSTALL, + parent_req=None, + return_all_versions=True, + ) + assert len(results1) == 2 + assert mock_resolve.call_count == 1 + + # Second call - should use cache + results2 = resolver.resolve( + req=req, + req_type=RequirementType.INSTALL, + parent_req=None, + return_all_versions=True, + ) + + # Should not call _resolve again + assert mock_resolve.call_count == 1 + # Should return same results + assert results2 == results1 + + +def test_resolve_return_all_versions_with_previous_graph( + tmp_context: WorkContext, +) -> None: + """resolve() with return_all_versions=True works with previous graph.""" + # Create graph with multiple versions of the same package + prev_graph = DependencyGraph() + prev_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=Requirement("testpkg==2.0"), + req_version=Version("2.0"), + ) + prev_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=Requirement("testpkg==1.0"), + req_version=Version("1.0"), + ) + + # Mock dependency_graph in context + tmp_context.dependency_graph = prev_graph + + resolver = BootstrapRequirementResolver(tmp_context, prev_graph) + + # Request with version spec that matches both + req = Requirement("testpkg>=1.0") + results = resolver.resolve( + req=req, + req_type=RequirementType.TOP_LEVEL, + parent_req=None, + return_all_versions=True, + ) + + # Should return both versions from graph + assert len(results) == 2 + # Verify versions (should be sorted highest first) + versions = [v for _, v in results] + assert Version("2.0") in versions + assert Version("1.0") in versions diff --git a/tests/test_bootstrap_test_mode.py b/tests/test_bootstrap_test_mode.py index 84b8bc60..150261f5 100644 --- a/tests/test_bootstrap_test_mode.py +++ b/tests/test_bootstrap_test_mode.py @@ -256,13 +256,15 @@ class TestBootstrapExceptionHandling: def test_resolution_failure_recorded_in_test_mode( self, tmp_context: context.WorkContext ) -> None: - """Test that resolve_version failures are recorded in test mode.""" + """Test that resolve_versions failures are recorded in test mode.""" bt = bootstrapper.Bootstrapper(ctx=tmp_context, test_mode=True) req = Requirement("nonexistent-package>=1.0") - # Mock resolve_version to raise an exception + # Mock resolve_versions to raise an exception with mock.patch.object( - bt, "resolve_version", side_effect=RuntimeError("Version resolution failed") + bt, + "resolve_versions", + side_effect=RuntimeError("Version resolution failed"), ): # Should not raise in test mode bt.bootstrap(req=req, req_type=RequirementType.TOP_LEVEL) @@ -280,13 +282,15 @@ def test_resolution_failure_recorded_in_test_mode( def test_resolution_failure_raises_in_normal_mode( self, tmp_context: context.WorkContext ) -> None: - """Test that resolve_version failures raise in normal mode.""" + """Test that resolve_versions failures raise in normal mode.""" bt = bootstrapper.Bootstrapper(ctx=tmp_context, test_mode=False) req = Requirement("nonexistent-package>=1.0") - # Mock resolve_version to raise an exception + # Mock resolve_versions to raise an exception with mock.patch.object( - bt, "resolve_version", side_effect=RuntimeError("Version resolution failed") + bt, + "resolve_versions", + side_effect=RuntimeError("Version resolution failed"), ): with pytest.raises(RuntimeError, match="Version resolution failed"): bt.bootstrap(req=req, req_type=RequirementType.TOP_LEVEL) diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 46b78c79..726125c3 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -3,6 +3,7 @@ from unittest.mock import Mock, patch from packaging.requirements import Requirement +from packaging.utils import canonicalize_name from packaging.version import Version from fromager import bootstrapper, requirements_file @@ -293,3 +294,79 @@ def test_build_from_source_returns_dataclass(tmp_context: WorkContext) -> None: assert result.sdist_root_dir == mock_sdist_root assert result.build_env is not None assert result.source_type == SourceType.SDIST + + +def test_multiple_versions_continues_on_error(tmp_context: WorkContext) -> None: + """Test that multiple versions mode continues when one version fails.""" + # Enable multiple versions mode + bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) + + # Mock the resolver to return 3 versions + with patch.object( + bt._resolver, + "resolve", + return_value=[ + ("https://pypi.org/testpkg-2.0.tar.gz", Version("2.0")), + ("https://pypi.org/testpkg-1.5.tar.gz", Version("1.5")), + ("https://pypi.org/testpkg-1.0.tar.gz", Version("1.0")), + ], + ): + # Mock _bootstrap_impl to fail for version 1.5 only + call_count = {"count": 0} + + def mock_bootstrap_impl( + req: Requirement, + req_type: RequirementType, + source_url: str, + resolved_version: Version, + build_sdist_only: bool, + ) -> None: + call_count["count"] += 1 + if str(resolved_version) == "1.5": + raise ValueError("Simulated failure for version 1.5") + # For other versions, just mark as seen to avoid actual build + bt._mark_as_seen(req, resolved_version, build_sdist_only) + + with patch.object(bt, "_bootstrap_impl", side_effect=mock_bootstrap_impl): + # Mock _has_been_seen to return False so we attempt bootstrap + with patch.object(bt, "_has_been_seen", return_value=False): + # Capture log output + with patch("fromager.bootstrapper.logger") as mock_logger: + req = Requirement("testpkg>=1.0") + + # Call bootstrap with INSTALL type (not TOP_LEVEL, since TOP_LEVEL + # nodes are added in resolve_and_add_top_level()) + bt.bootstrap( + req=req, + req_type=RequirementType.INSTALL, + ) + + # Verify _bootstrap_impl was called 3 times (all versions attempted) + assert call_count["count"] == 3 + + # Verify that version 1.5 is in failed_versions + assert len(bt._failed_versions) == 1 + pkg_name, version_str, exc = bt._failed_versions[0] + assert pkg_name == canonicalize_name("testpkg") + assert version_str == "1.5" + assert isinstance(exc, ValueError) + assert str(exc) == "Simulated failure for version 1.5" + + # Verify that a warning was logged for the failed version + warning_calls = [ + call + for call in mock_logger.warning.call_args_list + if "failed to bootstrap" in str(call) + ] + assert len(warning_calls) >= 1 + + # Verify that failed version 1.5 is NOT in the dependency graph + # (should have been removed) + failed_key = f"{canonicalize_name('testpkg')}==1.5" + assert failed_key not in tmp_context.dependency_graph.nodes + + # Verify that successful versions ARE in the dependency graph + success_key_20 = f"{canonicalize_name('testpkg')}==2.0" + success_key_10 = f"{canonicalize_name('testpkg')}==1.0" + assert success_key_20 in tmp_context.dependency_graph.nodes + assert success_key_10 in tmp_context.dependency_graph.nodes