diff --git a/docs/how-tos/bootstrap-constraints.rst b/docs/how-tos/bootstrap-constraints.rst index cdcacd54..b8645d88 100644 --- a/docs/how-tos/bootstrap-constraints.rst +++ b/docs/how-tos/bootstrap-constraints.rst @@ -31,7 +31,7 @@ production packages. .. code-block:: console - $ fromager --constraints-file constraints.txt build-sequence ./work-dir/build-order.json + $ fromager --constraints-file constraints.txt build-sequence ./work-dir/graph.json ./work-dir/build-order.json This will use the constraints in the ``constraints.txt`` file to build the production packages for ``my-package``. diff --git a/docs/using.md b/docs/using.md index 7439ed52..9c190e76 100644 --- a/docs/using.md +++ b/docs/using.md @@ -187,8 +187,10 @@ individual package compilation or integration into larger build systems. ### The build-sequence command -The `build-sequence` command processes a pre-determined build order file -(typically `build-order.json`) to build wheels in dependency order. +The `build-sequence` command processes a dependency graph (`graph.json`) and a +pre-determined build order file (`build-order.json`) to build wheels in +dependency order. Build dependencies are resolved from the graph rather than +running PEP 517 discovery hooks. The outputs are patched source distributions and built wheels for each item in the build-order file. @@ -198,11 +200,12 @@ for any wheels that have already been built with the current settings. For each package in the sequence: -1. **Build Order Reading** - Loads the build order file containing: +1. **Build Order Reading** - Loads the build order and graph files: - - Package names and versions to build - - Source URLs and types (PyPI, git, prebuilt) - - Dependency relationships and constraints + - `build-order.json`: Package names, versions, source URLs and types + (PyPI, git, prebuilt) in predetermined build order + - `graph.json`: Dependency relationships used to resolve build + requirements without running PEP 517 discovery hooks 2. **Build Status Checking** - Determines if building is needed: diff --git a/e2e/test_build_order.sh b/e2e/test_build_order.sh index d9a0f184..063c93de 100755 --- a/e2e/test_build_order.sh +++ b/e2e/test_build_order.sh @@ -19,8 +19,9 @@ fromager \ --settings-dir="$SCRIPTDIR/changelog_settings" \ bootstrap "${DIST}==${VERSION}" -# Save the build order file but remove everything else. +# Save the build order and graph files but remove everything else. cp "$OUTDIR/work-dir/build-order.json" "$OUTDIR/" +cp "$OUTDIR/work-dir/graph.json" "$OUTDIR/" # Rebuild everything even if it already exists log="$OUTDIR/build-logs/${DIST}-build.log" @@ -31,7 +32,7 @@ fromager \ --sdists-repo "$OUTDIR/sdists-repo" \ --wheels-repo "$OUTDIR/wheels-repo" \ --settings-dir="$SCRIPTDIR/changelog_settings" \ - build-sequence --force "$OUTDIR/build-order.json" + build-sequence --force "$OUTDIR/graph.json" "$OUTDIR/build-order.json" find "$OUTDIR/wheels-repo/" @@ -94,7 +95,7 @@ fromager \ --sdists-repo "$OUTDIR/sdists-repo" \ --wheels-repo "$OUTDIR/wheels-repo" \ --settings-dir="$SCRIPTDIR/changelog_settings" \ - build-sequence "$OUTDIR/build-order.json" + build-sequence "$OUTDIR/graph.json" "$OUTDIR/build-order.json" find "$OUTDIR/wheels-repo/" @@ -118,7 +119,7 @@ fromager \ --work-dir "$OUTDIR/work-dir" \ --sdists-repo "$OUTDIR/sdists-repo" \ --wheels-repo "$OUTDIR/wheels-repo" \ - build-sequence --cache-wheel-server-url="https://pypi.org/simple" "$OUTDIR/build-order.json" + build-sequence --cache-wheel-server-url="https://pypi.org/simple" "$OUTDIR/graph.json" "$OUTDIR/build-order.json" find "$OUTDIR/wheels-repo/" diff --git a/e2e/test_build_sequence_git_url.sh b/e2e/test_build_sequence_git_url.sh index b9aca258..a6e6d301 100755 --- a/e2e/test_build_sequence_git_url.sh +++ b/e2e/test_build_sequence_git_url.sh @@ -23,6 +23,7 @@ ls "$OUTDIR"/work-dir/*/build.log || true # Clean up the work directory so we can test build-sequence mv "$OUTDIR/work-dir/build-order.json" "$OUTDIR/" +cp "$OUTDIR/work-dir/graph.json" "$OUTDIR/" rm -rf "$OUTDIR/work-dir/wheels-repo" rm -rf "$OUTDIR/work-dir/sdists-repo" @@ -37,7 +38,7 @@ fromager \ --sdists-repo "$OUTDIR/sdists-repo" \ --wheels-repo "$OUTDIR/wheels-repo" \ --settings-dir="$SCRIPTDIR/changelog_settings" \ - build-sequence --force "$OUTDIR/build-order.json" + build-sequence --force "$OUTDIR/graph.json" "$OUTDIR/build-order.json" find "$OUTDIR/wheels-repo/" -name '*.whl' find "$OUTDIR/sdists-repo/" -name '*.tar.gz' diff --git a/e2e/test_prebuilt_wheel_hook.sh b/e2e/test_prebuilt_wheel_hook.sh index 7550869c..576bc9ff 100755 --- a/e2e/test_prebuilt_wheel_hook.sh +++ b/e2e/test_prebuilt_wheel_hook.sh @@ -21,8 +21,9 @@ fromager \ --settings-dir="$SCRIPTDIR/prebuilt_settings" \ bootstrap "${DIST}==${VERSION}" -# Save the build order file but remove everything else. +# Save the build order and graph files but remove everything else. cp "$OUTDIR/work-dir/build-order.json" "$OUTDIR/" +cp "$OUTDIR/work-dir/graph.json" "$OUTDIR/" # Remove downloaded wheels to trigger hook rm -rf "$OUTDIR/wheels-repo" @@ -34,7 +35,7 @@ fromager \ --sdists-repo "$OUTDIR/sdists-repo" \ --wheels-repo "$OUTDIR/wheels-repo" \ --settings-dir="$SCRIPTDIR/prebuilt_settings" \ - build-sequence "$OUTDIR/build-order.json" + build-sequence "$OUTDIR/graph.json" "$OUTDIR/build-order.json" PATTERNS=( "downloading prebuilt wheel ${DIST}==${VERSION}" diff --git a/src/fromager/build_environment.py b/src/fromager/build_environment.py index 3a9ee296..49bc18ca 100644 --- a/src/fromager/build_environment.py +++ b/src/fromager/build_environment.py @@ -18,7 +18,7 @@ from .requirements_file import RequirementType if typing.TYPE_CHECKING: - from . import context + from . import context, dependency_graph logger = logging.getLogger(__name__) @@ -314,6 +314,55 @@ def prepare_build_environment( return build_env +@metrics.timeit(description="prepare build environment from graph") +def prepare_build_environment_from_graph( + *, + ctx: context.WorkContext, + req: Requirement, + sdist_root_dir: pathlib.Path, + build_requirements: typing.Iterable[dependency_graph.DependencyNode], +) -> BuildEnvironment: + """Create a build environment populated from pre-resolved graph dependencies. + + Uses build requirements extracted from the dependency graph instead of + running PEP 517 discovery hooks. This is the preferred path for Stage 2 + build commands (build-sequence, build-parallel) where the graph is the + source of truth. + """ + logger.info("preparing build environment from dependency graph") + + build_env = BuildEnvironment( + ctx=ctx, + parent_dir=sdist_root_dir.parent, + ) + + reqs = { + Requirement(f"{node.canonicalized_name}=={node.version}") + for node in build_requirements + } + if reqs: + # Graph-resolved deps are a mix of build-system, build-backend, + # build-sdist, and their transitive install deps. We use + # BUILD_SYSTEM as the label since they all go into the build env. + _safe_install( + ctx=ctx, + req=req, + build_env=build_env, + deps=reqs, + dep_req_type=RequirementType.BUILD_SYSTEM, + ) + + try: + distributions = build_env.get_distributions() + except Exception: + # ignore error for debug call, error reason is logged in get_distributions() + pass + else: + logger.debug("build env %r has packages %r", build_env.path, distributions) + + return build_env + + def _safe_install( ctx: context.WorkContext, req: Requirement, diff --git a/src/fromager/commands/bootstrap.py b/src/fromager/commands/bootstrap.py index f42cfb71..447a859a 100644 --- a/src/fromager/commands/bootstrap.py +++ b/src/fromager/commands/bootstrap.py @@ -186,6 +186,10 @@ def bootstrap( progressbar.update() requirement_ctxvar.reset(token) + # Ensure graph.json is written even when no recursive dependencies + # were discovered (e.g., prebuilt-only bootstraps). + wkctx.write_to_graph_to_file() + # Finalize test mode and check for failures exit_code = bt.finalize() if exit_code != 0: diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 204f01ac..8a136629 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -142,22 +142,26 @@ def build( "cache_wheel_server_url", help="url to a wheel server from where fromager can check if it had already built the wheel", ) +@click.argument("graph_file") @click.argument("build_order_file") @click.pass_obj def build_sequence( wkctx: context.WorkContext, + graph_file: str, build_order_file: str, force: bool, cache_wheel_server_url: str | None, ) -> None: """Build a sequence of wheels in order - BUILD_ORDER_FILE is the build-order.json files to build + GRAPH_FILE is a graph.json file containing the dependency relationships + between packages, used to resolve build dependencies. - SDIST_SERVER_URL is the URL for a PyPI-compatible package index hosting sdists + BUILD_ORDER_FILE is the build-order.json file specifying the build order. Performs the equivalent of the 'build' command for each item in - the build order file. + the build order file, using the dependency graph to populate + build environments instead of PEP 517 discovery hooks. """ server.start_wheel_server(wkctx) @@ -173,6 +177,9 @@ def build_sequence( f"{wkctx.wheel_server_url=}, {cache_wheel_server_url=}" ) + logger.info("reading dependency graph from %s", graph_file) + graph = dependency_graph.DependencyGraph.from_file(graph_file) + entries: list[BuildSequenceEntry] = [] logger.info("reading build order from %s", build_order_file) @@ -191,6 +198,10 @@ def build_sequence( else: req = Requirement(f"{dist_name}=={resolved_version}") + build_requirements = _get_build_requirements_from_graph( + graph, dist_name, resolved_version + ) + with req_ctxvar_context(req, resolved_version): logger.info("building %s", resolved_version) entry = _build( @@ -200,6 +211,7 @@ def build_sequence( source_download_url=source_download_url, force=force, cache_wheel_server_url=cache_wheel_server_url, + build_requirements=build_requirements, ) if entry.prebuilt: logger.info( @@ -326,6 +338,7 @@ def _build( source_download_url: str, force: bool, cache_wheel_server_url: str | None, + build_requirements: typing.Iterable[dependency_graph.DependencyNode] | None = None, ) -> BuildSequenceEntry: """Handle one version of one wheel. @@ -417,12 +430,20 @@ def _build( ) # Build environment - build_env = build_environment.prepare_build_environment( - ctx=wkctx, - req=req, - version=resolved_version, - sdist_root_dir=source_root_dir, - ) + if build_requirements is not None: + build_env = build_environment.prepare_build_environment_from_graph( + ctx=wkctx, + req=req, + sdist_root_dir=source_root_dir, + build_requirements=build_requirements, + ) + else: + build_env = build_environment.prepare_build_environment( + ctx=wkctx, + req=req, + version=resolved_version, + sdist_root_dir=source_root_dir, + ) # Make a new source distribution, in case we patched the code. sdist_filename = sources.build_sdist( @@ -544,6 +565,7 @@ def _build_parallel( source_download_url: str, force: bool, cache_wheel_server_url: str | None, + build_requirements: typing.Iterable[dependency_graph.DependencyNode] | None = None, ) -> BuildSequenceEntry: """ This function runs in a thread to manage the build of a single package. @@ -556,7 +578,24 @@ def _build_parallel( source_download_url=source_download_url, force=force, cache_wheel_server_url=cache_wheel_server_url, + build_requirements=build_requirements, + ) + + +def _get_build_requirements_from_graph( + graph: dependency_graph.DependencyGraph, + dist_name: str, + version: Version, +) -> list[dependency_graph.DependencyNode]: + """Look up build requirements for a package from the dependency graph.""" + node_key = f"{canonicalize_name(dist_name)}=={version}" + node = graph.nodes.get(node_key) + if node is None: + raise KeyError( + f"package {node_key} not found in dependency graph; " + f"ensure the graph file matches the build order" ) + return list(node.iter_build_requirements()) def _nodes_to_string(nodes: typing.Iterable[dependency_graph.DependencyNode]) -> str: @@ -681,6 +720,7 @@ def update_progressbar_cb(future: concurrent.futures.Future) -> None: source_download_url=node.download_url, force=force, cache_wheel_server_url=cache_wheel_server_url, + build_requirements=list(node.iter_build_requirements()), ) future.add_done_callback(update_progressbar_cb) future2node[future] = node diff --git a/tests/test_build_environment.py b/tests/test_build_environment.py index 7b263389..f03c33f6 100644 --- a/tests/test_build_environment.py +++ b/tests/test_build_environment.py @@ -1,11 +1,16 @@ +import pathlib import textwrap from unittest.mock import Mock, patch +import pytest from packaging.requirements import Requirement +from packaging.utils import canonicalize_name from packaging.version import Version from fromager import build_environment +from fromager.commands.build import _get_build_requirements_from_graph from fromager.context import WorkContext +from fromager.dependency_graph import DependencyGraph, DependencyNode from fromager.requirements_file import RequirementType @@ -81,3 +86,96 @@ def test_missing_dependency_pattern_resolution_impossible() -> None: """) match = build_environment._uv_missing_dependency_pattern.search(msg) assert match is not None + + +@patch("fromager.build_environment.BuildEnvironment") +def test_prepare_build_environment_from_graph_installs_deps( + mock_build_env_cls: Mock, + tmp_context: WorkContext, + tmp_path: pathlib.Path, +) -> None: + """Verify graph-based build env installs resolved deps without discovery.""" + mock_build_env = Mock() + mock_build_env.get_distributions.return_value = {} + mock_build_env_cls.return_value = mock_build_env + + sdist_root_dir = tmp_path / "pkg-1.0" / "pkg-1.0" + sdist_root_dir.mkdir(parents=True) + + nodes = [ + DependencyNode(canonicalize_name("setuptools"), Version("80.8.0")), + DependencyNode(canonicalize_name("wheel"), Version("0.46.1")), + ] + + result = build_environment.prepare_build_environment_from_graph( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + sdist_root_dir=sdist_root_dir, + build_requirements=nodes, + ) + + assert result is mock_build_env + mock_build_env.install.assert_called_once() + installed_reqs = mock_build_env.install.call_args[0][0] + installed_names = {str(r) for r in installed_reqs} + assert installed_names == {"setuptools==80.8.0", "wheel==0.46.1"} + + +@patch("fromager.build_environment.BuildEnvironment") +def test_prepare_build_environment_from_graph_no_deps( + mock_build_env_cls: Mock, + tmp_context: WorkContext, + tmp_path: pathlib.Path, +) -> None: + """Verify graph-based build env works with no build deps.""" + mock_build_env = Mock() + mock_build_env.get_distributions.return_value = {} + mock_build_env_cls.return_value = mock_build_env + + sdist_root_dir = tmp_path / "pkg-1.0" / "pkg-1.0" + sdist_root_dir.mkdir(parents=True) + + result = build_environment.prepare_build_environment_from_graph( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + sdist_root_dir=sdist_root_dir, + build_requirements=[], + ) + + assert result is mock_build_env + mock_build_env.install.assert_not_called() + + +def test_get_build_requirements_from_graph() -> None: + """Verify build requirements are extracted from graph nodes.""" + graph = DependencyGraph() + graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=Requirement("pkg==1.0"), + req_version=Version("1.0"), + download_url="https://example.com/pkg-1.0.tar.gz", + ) + graph.add_dependency( + parent_name=canonicalize_name("pkg"), + parent_version=Version("1.0"), + req_type=RequirementType.BUILD_SYSTEM, + req=Requirement("setuptools>=61.2"), + req_version=Version("80.8.0"), + download_url="https://example.com/setuptools-80.8.0.tar.gz", + ) + + result = _get_build_requirements_from_graph(graph, "pkg", Version("1.0")) + + assert result is not None + assert len(result) == 1 + assert result[0].canonicalized_name == canonicalize_name("setuptools") + assert result[0].version == Version("80.8.0") + + +def test_get_build_requirements_from_graph_missing_node() -> None: + """Verify missing node raises KeyError.""" + graph = DependencyGraph() + with pytest.raises(KeyError, match="nonexistent"): + _get_build_requirements_from_graph(graph, "nonexistent", Version("1.0"))