diff --git a/docs/how-tos/multiple-version-bootstrap.rst b/docs/how-tos/multiple-version-bootstrap.rst index 4ea1bbeb..41bb9d1a 100644 --- a/docs/how-tos/multiple-version-bootstrap.rst +++ b/docs/how-tos/multiple-version-bootstrap.rst @@ -80,18 +80,47 @@ Output Files When ``--multiple-versions`` is active: +.. versionchanged:: 0.95.0 + Failed versions are recorded in ``partial-failures.json``. The file is + removed when a subsequent run has no reportable failures. + - ``build-order.json`` --- created normally, listing every version built - ``graph.json`` --- created normally, containing all versions in the dependency graph +- ``partial-failures.json`` --- created when one or more versions fail. Each + failure includes the package name, version, phase, exception type, and + message. - ``constraints.txt`` --- **not generated** +The report contains a top-level ``failures`` list. For a resolved version, +``name`` includes the package version (for example, ``requests==2.28.0``), +and ``version`` contains the version separately. For an unresolved +requirement, ``version`` is ``null`` and ``name`` contains only the package +name. + +.. code-block:: json + + { + "failures": [ + { + "name": "requests==2.28.0", + "version": "2.28.0", + "phase": "failed during build phase", + "error_type": "CalledProcessError", + "message": "build command failed" + } + ] + } + Combining with Other Flags -------------------------- ``--test-mode`` Supported with serial ``bootstrap`` only. Failures are collected and reported at the end rather than aborting early (same behavior as without - ``--multiple-versions``). + ``--multiple-versions``). When combined with ``--multiple-versions``, + failed versions are also included in ``partial-failures.json`` and test + mode retains its non-zero exit code for recorded failures. ``--skip-constraints`` Redundant when ``--multiple-versions`` is set. Fromager automatically diff --git a/docs/proposals/test-mode.md b/docs/proposals/test-mode.md index fba1aef7..6545c666 100644 --- a/docs/proposals/test-mode.md +++ b/docs/proposals/test-mode.md @@ -89,7 +89,7 @@ graph node `pre_built` reflects settings only (not updated after fallback). | -- | -- | -- | | Purpose | Source-build gap analysis | All matching versions | | On build failure | Pre-built fallback, keep traversing | Remove version from graph | -| Output | `test-mode-failures-*.json` | Logs only | +| Output | `test-mode-failures-*.json` | `partial-failures.json` | ## Usage @@ -97,7 +97,9 @@ graph node `pre_built` reflects settings only (not updated after fallback). fromager bootstrap --test-mode -r requirements.txt ``` -Review `test-mode-failures-*.json`, `build-order.json` (`source_url_type: "prebuilt"`), and `graph.json`. Tests: `tests/test_bootstrap_test_mode.py`, +Review `test-mode-failures-*.json`, `partial-failures.json` for failed +multi-version builds, `build-order.json` (`source_url_type: "prebuilt"`), and +`graph.json`. Tests: `tests/test_bootstrap_test_mode.py`, `e2e/test_mode_*.sh`. ## Limitations diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index 8d33ad89..a12e6753 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -124,7 +124,8 @@ def __init__( self.failed_packages: list[FailureRecord] = [] # Track failed versions in multiple_versions mode - self._failed_versions: dict[tuple[str, str], Exception] = {} + # Maps (package_name, version) -> (exception, detail) + self._failed_versions: dict[tuple[str, str], tuple[Exception, str]] = {} @property def resolver(self) -> bootstrap_requirement_resolver.BootstrapRequirementResolver: @@ -1001,6 +1002,13 @@ def _handle_phase_error( self.record_test_mode_failure( wi.req, str(wi.resolved_version), err, "bootstrap" ) + if self.multiple_versions: + self._record_failed_version( + wi.req, + str(wi.resolved_version), + err, + f"failed during {type(item).phase} phase", + ) return [] # Multiple versions mode: record failure, remove from graph, continue @@ -1035,7 +1043,7 @@ def _record_failed_version( ) -> None: """Record a version failure in multiple versions mode.""" pkg_name = canonicalize_name(req.name) - self._failed_versions[(pkg_name, version)] = err + self._failed_versions[(pkg_name, version)] = (err, detail) logger.warning( "%s==%s: %s: %s: %s", req.name, @@ -1048,9 +1056,39 @@ def _record_failed_version( def _log_failed_versions_table(self) -> None: """Log a summary table of all failed versions.""" logger.warning("%d version(s) failed to bootstrap:", len(self._failed_versions)) - for (name, ver), exc in self._failed_versions.items(): + for (name, ver), (exc, _detail) in self._failed_versions.items(): logger.warning(" %s==%s: %s: %s", name, ver, type(exc).__name__, exc) + def _write_partial_failures_report(self) -> None: + """Write a JSON report of multi-version bootstrap failures. + + Produces a ``partial-failures.json`` file in the work directory + so that downstream analysis tools can detect soft failures in + jobs that exited successfully. + """ + failures_file = self.ctx.work_dir / "partial-failures.json" + failures = sorted( + [ + { + "name": name if ver == "unresolved" else f"{name}=={ver}", + "version": None if ver == "unresolved" else ver, + "phase": detail, + "error_type": type(exc).__name__, + "message": str(exc), + } + for (name, ver), (exc, detail) in self._failed_versions.items() + ], + key=lambda f: f["name"] or "", + ) + with open(failures_file, "w") as f: + json.dump({"failures": failures}, f, indent=2) + f.write("\n") + logger.info( + "multi-version: wrote %d failure(s) to %s", + len(failures), + failures_file, + ) + def finalize(self) -> int: """Finalize bootstrap and return exit code. @@ -1089,6 +1127,9 @@ def finalize(self) -> int: if self.multiple_versions and self._failed_versions: self._log_failed_versions_table() + self._write_partial_failures_report() + else: + (self.ctx.work_dir / "partial-failures.json").unlink(missing_ok=True) if not self.test_mode: return 0 diff --git a/src/fromager/resolver.py b/src/fromager/resolver.py index 9f2cfa1b..d23819bb 100644 --- a/src/fromager/resolver.py +++ b/src/fromager/resolver.py @@ -1181,7 +1181,7 @@ def _find_tags( identifier: str, ) -> Iterable[Candidate]: headers = {"accept": "application/vnd.github+json"} - nexturl = self.api_url.format(self=self) + nexturl: str | None = self.api_url.format(self=self) while nexturl: resp = session.get(nexturl, headers=headers) resp.raise_for_status() @@ -1288,7 +1288,7 @@ def _find_tags( self, identifier: str, ) -> Iterable[Candidate]: - nexturl: str = self.api_url + nexturl: str | None = self.api_url created_at: datetime.datetime | None project_name = self.project_path.split("/")[-1] if self.override_download_url is None: diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 47c89402..1f3b2516 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -426,9 +426,10 @@ def prepare_source_run( pkg_name = canonicalize_name("testpkg") version_str = "1.5" assert (pkg_name, version_str) in bt._failed_versions - exc = bt._failed_versions[(pkg_name, version_str)] + exc, detail = bt._failed_versions[(pkg_name, version_str)] assert isinstance(exc, ValueError) assert str(exc) == "Simulated failure for version 1.5" + assert "failed during" in detail # Verify that failed version 1.5 is NOT in the dependency graph failed_key = f"{canonicalize_name('testpkg')}==1.5" @@ -1231,3 +1232,95 @@ def test_resolve_versions_rejects_url_requirement( bs = bootstrapper.Bootstrapper(tmp_context) with pytest.raises(ValueError, match="no longer supported"): bs.resolve_versions(req=req, req_type=RequirementType.TOP_LEVEL) + + +class TestPartialFailuresReport: + """Test partial-failures.json generation for multi-version mode.""" + + def test_finalize_writes_partial_failures(self, tmp_context: WorkContext) -> None: + bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) + bt._failed_versions = { + ("pkg-a", "1.0"): (ValueError("build failed"), "failed during build phase"), + ("pkg-b", "2.0"): (RuntimeError("timeout"), "failed to resolve"), + } + + assert bt.finalize() == 0 + + report = tmp_context.work_dir / "partial-failures.json" + assert report.exists() + data = json.loads(report.read_text()) + assert len(data["failures"]) == 2 + assert data["failures"][0] == { + "name": "pkg-a==1.0", + "version": "1.0", + "phase": "failed during build phase", + "error_type": "ValueError", + "message": "build failed", + } + assert data["failures"][1] == { + "name": "pkg-b==2.0", + "version": "2.0", + "phase": "failed to resolve", + "error_type": "RuntimeError", + "message": "timeout", + } + + def test_finalize_includes_unresolved_failures( + self, tmp_context: WorkContext + ) -> None: + bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) + bt._failed_versions = { + ("somepkg", "unresolved"): ( + RuntimeError("no versions found for somepkg"), + "failed to resolve", + ), + } + + assert bt.finalize() == 0 + + report = tmp_context.work_dir / "partial-failures.json" + assert report.exists() + data = json.loads(report.read_text()) + assert len(data["failures"]) == 1 + assert data["failures"][0] == { + "name": "somepkg", + "version": None, + "phase": "failed to resolve", + "error_type": "RuntimeError", + "message": "no versions found for somepkg", + } + + def test_finalize_no_report_when_no_failures( + self, tmp_context: WorkContext + ) -> None: + bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) + + bt.finalize() + + report = tmp_context.work_dir / "partial-failures.json" + assert not report.exists() + + def test_finalize_removes_stale_report(self, tmp_context: WorkContext) -> None: + """A clean run removes a partial-failures.json left by a previous run.""" + report = tmp_context.work_dir / "partial-failures.json" + report.write_text('{"failures": [{"name": "old==1.0"}]}') + + bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) + bt.finalize() + + assert not report.exists() + + def test_finalize_no_report_without_multiple_versions( + self, tmp_context: WorkContext + ) -> None: + report = tmp_context.work_dir / "partial-failures.json" + report.write_text('{"failures": [{"name": "stale==1.0"}]}') + + bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=False) + bt._failed_versions = { + ("pkg-a", "1.0"): (ValueError("build failed"), "failed during build phase"), + } + + bt.finalize() + + assert not report.exists() diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index 79ff439c..29fb89bf 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -489,8 +489,9 @@ def test_filters_failed_versions_in_multiple_versions_mode( bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) item = _make_resolve_item() - bt._failed_versions[(canonicalize_name("testpkg"), "2.0")] = RuntimeError( - "boom" + bt._failed_versions[(canonicalize_name("testpkg"), "2.0")] = ( + RuntimeError("boom"), + "failed during build phase", ) item.bg_future = _make_resolved_future( [ @@ -515,8 +516,9 @@ def test_failed_version_filter_does_not_apply_in_single_version_mode( bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=False) item = _make_resolve_item() - bt._failed_versions[(canonicalize_name("testpkg"), "1.0")] = RuntimeError( - "boom" + bt._failed_versions[(canonicalize_name("testpkg"), "1.0")] = ( + RuntimeError("boom"), + "failed during build phase", ) item.bg_future = _make_resolved_future([("url-1.0", Version("1.0"))]) @@ -532,8 +534,9 @@ def test_all_versions_failed_raises_runtime_error( bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) item = _make_resolve_item() - bt._failed_versions[(canonicalize_name("testpkg"), "1.0")] = RuntimeError( - "boom" + bt._failed_versions[(canonicalize_name("testpkg"), "1.0")] = ( + RuntimeError("boom"), + "failed during build phase", ) item.bg_future = _make_resolved_future([("url-1.0", Version("1.0"))]) @@ -880,7 +883,7 @@ def test_resolve_error_in_multiple_versions_mode_continues( assert len(bt._failed_versions) == 1 key = (canonicalize_name("testpkg"), "unresolved") assert key in bt._failed_versions - assert bt._failed_versions[key] is err + assert bt._failed_versions[key][0] is err def test_resolve_error_enriched_before_test_mode_record( self, tmp_context: WorkContext @@ -926,7 +929,7 @@ def test_resolve_error_enriched_before_multiple_versions_record( bt._handle_phase_error(item, err) key = (canonicalize_name("flashinfer-python"), "unresolved") - assert "dependency chain: its-hub==1.0" in str(bt._failed_versions[key]) + assert "dependency chain: its-hub==1.0" in str(bt._failed_versions[key][0]) def test_resolve_error_enriched_before_normal_mode_raise( self, tmp_context: WorkContext diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 00000000..85e394c1 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import logging + +import pytest +from packaging.requirements import Requirement +from packaging.version import Version + +from fromager import context, metrics + + +@metrics.timeit(description="test description") +def _test_func( + *, + ctx: context.WorkContext, + req: Requirement | None = None, + version: str | None = None, +) -> str: + return "ok" + + +@metrics.timeit(description="test description") +def _test_returns_version( + *, + ctx: context.WorkContext, + req: Requirement | None = None, +) -> tuple[str, Version]: + return ("http://example.com", Version("1.2.3")) + + +@metrics.timeit(description="test description") +def _test_raises( + *, + ctx: context.WorkContext, + req: Requirement | None = None, + version: str | None = None, +) -> None: + raise RuntimeError("test error") + + +def test_timeit_stores_timing(tmp_context: context.WorkContext) -> None: + req = Requirement("numpy>=1.0") + + _test_func(ctx=tmp_context, req=req, version="1.26.0") + + key = "numpy==1.26.0" + assert key in tmp_context.time_store + assert tmp_context.time_store[key]["_test_func"] > 0 + + +def test_timeit_stores_description(tmp_context: context.WorkContext) -> None: + _test_func(ctx=tmp_context) + + assert tmp_context.time_description_store["_test_func"] == "test description" + + +def test_timeit_no_storage_without_req(tmp_context: context.WorkContext) -> None: + _test_func(ctx=tmp_context, req=None, version="1.0") + + assert len(tmp_context.time_store) == 0 + + +def test_timeit_no_storage_without_version(tmp_context: context.WorkContext) -> None: + req = Requirement("numpy>=1.0") + + _test_func(ctx=tmp_context, req=req) + + assert len(tmp_context.time_store) == 0 + + +def test_timeit_returns_original_result(tmp_context: context.WorkContext) -> None: + result = _test_func(ctx=tmp_context) + + assert result == "ok" + + +def test_timeit_extracts_version_from_return( + tmp_context: context.WorkContext, +) -> None: + req = Requirement("mypkg") + + _test_returns_version(ctx=tmp_context, req=req) + + assert "mypkg==1.2.3" in tmp_context.time_store + + +def test_timeit_propagates_exception(tmp_context: context.WorkContext) -> None: + with pytest.raises(RuntimeError, match="test error"): + _test_raises(ctx=tmp_context) + + +def test_timeit_no_storage_on_exception(tmp_context: context.WorkContext) -> None: + req = Requirement("numpy>=1.0") + + with pytest.raises(RuntimeError): + _test_raises(ctx=tmp_context, req=req, version="1.0") + + assert len(tmp_context.time_store) == 0 + + +def test_summarize_logs_timing( + tmp_context: context.WorkContext, + caplog: pytest.LogCaptureFixture, +) -> None: + req = Requirement("numpy>=1.0") + _test_func(ctx=tmp_context, req=req, version="1.26.0") + + with caplog.at_level(logging.INFO, logger="fromager.metrics"): + metrics.summarize(tmp_context, "Building") + + records = [r for r in caplog.records if r.name == "fromager.metrics"] + assert len(records) == 1 + msg = records[0].message + assert "Building" in msg + assert "numpy==1.26.0" in msg + assert "test description" in msg + + +def test_summarize_empty( + tmp_context: context.WorkContext, + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.INFO, logger="fromager.metrics"): + metrics.summarize(tmp_context, "Building") + + assert len(caplog.records) == 0 + + +def test_extract_version_from_tuple() -> None: + ret = ("http://example.com", Version("2.0.0")) + assert metrics._extract_version_from_return(ret) == Version("2.0.0") + + +def test_extract_version_bare() -> None: + ret = Version("3.0.0") + assert metrics._extract_version_from_return(ret) == Version("3.0.0") + + +def test_extract_version_no_version_in_iterable() -> None: + ret = ("http://example.com", "not-a-version") + assert metrics._extract_version_from_return(ret) is None + + +def test_extract_version_non_iterable() -> None: + assert metrics._extract_version_from_return(42) is None + + +def test_extract_version_none() -> None: + assert metrics._extract_version_from_return(None) is None