Updating new-resolver-config branch

docs/how-tos/multiple-version-bootstrap.rst

@@ -80,18 +80,47 @@ Output Files
 
 When ``--multiple-versions`` is active:
 
+.. versionchanged:: 0.95.0
+   Failed versions are recorded in ``partial-failures.json``. The file is
+   removed when a subsequent run has no reportable failures.
+
 - ``build-order.json`` --- created normally, listing every version built
 - ``graph.json`` --- created normally, containing all versions in the
   dependency graph
+- ``partial-failures.json`` --- created when one or more versions fail. Each
+  failure includes the package name, version, phase, exception type, and
+  message.
 - ``constraints.txt`` --- **not generated**
 
+The report contains a top-level ``failures`` list. For a resolved version,
+``name`` includes the package version (for example, ``requests==2.28.0``),
+and ``version`` contains the version separately. For an unresolved
+requirement, ``version`` is ``null`` and ``name`` contains only the package
+name.
+
+.. code-block:: json
+
+   {
+     "failures": [
+       {
+         "name": "requests==2.28.0",
+         "version": "2.28.0",
+         "phase": "failed during build phase",
+         "error_type": "CalledProcessError",
+         "message": "build command failed"
+       }
+     ]
+   }
+
 Combining with Other Flags
 --------------------------
 
 ``--test-mode``
   Supported with serial ``bootstrap`` only. Failures are collected and
   reported at the end rather than aborting early (same behavior as without
-  ``--multiple-versions``).
+  ``--multiple-versions``). When combined with ``--multiple-versions``,
+  failed versions are also included in ``partial-failures.json`` and test
+  mode retains its non-zero exit code for recorded failures.
 
 ``--skip-constraints``
   Redundant when ``--multiple-versions`` is set. Fromager automatically

docs/proposals/test-mode.md

@@ -89,15 +89,17 @@ graph node `pre_built` reflects settings only (not updated after fallback).
 | -- | -- | -- |
 | Purpose | Source-build gap analysis | All matching versions |
 | On build failure | Pre-built fallback, keep traversing | Remove version from graph |
-| Output | `test-mode-failures-*.json` | Logs only |
+| Output | `test-mode-failures-*.json` | `partial-failures.json` |
 
 ## Usage
 
 ```bash
 fromager bootstrap --test-mode -r requirements.txt
 ```
 
-Review `test-mode-failures-*.json`, `build-order.json` (`source_url_type: "prebuilt"`), and `graph.json`. Tests: `tests/test_bootstrap_test_mode.py`,
+Review `test-mode-failures-*.json`, `partial-failures.json` for failed
+multi-version builds, `build-order.json` (`source_url_type: "prebuilt"`), and
+`graph.json`. Tests: `tests/test_bootstrap_test_mode.py`,
 `e2e/test_mode_*.sh`.
 
 ## Limitations

src/fromager/bootstrapper/_bootstrapper.py

@@ -124,7 +124,8 @@ def __init__(
         self.failed_packages: list[FailureRecord] = []
 
         # Track failed versions in multiple_versions mode
-        self._failed_versions: dict[tuple[str, str], Exception] = {}
+        # Maps (package_name, version) -> (exception, detail)
+        self._failed_versions: dict[tuple[str, str], tuple[Exception, str]] = {}
 
     @property
     def resolver(self) -> bootstrap_requirement_resolver.BootstrapRequirementResolver:
@@ -1001,6 +1002,13 @@ def _handle_phase_error(
             self.record_test_mode_failure(
                 wi.req, str(wi.resolved_version), err, "bootstrap"
             )
+            if self.multiple_versions:
+                self._record_failed_version(
+                    wi.req,
+                    str(wi.resolved_version),
+                    err,
+                    f"failed during {type(item).phase} phase",
+                )
             return []
 
         # Multiple versions mode: record failure, remove from graph, continue
@@ -1035,7 +1043,7 @@ def _record_failed_version(
     ) -> None:
         """Record a version failure in multiple versions mode."""
         pkg_name = canonicalize_name(req.name)
-        self._failed_versions[(pkg_name, version)] = err
+        self._failed_versions[(pkg_name, version)] = (err, detail)
         logger.warning(
             "%s==%s: %s: %s: %s",
             req.name,
@@ -1048,9 +1056,39 @@ def _record_failed_version(
     def _log_failed_versions_table(self) -> None:
         """Log a summary table of all failed versions."""
         logger.warning("%d version(s) failed to bootstrap:", len(self._failed_versions))
-        for (name, ver), exc in self._failed_versions.items():
+        for (name, ver), (exc, _detail) in self._failed_versions.items():
             logger.warning("  %s==%s: %s: %s", name, ver, type(exc).__name__, exc)
 
+    def _write_partial_failures_report(self) -> None:
+        """Write a JSON report of multi-version bootstrap failures.
+
+        Produces a ``partial-failures.json`` file in the work directory
+        so that downstream analysis tools can detect soft failures in
+        jobs that exited successfully.
+        """
+        failures_file = self.ctx.work_dir / "partial-failures.json"
+        failures = sorted(
+            [
+                {
+                    "name": name if ver == "unresolved" else f"{name}=={ver}",
+                    "version": None if ver == "unresolved" else ver,
+                    "phase": detail,
+                    "error_type": type(exc).__name__,
+                    "message": str(exc),
+                }
+                for (name, ver), (exc, detail) in self._failed_versions.items()
+            ],
+            key=lambda f: f["name"] or "",
+        )
+        with open(failures_file, "w") as f:
+            json.dump({"failures": failures}, f, indent=2)
+            f.write("\n")
+        logger.info(
+            "multi-version: wrote %d failure(s) to %s",
+            len(failures),
+            failures_file,
+        )
+
     def finalize(self) -> int:
         """Finalize bootstrap and return exit code.
 
@@ -1089,6 +1127,9 @@ def finalize(self) -> int:
 
         if self.multiple_versions and self._failed_versions:
             self._log_failed_versions_table()
+            self._write_partial_failures_report()
+        else:
+            (self.ctx.work_dir / "partial-failures.json").unlink(missing_ok=True)
 
         if not self.test_mode:
             return 0

src/fromager/resolver.py

@@ -1181,7 +1181,7 @@ def _find_tags(
         identifier: str,
     ) -> Iterable[Candidate]:
         headers = {"accept": "application/vnd.github+json"}
-        nexturl = self.api_url.format(self=self)
+        nexturl: str | None = self.api_url.format(self=self)
         while nexturl:
             resp = session.get(nexturl, headers=headers)
             resp.raise_for_status()
@@ -1288,7 +1288,7 @@ def _find_tags(
         self,
         identifier: str,
     ) -> Iterable[Candidate]:
-        nexturl: str = self.api_url
+        nexturl: str | None = self.api_url
         created_at: datetime.datetime | None
         project_name = self.project_path.split("/")[-1]
         if self.override_download_url is None:

tests/test_bootstrapper.py

@@ -426,9 +426,10 @@ def prepare_source_run(
                 pkg_name = canonicalize_name("testpkg")
                 version_str = "1.5"
                 assert (pkg_name, version_str) in bt._failed_versions
-                exc = bt._failed_versions[(pkg_name, version_str)]
+                exc, detail = bt._failed_versions[(pkg_name, version_str)]
                 assert isinstance(exc, ValueError)
                 assert str(exc) == "Simulated failure for version 1.5"
+                assert "failed during" in detail
 
         # Verify that failed version 1.5 is NOT in the dependency graph
         failed_key = f"{canonicalize_name('testpkg')}==1.5"
@@ -1231,3 +1232,95 @@ def test_resolve_versions_rejects_url_requirement(
     bs = bootstrapper.Bootstrapper(tmp_context)
     with pytest.raises(ValueError, match="no longer supported"):
         bs.resolve_versions(req=req, req_type=RequirementType.TOP_LEVEL)
+
+
+class TestPartialFailuresReport:
+    """Test partial-failures.json generation for multi-version mode."""
+
+    def test_finalize_writes_partial_failures(self, tmp_context: WorkContext) -> None:
+        bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True)
+        bt._failed_versions = {
+            ("pkg-a", "1.0"): (ValueError("build failed"), "failed during build phase"),
+            ("pkg-b", "2.0"): (RuntimeError("timeout"), "failed to resolve"),
+        }
+
+        assert bt.finalize() == 0
+
+        report = tmp_context.work_dir / "partial-failures.json"
+        assert report.exists()
+        data = json.loads(report.read_text())
+        assert len(data["failures"]) == 2
+        assert data["failures"][0] == {
+            "name": "pkg-a==1.0",
+            "version": "1.0",
+            "phase": "failed during build phase",
+            "error_type": "ValueError",
+            "message": "build failed",
+        }
+        assert data["failures"][1] == {
+            "name": "pkg-b==2.0",
+            "version": "2.0",
+            "phase": "failed to resolve",
+            "error_type": "RuntimeError",
+            "message": "timeout",
+        }
+
+    def test_finalize_includes_unresolved_failures(
+        self, tmp_context: WorkContext
+    ) -> None:
+        bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True)
+        bt._failed_versions = {
+            ("somepkg", "unresolved"): (
+                RuntimeError("no versions found for somepkg"),
+                "failed to resolve",
+            ),
+        }
+
+        assert bt.finalize() == 0
+
+        report = tmp_context.work_dir / "partial-failures.json"
+        assert report.exists()
+        data = json.loads(report.read_text())
+        assert len(data["failures"]) == 1
+        assert data["failures"][0] == {
+            "name": "somepkg",
+            "version": None,
+            "phase": "failed to resolve",
+            "error_type": "RuntimeError",
+            "message": "no versions found for somepkg",
+        }
+
+    def test_finalize_no_report_when_no_failures(
+        self, tmp_context: WorkContext
+    ) -> None:
+        bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True)
+
+        bt.finalize()
+
+        report = tmp_context.work_dir / "partial-failures.json"
+        assert not report.exists()
+
+    def test_finalize_removes_stale_report(self, tmp_context: WorkContext) -> None:
+        """A clean run removes a partial-failures.json left by a previous run."""
+        report = tmp_context.work_dir / "partial-failures.json"
+        report.write_text('{"failures": [{"name": "old==1.0"}]}')
+
+        bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True)
+        bt.finalize()
+
+        assert not report.exists()
+
+    def test_finalize_no_report_without_multiple_versions(
+        self, tmp_context: WorkContext
+    ) -> None:
+        report = tmp_context.work_dir / "partial-failures.json"
+        report.write_text('{"failures": [{"name": "stale==1.0"}]}')
+
+        bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=False)
+        bt._failed_versions = {
+            ("pkg-a", "1.0"): (ValueError("build failed"), "failed during build phase"),
+        }
+
+        bt.finalize()
+
+        assert not report.exists()

tests/test_bootstrapper_iterative.py

@@ -489,8 +489,9 @@ def test_filters_failed_versions_in_multiple_versions_mode(
         bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True)
         item = _make_resolve_item()
 
-        bt._failed_versions[(canonicalize_name("testpkg"), "2.0")] = RuntimeError(
-            "boom"
+        bt._failed_versions[(canonicalize_name("testpkg"), "2.0")] = (
+            RuntimeError("boom"),
+            "failed during build phase",
         )
         item.bg_future = _make_resolved_future(
             [
@@ -515,8 +516,9 @@ def test_failed_version_filter_does_not_apply_in_single_version_mode(
         bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=False)
         item = _make_resolve_item()
 
-        bt._failed_versions[(canonicalize_name("testpkg"), "1.0")] = RuntimeError(
-            "boom"
+        bt._failed_versions[(canonicalize_name("testpkg"), "1.0")] = (
+            RuntimeError("boom"),
+            "failed during build phase",
         )
         item.bg_future = _make_resolved_future([("url-1.0", Version("1.0"))])
 
@@ -532,8 +534,9 @@ def test_all_versions_failed_raises_runtime_error(
         bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True)
         item = _make_resolve_item()
 
-        bt._failed_versions[(canonicalize_name("testpkg"), "1.0")] = RuntimeError(
-            "boom"
+        bt._failed_versions[(canonicalize_name("testpkg"), "1.0")] = (
+            RuntimeError("boom"),
+            "failed during build phase",
         )
         item.bg_future = _make_resolved_future([("url-1.0", Version("1.0"))])
 
@@ -880,7 +883,7 @@ def test_resolve_error_in_multiple_versions_mode_continues(
         assert len(bt._failed_versions) == 1
         key = (canonicalize_name("testpkg"), "unresolved")
         assert key in bt._failed_versions
-        assert bt._failed_versions[key] is err
+        assert bt._failed_versions[key][0] is err
 
     def test_resolve_error_enriched_before_test_mode_record(
         self, tmp_context: WorkContext
@@ -926,7 +929,7 @@ def test_resolve_error_enriched_before_multiple_versions_record(
         bt._handle_phase_error(item, err)
 
         key = (canonicalize_name("flashinfer-python"), "unresolved")
-        assert "dependency chain: its-hub==1.0" in str(bt._failed_versions[key])
+        assert "dependency chain: its-hub==1.0" in str(bt._failed_versions[key][0])
 
     def test_resolve_error_enriched_before_normal_mode_raise(
         self, tmp_context: WorkContext