From c6129a628e15aac4a4ba3db46d35b67865b1db3b Mon Sep 17 00:00:00 2001 From: "zackary.l.jackson" Date: Sun, 25 Oct 2026 21:49:57 +0000 Subject: [PATCH 1/2] fix(evals): pass the eval self-tests on native Windows Report ambient-tokio findings with POSIX paths, write the adoption report as UTF-8, run shell fixtures under Git's MSYS bash instead of the System32 WSL launcher, and print build-eval-binary paths in a form native Windows callers can open. The real-model runner owns agents through POSIX process groups, so its test is incompatible on Windows like evals/hermetic:run_test. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- evals/agent_adoption/grade.py | 8 ++++--- evals/agent_adoption/selftest.py | 14 ++++++++++-- evals/memory/BUILD.bazel | 5 ++++ scripts/build-eval-binary.sh | 8 ++++++- scripts/check-ambient-tokio.py | 2 +- scripts/test-build-eval-binary.py | 38 ++++++++++++++++++++++--------- 6 files changed, 57 insertions(+), 18 deletions(-) diff --git a/evals/agent_adoption/grade.py b/evals/agent_adoption/grade.py index ba222e68e2..aa40747c27 100755 --- a/evals/agent_adoption/grade.py +++ b/evals/agent_adoption/grade.py @@ -2134,10 +2134,12 @@ def main() -> int: with open(os.path.join(run_dir, "scoreboard.json"), "w") as f: json.dump(scoreboard, f, indent=2) f.write("\n") - with open(os.path.join(run_dir, "report.md"), "w") as f: - f.write(render_report(scoreboard)) + report = render_report(scoreboard) + with open(os.path.join(run_dir, "report.md"), "w", encoding="utf-8") as f: + f.write(report) - print(render_report(scoreboard)) + sys.stdout.reconfigure(encoding="utf-8") + print(report) return 0 diff --git a/evals/agent_adoption/selftest.py b/evals/agent_adoption/selftest.py index 319dfed126..e436f2c294 100644 --- a/evals/agent_adoption/selftest.py +++ b/evals/agent_adoption/selftest.py @@ -16,6 +16,7 @@ import json import os import re +import shutil import subprocess import sys import tempfile @@ -30,6 +31,15 @@ FAILURES: list[str] = [] +def bash() -> str: + """Git Bash on Windows, where the System32 bash.exe is the WSL launcher.""" + if os.name == "nt" and (git := shutil.which("git")): + git_bash = Path(git).resolve().parents[1] / "bin" / "bash.exe" + if git_bash.is_file(): + return str(git_bash) + return shutil.which("bash") or "bash" + + def check(name: str, cond: bool, extra: str = "") -> None: status = "ok " if cond else "FAIL" if not cond: @@ -932,7 +942,7 @@ def test_claude_permissions(): native = {"Read", "Write", "Edit", "Glob", "Grep", "Task", "Agent", "Skill"} for condition in grade.CONDITIONS: result = subprocess.run( - ["bash", "-c", 'source "$1" && claude_allowed_tools "$2" "$3"', + [bash(), "-c", 'source "$1" && claude_allowed_tools "$2" "$3"', "permissions", str(Path(HERE) / "claude_tools.sh"), condition, "/tmp/eval binary"], capture_output=True, text=True, ) @@ -1097,7 +1107,7 @@ def test_carry_claude_profile(): set(carried) == {"env", "model"} and carried["model"] == "k3[1m]" and carried["env"]["ANTHROPIC_BASE_URL"] == "https://example.invalid/") check("carry: copy is read-only", - oct(os.stat(dest).st_mode & 0o777) == "0o400") + oct(os.stat(dest).st_mode & 0o777) == ("0o444" if os.name == "nt" else "0o400")) with open(src, "w") as fh: json.dump({"permissions": {"allow": []}}, fh) dest2 = os.path.join(td, "none.json") diff --git a/evals/memory/BUILD.bazel b/evals/memory/BUILD.bazel index 5dd316b191..11d7f218bd 100644 --- a/evals/memory/BUILD.bazel +++ b/evals/memory/BUILD.bazel @@ -12,6 +12,11 @@ py_test( imports = ["."], legacy_create_init = 0, main = "test_run_real_model.py", + # The runner owns agents and daemons through POSIX process groups. + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), ) filegroup( diff --git a/scripts/build-eval-binary.sh b/scripts/build-eval-binary.sh index 2b2e82a804..d2bb232dbc 100755 --- a/scripts/build-eval-binary.sh +++ b/scripts/build-eval-binary.sh @@ -29,10 +29,15 @@ fail() { exit 2 } +# MSYS `pwd -W` keeps the path usable by native Windows callers. +absolute() { + echo "$(cd "$(dirname "$1")" && { pwd -W 2>/dev/null || pwd; })/$(basename "$1")" +} + if [[ -n "${TRACEDECAY_BIN:-}" ]]; then [[ -f "$TRACEDECAY_BIN" && -x "$TRACEDECAY_BIN" ]] || fail "TRACEDECAY_BIN=$TRACEDECAY_BIN is not an executable file; unset it to build the default with: $build_cmd" - bin="$(cd "$(dirname "$TRACEDECAY_BIN")" && pwd)/$(basename "$TRACEDECAY_BIN")" + bin="$(absolute "$TRACEDECAY_BIN")" origin="explicit TRACEDECAY_BIN" else command -v bazel >/dev/null 2>&1 || @@ -49,6 +54,7 @@ else bin="$bazel_bin/crates/tracedecay-cli/tracedecay${exe}" [[ -f "$bin" && -x "$bin" ]] || fail "expected a binary at $bin after: $build_cmd" + bin="$(absolute "$bin")" origin="bazel --config=$config $target" fi diff --git a/scripts/check-ambient-tokio.py b/scripts/check-ambient-tokio.py index fa735f5254..964c80dd9e 100755 --- a/scripts/check-ambient-tokio.py +++ b/scripts/check-ambient-tokio.py @@ -275,7 +275,7 @@ def check(root: Path) -> tuple[int, list[Finding]]: if resolved in skipped or any(parent in skipped_dirs for parent in resolved.parents): continue scanned += 1 - findings += check_file(str(path.relative_to(root)), source, code) + findings += check_file(path.relative_to(root).as_posix(), source, code) return scanned, findings diff --git a/scripts/test-build-eval-binary.py b/scripts/test-build-eval-binary.py index 9f6751f422..77badbaed4 100644 --- a/scripts/test-build-eval-binary.py +++ b/scripts/test-build-eval-binary.py @@ -11,6 +11,20 @@ SCRIPT = Path(__file__).with_name("build-eval-binary.sh") + +def bash() -> str: + """MSYS bash on Windows: System32 bash.exe is the WSL launcher, and Git's + bin/bash.exe wrapper puts its own tools ahead of the fakes on PATH.""" + if os.name == "nt" and (git := shutil.which("git")): + msys_bash = Path(git).resolve().parents[1] / "usr" / "bin" / "bash.exe" + if msys_bash.is_file(): + return str(msys_bash) + return shutil.which("bash") or "bash" + + +BASH = bash() + + # Fake bazel: records its argv; `build` exits $FAKE_BUILD_RC, `info` prints $FAKE_BAZEL_BIN. FAKE_BAZEL = """#!/bin/sh echo "$@" >> "$FAKE_BAZEL_LOG" @@ -34,24 +48,28 @@ def setUp(self): cli.mkdir(parents=True) # A binary from an earlier build, present before the build runs. self.built = cli / "tracedecay" - self.built.write_text("#!/bin/sh\necho tracedecay 1.2.3\n") + self.built.write_text("#!/bin/sh\necho tracedecay 1.2.3\n", newline="\n") self.built.chmod(0o755) self.log = root / "bazel.log" self.fakes = root / "fakes" fakes = self.fakes fakes.mkdir() bazel = fakes / "bazel" - bazel.write_text(FAKE_BAZEL) + bazel.write_text(FAKE_BAZEL, newline="\n") bazel.chmod(bazel.stat().st_mode | stat.S_IXUSR) + # Pin the host so the suffix choice does not follow the machine running the test. + uname = fakes / "uname" + uname.write_text("#!/bin/sh\necho Linux\n", newline="\n") + uname.chmod(0o755) self.env = { - "PATH": f"{fakes}:{os.environ['PATH']}", + "PATH": os.pathsep.join([str(fakes), str(Path(BASH).parent), os.environ["PATH"]]), "FAKE_BAZEL_LOG": str(self.log), "FAKE_BAZEL_BIN": str(self.bazel_bin), } def run_script(self, **env): return subprocess.run( - ["bash", str(self.repo / "scripts" / SCRIPT.name)], + [BASH, str(self.repo / "scripts" / SCRIPT.name)], env={**self.env, **env}, text=True, capture_output=True, @@ -61,7 +79,7 @@ def run_script(self, **env): def test_builds_the_shipping_target_and_prints_its_identity(self): result = self.run_script() self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), str(self.built)) + self.assertTrue(Path(result.stdout.strip()).samefile(self.built), result.stdout) self.assertIn( "build --config=release //crates/tracedecay-cli:tracedecay", self.log.read_text(), @@ -72,12 +90,10 @@ def test_builds_the_shipping_target_and_prints_its_identity(self): def test_windows_build_selects_the_executable_suffix(self): windows_binary = self.built.with_suffix(".exe") self.built.rename(windows_binary) - uname = self.fakes / "uname" - uname.write_text("#!/bin/sh\necho MINGW64_NT-10.0\n") - uname.chmod(0o755) + (self.fakes / "uname").write_text("#!/bin/sh\necho MINGW64_NT-10.0\n", newline="\n") result = self.run_script() self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), str(windows_binary)) + self.assertTrue(Path(result.stdout.strip()).samefile(windows_binary), result.stdout) def test_config_override_reaches_bazel(self): result = self.run_script(TRACEDECAY_EVAL_BAZEL_CONFIG="dist") @@ -92,11 +108,11 @@ def test_failed_build_does_not_fall_back_to_the_stale_binary(self): def test_explicit_binary_skips_the_build(self): explicit = self.built.with_name("candidate") - explicit.write_text("#!/bin/sh\necho tracedecay 9.9.9\n") + explicit.write_text("#!/bin/sh\necho tracedecay 9.9.9\n", newline="\n") explicit.chmod(0o755) result = self.run_script(TRACEDECAY_BIN=str(explicit)) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), str(explicit)) + self.assertTrue(Path(result.stdout.strip()).samefile(explicit), result.stdout) self.assertIn("tracedecay 9.9.9", result.stderr) self.assertFalse(self.log.exists()) From fadecbd811625735291394e88f5fd25de9aa2eef Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:17:54 -0700 Subject: [PATCH 2/2] fix(ci): bound Windows compiler concurrency --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e82ac6c34..dc96d97cb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -596,7 +596,9 @@ jobs: shell: bash # Compile and link every target, but defer unused test runfiles trees. # Bazel still materializes runfiles required by build-time tools. - run: bazel build --config=ci --nobuild_runfile_links //... + # Four concurrent Rust actions exhausted the hosted runner's commit + # limit while mapping asupersync metadata (Windows error 1455). + run: bazel build --config=ci --jobs=2 --nobuild_runfile_links //... - name: Build and test native targets if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_os }} @@ -619,7 +621,7 @@ jobs: *) echo "Expected a Bazel target pattern: $target" >&2; exit 1 ;; esac done - bazel test --config=ci --test_output=all "${targets[@]}" + bazel test --config=ci --jobs=2 --test_output=all "${targets[@]}" - name: Upload failed test diagnostics if: failure()