From e5f36e8520e20d6774596e98bff39c9e2ba18e52 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 28 Aug 2026 02:16:07 -0700 Subject: [PATCH] fix(native-eval): separate execution validity from diagnostic rewards Integrate PR #64 on current main and retain its structured outcomes, run acceptance, legacy classification, and fail-closed recovery. Archive the terminal job exit code before packaging so recovery can prove completion. Keep the first exception for diagnostics while classifying later verifier or infrastructure failures for acceptance. Cover real shell exports for exits 0 and 2, post-agent failures, aggregation validity, and explicit versus unknown recovery status. Fixes #63 Co-authored-by: Vincent Koc --- CHANGELOG.md | 1 + scripts/native_eval/aggregate.py | 102 ++++++++++++++++++++++++-- scripts/native_eval/fleet.py | 23 ------ scripts/native_eval/remote_run.sh | 1 + scripts/native_eval/run_job.py | 35 +++++++++ scripts/native_eval/runtime.py | 64 +++++++++++++++- tests/test_native_eval_aggregate.py | 74 +++++++++++++++++++ tests/test_native_eval_fleet.py | 109 +++++++++++++++++++++++---- tests/test_native_eval_runner.py | 110 ++++++++++++++++++++++++++++ 9 files changed, 476 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9cbd74..ccbe8a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - Rehydrate replacement native fleet leases instead of trusting stale bootstrap timestamps (#58, thanks @vincentkoc). +- Separate native execution validity from diagnostic rewards, reject wholly invalid runs, and preserve terminal exit status in recovery archives (#63, thanks @vincentkoc). - Install only the assigned native harness on each fleet lease and record only its version in the toolchain manifest (#65). - Refresh Python runtime, MLflow, lint, and HF mirror dependencies, including websockets 17 with a real gateway socket regression test, while preserving Python 3.11 NumPy support. - Record and enforce GPT reasoning effort in native runs, preserve it across diff --git a/scripts/native_eval/aggregate.py b/scripts/native_eval/aggregate.py index d316090..0ebf428 100644 --- a/scripts/native_eval/aggregate.py +++ b/scripts/native_eval/aggregate.py @@ -91,6 +91,9 @@ "trial_name", "result_path", "classification", + "execution_outcome", + "execution_exit_code", + "execution_reason", "reward", "exception_type", "exception_message", @@ -142,6 +145,8 @@ "infra_dominated", "harness_wide_failure", "harness_wide_failure_signature", + "run_accepted", + "run_acceptance_reason", "canonical_model_identity", "trajectory_complete", "parity_validated", @@ -278,8 +283,18 @@ def _is_infra(exception_type: str, exception_message: str) -> bool: return any(pattern in combined for pattern in INFRA_MESSAGE_PATTERNS) -def _classify(result: dict[str, Any], reward: int | float | None) -> str: +def _classify( + result: dict[str, Any], + reward: int | float | None, + execution_kind: str, +) -> str: exception_type, exception_message, _ = _exception_fields(result) + if execution_kind in {"harness_error", "infra_error"}: + return "infra" + if execution_kind == "verifier_error": + return "verifier_missing_reward" + if execution_kind == "agent_error": + return "agent_exit" if exception_type in MISSING_REWARD_EXCEPTION_TYPES: return "verifier_missing_reward" if exception_type: @@ -295,6 +310,40 @@ def _classify(result: dict[str, Any], reward: int | float | None) -> str: return "pass" +def _execution_fields( + result: dict[str, Any], + *, + harness: str, +) -> tuple[str, int | None, str]: + outcome = result.get("execution_outcome") + if isinstance(outcome, dict): + kind = str(outcome.get("kind") or "") + if kind and kind != "pending": + exit_code = _number(outcome.get("exit_code")) + return ( + kind, + int(exit_code) if exit_code is not None else None, + str(outcome.get("reason") or ""), + ) + + exception_type, exception_message, _ = _exception_fields(result) + exit_match = re.search(r"\bcode\s+(\d+)\b", exception_message) + exit_code = int(exit_match.group(1)) if exit_match else None + if ( + harness == "openclaw" + and exception_type == "NonZeroAgentExitCodeError" + and exit_code in {70, 71} + ): + return "harness_error", exit_code, exception_message + if exception_type in MISSING_REWARD_EXCEPTION_TYPES: + return "verifier_error", exit_code, exception_message + if exception_type and _is_infra(exception_type, exception_message): + return "infra_error", exit_code, exception_message + if exception_type: + return "agent_error", exit_code, exception_message + return "clean", None, "" + + def _scorecard_infra_error(trial_dir: Path) -> tuple[str, str] | None: scorecard_path = trial_dir / "verifier" / "scorecard.json" if not scorecard_path.is_file(): @@ -525,6 +574,7 @@ def _normalize_result( run_label: str, pair_label: str, repetition: int | None, + harness: str, ) -> dict[str, Any]: task_name, task_path, trial_name = _task_identity(result, result_path.parent) reward = _reward(result) @@ -532,6 +582,13 @@ def _normalize_result( scorecard_infra = _scorecard_infra_error(result_path.parent) if scorecard_infra is not None: exception_type, exception_message = scorecard_infra + execution_kind, execution_exit_code, execution_reason = _execution_fields( + result, + harness=harness, + ) + if scorecard_infra is not None: + execution_kind = "infra_error" + execution_reason = exception_message agent_result = result.get("agent_result") agent_result = agent_result if isinstance(agent_result, dict) else {} row: dict[str, Any] = { @@ -543,8 +600,13 @@ def _normalize_result( "trial_name": trial_name, "result_path": str(result_path), "classification": ( - "infra" if scorecard_infra is not None else _classify(result, reward) + "infra" + if scorecard_infra is not None + else _classify(result, reward, execution_kind) ), + "execution_outcome": execution_kind, + "execution_exit_code": execution_exit_code, + "execution_reason": execution_reason, "reward": reward, "exception_type": exception_type, "exception_message": exception_message, @@ -654,6 +716,7 @@ def _load_run( run_label = str(manifest.get("run_label") or run_label) pair_label = _pair_label(manifest, run_label) repetition = _repetition(manifest, run_label) + harness = str(manifest.get("harness") or "") rows: list[dict[str, Any]] = [] trial_dirs = sorted(path for path in run_dir.iterdir() if path.is_dir()) @@ -699,6 +762,7 @@ def _load_run( run_label=run_label, pair_label=pair_label, repetition=repetition, + harness=harness, ) ) @@ -840,6 +904,10 @@ def _summarize_run( classifications = [str(row["classification"]) for row in scored_rows] rewards = [_number(row.get("reward")) for row in scored_rows] all_classifications = [str(row["classification"]) for row in rows] + execution_kinds = [ + str(row.get("execution_outcome") or "") + for row in scored_rows + ] result_file_count = sum(bool(row.get("_has_result_file")) for row in rows) valid_result_count = sum(bool(row.get("_valid_result")) for row in rows) completed_result_count = sum(bool(row.get("_completed_result")) for row in rows) @@ -874,6 +942,25 @@ def _summarize_run( rows, expected_count, ) + acceptance = manifest.get("execution_acceptance") + acceptance = acceptance if isinstance(acceptance, dict) else {} + accepted_value = acceptance.get("accepted") + if isinstance(accepted_value, bool): + run_accepted = accepted_value + else: + invalid_execution_kinds = { + "harness_error", + "infra_error", + "verifier_error", + } + run_accepted = not ( + expected_count > 0 + and len(execution_kinds) == expected_count + and all(kind in invalid_execution_kinds for kind in execution_kinds) + ) + run_acceptance_reason = str(acceptance.get("reason") or "") + if not run_accepted and not run_acceptance_reason: + run_acceptance_reason = "all_trials_harness_or_infrastructure_errors" native_run = ( manifest.get("runner") == "shellbench-native" or any(row.get("source") == "shellbench-native" for row in rows) @@ -917,6 +1004,7 @@ def _summarize_run( ) = _parity_validation(manifest, native_run=native_run) eligible = ( not incomplete + and run_accepted and not infra_dominated and not harness_wide_failure and canonical_model_identity @@ -926,6 +1014,8 @@ def _summarize_run( exclusion_reason = "incomplete" elif infra_dominated: exclusion_reason = "infra_dominated" + elif not run_accepted: + exclusion_reason = "run_rejected" elif harness_wide_failure: exclusion_reason = "harness_wide_failure" elif not canonical_model_identity: @@ -962,6 +1052,8 @@ def _summarize_run( "infra_dominated": infra_dominated, "harness_wide_failure": harness_wide_failure, "harness_wide_failure_signature": harness_wide_failure_signature, + "run_accepted": run_accepted, + "run_acceptance_reason": run_acceptance_reason, "canonical_model_identity": canonical_model_identity, "trajectory_complete": trajectory_complete, "parity_validated": parity_validated, @@ -1184,9 +1276,9 @@ def _write_leaderboard(path: Path, pairs: Sequence[dict[str, Any]]) -> None: lines = [ "# Cleaned Native ShellBench Leaderboard", "", - "Incomplete, infra-dominated, harness-wide failure, identity-invalid, and " - "trajectory-incomplete repetitions are excluded. Harbor parity validation is " - "reported separately and is not an eligibility gate.", + "Incomplete, rejected, infra-dominated, harness-wide failure, identity-invalid, " + "and trajectory-incomplete repetitions are excluded. Harbor parity validation " + "is reported separately and is not an eligibility gate.", "", "| Rank | Pair | Repetitions | Parity validated | Mean | Stdev | Min | Max | Mean exact passes | Pass rate | Clean complete |", "| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 2624b1c..b258495 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -1250,16 +1250,6 @@ def _archived_exit_status(self, run_label: str) -> int | None: raise FleetError(f"conflicting archived exit_status values for {run_label}") return candidates[0] if candidates else None - def _checkpoint_log_has_final(self, run_label: str) -> bool: - log_path = self.config.local_root / "logs" / f"{run_label}.checkpoints.log" - if not log_path.is_file(): - return False - for line in log_path.read_text(encoding="utf-8").splitlines(): - fields = line.split("\t", 4) - if len(fields) >= 2 and fields[1] == "final": - return True - return False - def _finish_exported(self, entry: dict[str, Any], run: RunSpec) -> bool: verified, result_count, artifacts = self._verify_final(run.run_label) if not verified: @@ -1286,19 +1276,6 @@ def _finish_exported(self, entry: dict[str, Any], run: RunSpec) -> bool: "run_exit_code": run_exit_code, "run_exit_code_source": "archived_exit_status", } - elif ( - result_count == run.expected_task_count - and self._checkpoint_log_has_final(run.run_label) - ): - run_exit_code = 0 - exit_code_changes = { - "run_exit_code": 0, - "run_exit_code_source": "recovered_full_coverage_remote_done", - "run_exit_code_inference": ( - "inferred zero from verified full-coverage archive and " - "checkpoint final event after remote done" - ), - } stop_command = [ self.config.crabbox_bin, "stop", diff --git a/scripts/native_eval/remote_run.sh b/scripts/native_eval/remote_run.sh index 91c6f3c..0a4976a 100755 --- a/scripts/native_eval/remote_run.sh +++ b/scripts/native_eval/remote_run.sh @@ -141,6 +141,7 @@ RUN_STATUS="$?" set -e META_DIR="/tmp/shellbench_meta-$RUN_LABEL" +printf '%s\n' "$RUN_STATUS" > "$META_DIR/exit_status" date -u +%Y-%m-%dT%H:%M:%SZ > "$META_DIR/exported_at_utc.txt" hostname > "$META_DIR/hostname.txt" git -C "$ROOT/runner" rev-parse HEAD > "$META_DIR/runner_commit.txt" || true diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index ccafe5e..d1d8ab4 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -6,6 +6,7 @@ import os import subprocess import uuid +from collections import Counter from dataclasses import asdict from pathlib import Path from typing import Any @@ -127,9 +128,12 @@ async def execute(task: TaskSpec) -> None: state["finished_at"] = utc_now() state["updated_at"] = state["finished_at"] _update_job_result(state, results, len(tasks)) + acceptance = _execution_acceptance(results, len(tasks)) + state["execution_acceptance"] = acceptance atomic_write_json(job_dir / "result.json", state) manifest["finished_at_utc"] = state["finished_at"] manifest["result_json_count"] = len(results) + manifest["execution_acceptance"] = acceptance agent_results = [ result.get("agent_result") for result in results @@ -204,6 +208,30 @@ def _update_job_result( state["updated_at"] = utc_now() +def _execution_acceptance( + results: list[dict[str, Any]], + total: int, +) -> dict[str, Any]: + kinds = Counter( + str((result.get("execution_outcome") or {}).get("kind") or "unknown") + for result in results + ) + invalid_kinds = {"harness_error", "infra_error", "verifier_error"} + invalid_count = sum(kinds[kind] for kind in invalid_kinds) + accepted = len(results) == total and invalid_count < total + if len(results) != total: + reason = "incomplete_result_coverage" + elif invalid_count == total and total: + reason = "all_trials_harness_or_infrastructure_errors" + else: + reason = None + return { + "accepted": accepted, + "reason": reason, + "outcome_counts": dict(sorted(kinds.items())), + } + + def _run_manifest( run: RunSpec, *, @@ -293,6 +321,11 @@ def _run_manifest( "started_at_utc": started_at, "finished_at_utc": None, "result_json_count": 0, + "execution_acceptance": { + "accepted": None, + "reason": "run_in_progress", + "outcome_counts": {}, + }, } @@ -434,6 +467,8 @@ def main() -> None: ) ) print(json.dumps(state, indent=2)) + if state["execution_acceptance"]["accepted"] is not True: + raise SystemExit(2) if __name__ == "__main__": diff --git a/scripts/native_eval/runtime.py b/scripts/native_eval/runtime.py index f26f693..1517a6d 100644 --- a/scripts/native_eval/runtime.py +++ b/scripts/native_eval/runtime.py @@ -52,6 +52,12 @@ class RewardFileNotFoundError(NativeEvalError): pass +OPENCLAW_HARNESS_EXIT_REASONS = { + 70: "gateway_start_failed", + 71: "terminal_session_evidence_unavailable", +} + + CODEX_DIAGNOSTIC_LINE = re.compile( r"^\d{4}-\d{2}-\d{2}T\S+\s+" r"(?:TRACE|DEBUG|INFO|WARN|ERROR)\s+codex[\w:.-]*:" @@ -473,6 +479,8 @@ async def run_trial( toolchain_root=toolchain_root, ) recorded_exception: BaseException | None = None + execution_exception: BaseException | None = None + agent_exit_code: int | None = None agent_command = build_harness_command( run, proxy_url=proxy_url, @@ -517,6 +525,7 @@ async def run_trial( stderr_path=trial_dir / "agent" / "stderr.txt", ) if agent.returncode: + agent_exit_code = agent.returncode raise NonZeroAgentExitCodeError( f"Agent exited with code {agent.returncode}" ) @@ -570,20 +579,30 @@ async def run_trial( } reward = read_reward(trial_dir / "verifier") result["verifier_result"] = {"rewards": reward} - if verifier.returncode and recorded_exception is None and not reward: - recorded_exception = RewardFileNotFoundError( + if verifier.returncode and not reward: + raise RewardFileNotFoundError( f"Verifier exited {verifier.returncode} without a reward" ) except BaseException as exc: + execution_exception = exc if recorded_exception is None: recorded_exception = exc finally: try: await environment.stop() except Exception as exc: + if execution_exception is None: + execution_exception = exc if recorded_exception is None: recorded_exception = exc result["finished_at"] = utc_now() + result["execution_outcome"] = execution_outcome( + harness=run.harness, + # Keep the first diagnostic exception, but do not let an agent exit + # hide a later verifier or infrastructure failure from acceptance. + exception=execution_exception or recorded_exception, + agent_exit_code=agent_exit_code, + ) if recorded_exception is not None: result["exception_info"] = exception_info(recorded_exception) (trial_dir / "exception.txt").write_text( @@ -640,6 +659,11 @@ def _initial_trial_result( }, "agent_result": None, "verifier_result": None, + "execution_outcome": { + "kind": "pending", + "exit_code": None, + "reason": None, + }, "verifier_environment_mode": "shared", "exception_info": None, "started_at": started_at, @@ -1287,6 +1311,42 @@ def exception_info(exc: BaseException) -> dict[str, str]: } +def execution_outcome( + *, + harness: str, + exception: BaseException | None, + agent_exit_code: int | None, +) -> dict[str, Any]: + if exception is None: + return {"kind": "clean", "exit_code": None, "reason": None} + if ( + harness == "openclaw" + and agent_exit_code in OPENCLAW_HARNESS_EXIT_REASONS + ): + return { + "kind": "harness_error", + "exit_code": agent_exit_code, + "reason": OPENCLAW_HARNESS_EXIT_REASONS[agent_exit_code], + } + if isinstance(exception, NonZeroAgentExitCodeError): + return { + "kind": "agent_error", + "exit_code": agent_exit_code, + "reason": str(exception), + } + if isinstance(exception, (AgentSetupError, AgentSetupTimeoutError)): + kind = "harness_error" + elif isinstance(exception, RewardFileNotFoundError): + kind = "verifier_error" + else: + kind = "infra_error" + return { + "kind": kind, + "exit_code": agent_exit_code, + "reason": str(exception), + } + + def _timing(result: CommandResult) -> dict[str, str]: return { "started_at": result.started_at, diff --git a/tests/test_native_eval_aggregate.py b/tests/test_native_eval_aggregate.py index c1e26d1..dff1291 100644 --- a/tests/test_native_eval_aggregate.py +++ b/tests/test_native_eval_aggregate.py @@ -509,6 +509,80 @@ def test_native_identity_checks_ignore_unavailable_agent_exit_trajectory( assert run["eligible"] is True +@pytest.mark.parametrize( + ("kinds", "accepted"), + [ + (("clean", "clean"), True), + (("agent_error", "clean"), True), + (("harness_error", "clean"), True), + (("verifier_error", "verifier_error"), False), + (("harness_error", "infra_error"), False), + ], +) +def test_structured_execution_validity_preserves_diagnostic_scores( + tmp_path: Path, kinds: tuple[str, str], accepted: bool +) -> None: + results = [] + for index, kind in enumerate(kinds): + result = _native_result(str(index), reward=0.5) + result["execution_outcome"] = {"kind": kind, "exit_code": None, "reason": kind} + results.append(result) + jobs_root = tmp_path / "native" + _write_run( + jobs_root, "structured-execution", expected_task_count=2, + results=results, native=True, + ) + + run = aggregate(jobs_root, tmp_path / "summaries")["runs"][0] + + assert run["score"] == pytest.approx(0.5) + assert run["nonzero"] == 2 + assert run["run_accepted"] is accepted + assert run["eligible"] is accepted + + +def test_all_openclaw_harness_errors_preserve_rewards_but_reject_run( + tmp_path: Path, +): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + _write_run( + jobs_root, + "openclaw-gpt55-terminal-evidence-failure", + expected_task_count=2, + results=[ + _result( + "a", + reward=0.25, + exception_type="NonZeroAgentExitCodeError", + exception_message="Agent exited with code 71", + ), + _result( + "b", + reward=0.75, + exception_type="NonZeroAgentExitCodeError", + exception_message="Agent exited with code 71", + ), + ], + harness="openclaw", + ) + + report = aggregate(jobs_root, summaries_dir) + + run = report["runs"][0] + assert run["score"] == pytest.approx(0.5) + assert run["nonzero"] == 2 + assert run["run_accepted"] is False + assert run["run_acceptance_reason"] == ( + "all_trials_harness_or_infrastructure_errors" + ) + assert run["eligible"] is False + assert run["exclusion_reason"] == "infra_dominated" + with (summaries_dir / "per_task_results.csv").open(newline="") as handle: + rows = list(csv.DictReader(handle)) + assert {row["execution_outcome"] for row in rows} == {"harness_error"} + + def test_native_identity_checks_real_agent_exit_trajectory(tmp_path: Path): jobs_root = tmp_path / "native" summaries_dir = tmp_path / "summaries" diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 4d9c605..aee94cf 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -1,12 +1,15 @@ from __future__ import annotations import json +import os import shlex +import shutil import subprocess import tarfile import threading from pathlib import Path from typing import Sequence +from uuid import uuid4 import pytest @@ -80,6 +83,84 @@ def _planned(run: RunSpec) -> dict[str, object]: } +@pytest.mark.parametrize("exit_code", [0, 2]) +def test_remote_run_archives_terminal_status(tmp_path: Path, exit_code: int) -> None: + bash = shutil.which("bash") + assert bash is not None + if subprocess.run( + [ + bash, "-c", + "(( BASH_VERSINFO[0] > 4 || " + "(BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] >= 4) ))", + ], + check=False, + ).returncode: + pytest.skip("remote_run.sh requires modern Bash") + label = f"shellbench-test-{uuid4().hex}" + root = tmp_path / "remote" + (root / "runner").mkdir(parents=True) + toolchain = tmp_path / "toolchain" + proxy = toolchain / "litellm-venv" / "bin" / "litellm" + proxy.parent.mkdir(parents=True) + proxy.write_text("#!/bin/sh\nexec sleep 60\n", encoding="utf-8") + proxy.chmod(0o755) + env_file = tmp_path / "remote.env" + env_file.write_text("", encoding="utf-8") + shell_env = tmp_path / "bash_env" + shell_env.write_text( + '''python3() { + if [[ "$1" == "-" ]]; then cat >/dev/null; return 0; fi + mkdir -p "$TEST_ROOT/results/jobs/$TEST_LABEL/task__trial" + printf '{}\\n' > "$TEST_ROOT/results/jobs/$TEST_LABEL/task__trial/result.json" + printf '{}\\n' > "$TEST_ROOT/results/jobs/$TEST_LABEL/run_manifest.json" + return "$TEST_EXIT_CODE" +} +curl() { return 0; } +sudo() { "$@"; } +''', + encoding="utf-8", + ) + archive = Path("/tmp") / f"{label}-final-artifacts.tar.gz" + state_dir = Path("/tmp/shellbench-runs") / label + metadata_dir = Path("/tmp") / f"shellbench_meta-{label}" + script = Path(__file__).resolve().parents[1] / "scripts/native_eval/remote_run.sh" + try: + process = subprocess.run( + [ + bash, str(script), str(root), str(tmp_path), str(env_file), + label, "openclaw", "gpt55", "1", "1", "tasks-commit", + "20260828", "1", "test", "gpt-5.5", "openai", "sb-gpt55", "", + ], + env={ + **os.environ, + "BASH_ENV": str(shell_env), + "TOOLCHAIN_ROOT": str(toolchain), + "SHELLBENCH_PROXY_KEY": "synthetic-test-key", + "TEST_ROOT": str(root), + "TEST_LABEL": label, + "TEST_EXIT_CODE": str(exit_code), + }, + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert process.returncode == exit_code, process.stderr + assert state_dir.joinpath("exit_status").read_text().strip() == str(exit_code) + config = _config(tmp_path, tmp_path / "run_index.json") + raw = config.local_root / "raw" + raw.mkdir(parents=True) + shutil.copyfile(archive, raw / archive.name) + controller = FleetController( + config, executor=FakeExecutor(config.local_root, expected_counts={label: 1}) + ) + assert controller._archived_exit_status(label) == exit_code + finally: + archive.unlink(missing_ok=True) + shutil.rmtree(state_dir, ignore_errors=True) + shutil.rmtree(metadata_dir, ignore_errors=True) + + def test_provisioning_lease_is_not_ready_before_ssh_details_exist() -> None: with pytest.raises(LeaseNotReadyError, match="provisioning but not ready"): Lease.from_inspect( @@ -1372,7 +1453,7 @@ def run(self, command, *, capture_output=False): assert executor.dispatches == [label] -def test_recovery_infers_success_from_verified_full_archive_and_done_log( +def test_recovery_does_not_infer_success_from_result_count( tmp_path: Path, ) -> None: label = "openclaw-gpt55-full-2-r1-20260727" @@ -1386,7 +1467,7 @@ def test_recovery_infers_success_from_verified_full_archive_and_done_log( } run_index = tmp_path / "manifests" / "run_index.json" _write_index(run_index, [run]) - config = _config(tmp_path, run_index) + config = _config(tmp_path, run_index, max_attempts=1) _write_final(config.local_root, label, 2) checkpoint_log = config.local_root / "logs" / f"{label}.checkpoints.log" checkpoint_log.parent.mkdir(parents=True, exist_ok=True) @@ -1408,16 +1489,16 @@ def test_recovery_infers_success_from_verified_full_archive_and_done_log( } executor.active_leases = 1 - assert FleetController(config, executor=executor).run() == 0 + assert FleetController(config, executor=executor).run() == 1 recovered = json.loads(run_index.read_text(encoding="utf-8"))["runs"][0] - assert recovered["status"] == "completed" - assert recovered["run_exit_code"] == 0 - assert recovered["run_exit_code_source"] == "recovered_full_coverage_remote_done" - assert "inferred zero" in recovered["run_exit_code_inference"] + assert recovered["status"] == "failed" + assert recovered.get("run_exit_code") is None + assert recovered["last_error"] == "run exit unknown; result coverage 2/2" -def test_recovery_preserves_archived_nonzero_exit_status(tmp_path: Path) -> None: +@pytest.mark.parametrize("exit_code", [0, 2, 7]) +def test_recovery_preserves_archived_exit_status(tmp_path: Path, exit_code: int) -> None: label = "openclaw-gpt55-full-2-r1-20260727" run = _planned(_run_spec(label)) run["status"] = "recovery_required" @@ -1430,17 +1511,19 @@ def test_recovery_preserves_archived_nonzero_exit_status(tmp_path: Path) -> None run_index = tmp_path / "manifests" / "run_index.json" _write_index(run_index, [run]) config = _config(tmp_path, run_index, max_attempts=1) - _write_final(config.local_root, label, 2, exit_status=7) + _write_final(config.local_root, label, 2, exit_status=exit_code) executor = FakeExecutor(config.local_root, expected_counts={label: 2}) executor.active_leases = 1 - assert FleetController(config, executor=executor).run() == 1 + assert FleetController(config, executor=executor).run() == int(exit_code != 0) recovered = json.loads(run_index.read_text(encoding="utf-8"))["runs"][0] - assert recovered["status"] == "failed" - assert recovered["run_exit_code"] == 7 + assert recovered["status"] == ("completed" if exit_code == 0 else "failed") + assert recovered["run_exit_code"] == exit_code assert recovered["run_exit_code_source"] == "archived_exit_status" - assert recovered["last_error"] == "run exit 7; result coverage 2/2" + assert recovered["last_error"] == ( + None if exit_code == 0 else f"run exit {exit_code}; result coverage 2/2" + ) def test_stop_failure_is_left_pending_without_tight_retry(tmp_path: Path) -> None: diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 6aef78f..280fdd9 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -9,6 +9,8 @@ from pathlib import Path from types import SimpleNamespace +import pytest + from scripts.native_eval.checkpoint_loop import ( count_result_json, next_checkpoint_sequence, @@ -33,6 +35,7 @@ from scripts.native_eval import plan as native_plan from scripts.native_eval.proxy import JUDGE_PROXY_MODEL_NAME, write_proxy_config from scripts.native_eval.run_job import ( + _execution_acceptance, _git_commit, _run_manifest, build_run_spec, @@ -40,8 +43,10 @@ ) from scripts.native_eval.runtime import ( DockerTaskEnvironment, + NonZeroAgentExitCodeError, build_judge_env, collect_agent_metrics, + execution_outcome, read_reward, write_agent_trajectory, ) @@ -59,6 +64,111 @@ def test_matrix_plan_contains_only_requested_models_and_harnesses() -> None: assert {run.repetition for run in plan} == {1, 2, 3} +def test_openclaw_terminal_evidence_exit_is_a_harness_error() -> None: + outcome = execution_outcome( + harness="openclaw", + exception=NonZeroAgentExitCodeError("Agent exited with code 71"), + agent_exit_code=71, + ) + + assert outcome == { + "kind": "harness_error", + "exit_code": 71, + "reason": "terminal_session_evidence_unavailable", + } + + +def test_all_harness_errors_reject_run_but_agent_errors_do_not() -> None: + rejected = _execution_acceptance( + [ + {"execution_outcome": {"kind": "harness_error"}}, + {"execution_outcome": {"kind": "infra_error"}}, + ], + 2, + ) + accepted = _execution_acceptance( + [ + {"execution_outcome": {"kind": "clean"}}, + {"execution_outcome": {"kind": "agent_error"}}, + ], + 2, + ) + + assert rejected == { + "accepted": False, + "reason": "all_trials_harness_or_infrastructure_errors", + "outcome_counts": {"harness_error": 1, "infra_error": 1}, + } + assert accepted == { + "accepted": True, + "reason": None, + "outcome_counts": {"agent_error": 1, "clean": 1}, + } + + +@pytest.mark.parametrize("failure_stage", ["reward", "artifacts", "stop", None]) +def test_trial_does_not_hide_execution_failure_after_agent_exit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_stage: str | None +) -> None: + class Environment: + def __init__(self, *, trial_dir: Path, **_kwargs: object) -> None: + self.trial_dir = trial_dir + + async def start(self): + return native_runtime.CommandResult(0, "start", "end") + + async def copy_instruction(self, _instruction: str) -> None: + pass + + async def exec(self, command: str, **_kwargs: object): + if command.endswith("/tests/test.sh") and failure_stage != "reward": + (self.trial_dir / "verifier" / "reward.txt").write_text("0.5") + return native_runtime.CommandResult(int(command == "run"), "start", "end") + + async def collect_artifacts(self) -> None: + if failure_stage == "artifacts": + raise native_runtime.DockerStartupError("artifact collection failed") + + async def install_tests(self) -> None: + pass + + async def stop(self) -> None: + if failure_stage == "stop": + raise native_runtime.DockerStartupError("container cleanup failed") + + monkeypatch.setattr(native_runtime, "DockerTaskEnvironment", Environment) + monkeypatch.setattr( + native_runtime, + "build_harness_command", + lambda *_args, **_kwargs: SimpleNamespace( + setup_command="setup", run_command="run", cleanup_command="cleanup", env={} + ), + ) + run = RunSpec( + run_label="trial-validity", harness="openclaw", harness_version="test", + model_slug="gpt55", model_id="gpt-5.5", provider="openai", + proxy_model_name="sb-gpt55", repetition=1, expected_task_count=1, + run_date="20260828", + ) + result = asyncio.run( + native_runtime.run_trial( + _trajectory_task(tmp_path, "do the task"), run, + job_dir=tmp_path / "job", toolchain_root=tmp_path, + proxy_url="http://localhost:4000", proxy_key="synthetic-test-key", + ) + ) + + assert result["exception_info"]["exception_type"] == "NonZeroAgentExitCodeError" + expected_kind = ( + "verifier_error" if failure_stage == "reward" + else "infra_error" if failure_stage else "agent_error" + ) + assert result["execution_outcome"]["kind"] == expected_kind + assert _execution_acceptance([result], 1)["accepted"] is (failure_stage is None) + if failure_stage in {None, "stop"}: + assert result["verifier_result"]["rewards"] == {"reward": 0.5} + + def test_run_index_records_agent_and_judge_reasoning( tmp_path: Path, monkeypatch,