diff --git a/CHANGELOG.md b/CHANGELOG.md index 17106b2..b463127 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed +- Separate native execution validity from diagnostic rewards, reject wholly invalid runs, and preserve terminal exit status in recovery archives (#63, thanks @vincentkoc). - Install only the assigned native harness on each fleet lease and record only its version in the toolchain manifest (#65). - Refresh Python runtime, MLflow, lint, and HF mirror dependencies, including websockets 17 with a real gateway socket regression test, while preserving Python 3.11 NumPy support. - Record and enforce GPT reasoning effort in native runs, preserve it across diff --git a/scripts/native_eval/aggregate.py b/scripts/native_eval/aggregate.py index d316090..0ebf428 100644 --- a/scripts/native_eval/aggregate.py +++ b/scripts/native_eval/aggregate.py @@ -91,6 +91,9 @@ "trial_name", "result_path", "classification", + "execution_outcome", + "execution_exit_code", + "execution_reason", "reward", "exception_type", "exception_message", @@ -142,6 +145,8 @@ "infra_dominated", "harness_wide_failure", "harness_wide_failure_signature", + "run_accepted", + "run_acceptance_reason", "canonical_model_identity", "trajectory_complete", "parity_validated", @@ -278,8 +283,18 @@ def _is_infra(exception_type: str, exception_message: str) -> bool: return any(pattern in combined for pattern in INFRA_MESSAGE_PATTERNS) -def _classify(result: dict[str, Any], reward: int | float | None) -> str: +def _classify( + result: dict[str, Any], + reward: int | float | None, + execution_kind: str, +) -> str: exception_type, exception_message, _ = _exception_fields(result) + if execution_kind in {"harness_error", "infra_error"}: + return "infra" + if execution_kind == "verifier_error": + return "verifier_missing_reward" + if execution_kind == "agent_error": + return "agent_exit" if exception_type in MISSING_REWARD_EXCEPTION_TYPES: return "verifier_missing_reward" if exception_type: @@ -295,6 +310,40 @@ def _classify(result: dict[str, Any], reward: int | float | None) -> str: return "pass" +def _execution_fields( + result: dict[str, Any], + *, + harness: str, +) -> tuple[str, int | None, str]: + outcome = result.get("execution_outcome") + if isinstance(outcome, dict): + kind = str(outcome.get("kind") or "") + if kind and kind != "pending": + exit_code = _number(outcome.get("exit_code")) + return ( + kind, + int(exit_code) if exit_code is not None else None, + str(outcome.get("reason") or ""), + ) + + exception_type, exception_message, _ = _exception_fields(result) + exit_match = re.search(r"\bcode\s+(\d+)\b", exception_message) + exit_code = int(exit_match.group(1)) if exit_match else None + if ( + harness == "openclaw" + and exception_type == "NonZeroAgentExitCodeError" + and exit_code in {70, 71} + ): + return "harness_error", exit_code, exception_message + if exception_type in MISSING_REWARD_EXCEPTION_TYPES: + return "verifier_error", exit_code, exception_message + if exception_type and _is_infra(exception_type, exception_message): + return "infra_error", exit_code, exception_message + if exception_type: + return "agent_error", exit_code, exception_message + return "clean", None, "" + + def _scorecard_infra_error(trial_dir: Path) -> tuple[str, str] | None: scorecard_path = trial_dir / "verifier" / "scorecard.json" if not scorecard_path.is_file(): @@ -525,6 +574,7 @@ def _normalize_result( run_label: str, pair_label: str, repetition: int | None, + harness: str, ) -> dict[str, Any]: task_name, task_path, trial_name = _task_identity(result, result_path.parent) reward = _reward(result) @@ -532,6 +582,13 @@ def _normalize_result( scorecard_infra = _scorecard_infra_error(result_path.parent) if scorecard_infra is not None: exception_type, exception_message = scorecard_infra + execution_kind, execution_exit_code, execution_reason = _execution_fields( + result, + harness=harness, + ) + if scorecard_infra is not None: + execution_kind = "infra_error" + execution_reason = exception_message agent_result = result.get("agent_result") agent_result = agent_result if isinstance(agent_result, dict) else {} row: dict[str, Any] = { @@ -543,8 +600,13 @@ def _normalize_result( "trial_name": trial_name, "result_path": str(result_path), "classification": ( - "infra" if scorecard_infra is not None else _classify(result, reward) + "infra" + if scorecard_infra is not None + else _classify(result, reward, execution_kind) ), + "execution_outcome": execution_kind, + "execution_exit_code": execution_exit_code, + "execution_reason": execution_reason, "reward": reward, "exception_type": exception_type, "exception_message": exception_message, @@ -654,6 +716,7 @@ def _load_run( run_label = str(manifest.get("run_label") or run_label) pair_label = _pair_label(manifest, run_label) repetition = _repetition(manifest, run_label) + harness = str(manifest.get("harness") or "") rows: list[dict[str, Any]] = [] trial_dirs = sorted(path for path in run_dir.iterdir() if path.is_dir()) @@ -699,6 +762,7 @@ def _load_run( run_label=run_label, pair_label=pair_label, repetition=repetition, + harness=harness, ) ) @@ -840,6 +904,10 @@ def _summarize_run( classifications = [str(row["classification"]) for row in scored_rows] rewards = [_number(row.get("reward")) for row in scored_rows] all_classifications = [str(row["classification"]) for row in rows] + execution_kinds = [ + str(row.get("execution_outcome") or "") + for row in scored_rows + ] result_file_count = sum(bool(row.get("_has_result_file")) for row in rows) valid_result_count = sum(bool(row.get("_valid_result")) for row in rows) completed_result_count = sum(bool(row.get("_completed_result")) for row in rows) @@ -874,6 +942,25 @@ def _summarize_run( rows, expected_count, ) + acceptance = manifest.get("execution_acceptance") + acceptance = acceptance if isinstance(acceptance, dict) else {} + accepted_value = acceptance.get("accepted") + if isinstance(accepted_value, bool): + run_accepted = accepted_value + else: + invalid_execution_kinds = { + "harness_error", + "infra_error", + "verifier_error", + } + run_accepted = not ( + expected_count > 0 + and len(execution_kinds) == expected_count + and all(kind in invalid_execution_kinds for kind in execution_kinds) + ) + run_acceptance_reason = str(acceptance.get("reason") or "") + if not run_accepted and not run_acceptance_reason: + run_acceptance_reason = "all_trials_harness_or_infrastructure_errors" native_run = ( manifest.get("runner") == "shellbench-native" or any(row.get("source") == "shellbench-native" for row in rows) @@ -917,6 +1004,7 @@ def _summarize_run( ) = _parity_validation(manifest, native_run=native_run) eligible = ( not incomplete + and run_accepted and not infra_dominated and not harness_wide_failure and canonical_model_identity @@ -926,6 +1014,8 @@ def _summarize_run( exclusion_reason = "incomplete" elif infra_dominated: exclusion_reason = "infra_dominated" + elif not run_accepted: + exclusion_reason = "run_rejected" elif harness_wide_failure: exclusion_reason = "harness_wide_failure" elif not canonical_model_identity: @@ -962,6 +1052,8 @@ def _summarize_run( "infra_dominated": infra_dominated, "harness_wide_failure": harness_wide_failure, "harness_wide_failure_signature": harness_wide_failure_signature, + "run_accepted": run_accepted, + "run_acceptance_reason": run_acceptance_reason, "canonical_model_identity": canonical_model_identity, "trajectory_complete": trajectory_complete, "parity_validated": parity_validated, @@ -1184,9 +1276,9 @@ def _write_leaderboard(path: Path, pairs: Sequence[dict[str, Any]]) -> None: lines = [ "# Cleaned Native ShellBench Leaderboard", "", - "Incomplete, infra-dominated, harness-wide failure, identity-invalid, and " - "trajectory-incomplete repetitions are excluded. Harbor parity validation is " - "reported separately and is not an eligibility gate.", + "Incomplete, rejected, infra-dominated, harness-wide failure, identity-invalid, " + "and trajectory-incomplete repetitions are excluded. Harbor parity validation " + "is reported separately and is not an eligibility gate.", "", "| Rank | Pair | Repetitions | Parity validated | Mean | Stdev | Min | Max | Mean exact passes | Pass rate | Clean complete |", "| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index a8e73f0..5a6c768 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -1248,16 +1248,6 @@ def _archived_exit_status(self, run_label: str) -> int | None: raise FleetError(f"conflicting archived exit_status values for {run_label}") return candidates[0] if candidates else None - def _checkpoint_log_has_final(self, run_label: str) -> bool: - log_path = self.config.local_root / "logs" / f"{run_label}.checkpoints.log" - if not log_path.is_file(): - return False - for line in log_path.read_text(encoding="utf-8").splitlines(): - fields = line.split("\t", 4) - if len(fields) >= 2 and fields[1] == "final": - return True - return False - def _finish_exported(self, entry: dict[str, Any], run: RunSpec) -> bool: verified, result_count, artifacts = self._verify_final(run.run_label) if not verified: @@ -1284,19 +1274,6 @@ def _finish_exported(self, entry: dict[str, Any], run: RunSpec) -> bool: "run_exit_code": run_exit_code, "run_exit_code_source": "archived_exit_status", } - elif ( - result_count == run.expected_task_count - and self._checkpoint_log_has_final(run.run_label) - ): - run_exit_code = 0 - exit_code_changes = { - "run_exit_code": 0, - "run_exit_code_source": "recovered_full_coverage_remote_done", - "run_exit_code_inference": ( - "inferred zero from verified full-coverage archive and " - "checkpoint final event after remote done" - ), - } stop_command = [ self.config.crabbox_bin, "stop", diff --git a/scripts/native_eval/remote_run.sh b/scripts/native_eval/remote_run.sh index 91c6f3c..0a4976a 100755 --- a/scripts/native_eval/remote_run.sh +++ b/scripts/native_eval/remote_run.sh @@ -141,6 +141,7 @@ RUN_STATUS="$?" set -e META_DIR="/tmp/shellbench_meta-$RUN_LABEL" +printf '%s\n' "$RUN_STATUS" > "$META_DIR/exit_status" date -u +%Y-%m-%dT%H:%M:%SZ > "$META_DIR/exported_at_utc.txt" hostname > "$META_DIR/hostname.txt" git -C "$ROOT/runner" rev-parse HEAD > "$META_DIR/runner_commit.txt" || true diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index ccafe5e..d1d8ab4 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -6,6 +6,7 @@ import os import subprocess import uuid +from collections import Counter from dataclasses import asdict from pathlib import Path from typing import Any @@ -127,9 +128,12 @@ async def execute(task: TaskSpec) -> None: state["finished_at"] = utc_now() state["updated_at"] = state["finished_at"] _update_job_result(state, results, len(tasks)) + acceptance = _execution_acceptance(results, len(tasks)) + state["execution_acceptance"] = acceptance atomic_write_json(job_dir / "result.json", state) manifest["finished_at_utc"] = state["finished_at"] manifest["result_json_count"] = len(results) + manifest["execution_acceptance"] = acceptance agent_results = [ result.get("agent_result") for result in results @@ -204,6 +208,30 @@ def _update_job_result( state["updated_at"] = utc_now() +def _execution_acceptance( + results: list[dict[str, Any]], + total: int, +) -> dict[str, Any]: + kinds = Counter( + str((result.get("execution_outcome") or {}).get("kind") or "unknown") + for result in results + ) + invalid_kinds = {"harness_error", "infra_error", "verifier_error"} + invalid_count = sum(kinds[kind] for kind in invalid_kinds) + accepted = len(results) == total and invalid_count < total + if len(results) != total: + reason = "incomplete_result_coverage" + elif invalid_count == total and total: + reason = "all_trials_harness_or_infrastructure_errors" + else: + reason = None + return { + "accepted": accepted, + "reason": reason, + "outcome_counts": dict(sorted(kinds.items())), + } + + def _run_manifest( run: RunSpec, *, @@ -293,6 +321,11 @@ def _run_manifest( "started_at_utc": started_at, "finished_at_utc": None, "result_json_count": 0, + "execution_acceptance": { + "accepted": None, + "reason": "run_in_progress", + "outcome_counts": {}, + }, } @@ -434,6 +467,8 @@ def main() -> None: ) ) print(json.dumps(state, indent=2)) + if state["execution_acceptance"]["accepted"] is not True: + raise SystemExit(2) if __name__ == "__main__": diff --git a/scripts/native_eval/runtime.py b/scripts/native_eval/runtime.py index f26f693..1517a6d 100644 --- a/scripts/native_eval/runtime.py +++ b/scripts/native_eval/runtime.py @@ -52,6 +52,12 @@ class RewardFileNotFoundError(NativeEvalError): pass +OPENCLAW_HARNESS_EXIT_REASONS = { + 70: "gateway_start_failed", + 71: "terminal_session_evidence_unavailable", +} + + CODEX_DIAGNOSTIC_LINE = re.compile( r"^\d{4}-\d{2}-\d{2}T\S+\s+" r"(?:TRACE|DEBUG|INFO|WARN|ERROR)\s+codex[\w:.-]*:" @@ -473,6 +479,8 @@ async def run_trial( toolchain_root=toolchain_root, ) recorded_exception: BaseException | None = None + execution_exception: BaseException | None = None + agent_exit_code: int | None = None agent_command = build_harness_command( run, proxy_url=proxy_url, @@ -517,6 +525,7 @@ async def run_trial( stderr_path=trial_dir / "agent" / "stderr.txt", ) if agent.returncode: + agent_exit_code = agent.returncode raise NonZeroAgentExitCodeError( f"Agent exited with code {agent.returncode}" ) @@ -570,20 +579,30 @@ async def run_trial( } reward = read_reward(trial_dir / "verifier") result["verifier_result"] = {"rewards": reward} - if verifier.returncode and recorded_exception is None and not reward: - recorded_exception = RewardFileNotFoundError( + if verifier.returncode and not reward: + raise RewardFileNotFoundError( f"Verifier exited {verifier.returncode} without a reward" ) except BaseException as exc: + execution_exception = exc if recorded_exception is None: recorded_exception = exc finally: try: await environment.stop() except Exception as exc: + if execution_exception is None: + execution_exception = exc if recorded_exception is None: recorded_exception = exc result["finished_at"] = utc_now() + result["execution_outcome"] = execution_outcome( + harness=run.harness, + # Keep the first diagnostic exception, but do not let an agent exit + # hide a later verifier or infrastructure failure from acceptance. + exception=execution_exception or recorded_exception, + agent_exit_code=agent_exit_code, + ) if recorded_exception is not None: result["exception_info"] = exception_info(recorded_exception) (trial_dir / "exception.txt").write_text( @@ -640,6 +659,11 @@ def _initial_trial_result( }, "agent_result": None, "verifier_result": None, + "execution_outcome": { + "kind": "pending", + "exit_code": None, + "reason": None, + }, "verifier_environment_mode": "shared", "exception_info": None, "started_at": started_at, @@ -1287,6 +1311,42 @@ def exception_info(exc: BaseException) -> dict[str, str]: } +def execution_outcome( + *, + harness: str, + exception: BaseException | None, + agent_exit_code: int | None, +) -> dict[str, Any]: + if exception is None: + return {"kind": "clean", "exit_code": None, "reason": None} + if ( + harness == "openclaw" + and agent_exit_code in OPENCLAW_HARNESS_EXIT_REASONS + ): + return { + "kind": "harness_error", + "exit_code": agent_exit_code, + "reason": OPENCLAW_HARNESS_EXIT_REASONS[agent_exit_code], + } + if isinstance(exception, NonZeroAgentExitCodeError): + return { + "kind": "agent_error", + "exit_code": agent_exit_code, + "reason": str(exception), + } + if isinstance(exception, (AgentSetupError, AgentSetupTimeoutError)): + kind = "harness_error" + elif isinstance(exception, RewardFileNotFoundError): + kind = "verifier_error" + else: + kind = "infra_error" + return { + "kind": kind, + "exit_code": agent_exit_code, + "reason": str(exception), + } + + def _timing(result: CommandResult) -> dict[str, str]: return { "started_at": result.started_at, diff --git a/tests/test_native_eval_aggregate.py b/tests/test_native_eval_aggregate.py index c1e26d1..dff1291 100644 --- a/tests/test_native_eval_aggregate.py +++ b/tests/test_native_eval_aggregate.py @@ -509,6 +509,80 @@ def test_native_identity_checks_ignore_unavailable_agent_exit_trajectory( assert run["eligible"] is True +@pytest.mark.parametrize( + ("kinds", "accepted"), + [ + (("clean", "clean"), True), + (("agent_error", "clean"), True), + (("harness_error", "clean"), True), + (("verifier_error", "verifier_error"), False), + (("harness_error", "infra_error"), False), + ], +) +def test_structured_execution_validity_preserves_diagnostic_scores( + tmp_path: Path, kinds: tuple[str, str], accepted: bool +) -> None: + results = [] + for index, kind in enumerate(kinds): + result = _native_result(str(index), reward=0.5) + result["execution_outcome"] = {"kind": kind, "exit_code": None, "reason": kind} + results.append(result) + jobs_root = tmp_path / "native" + _write_run( + jobs_root, "structured-execution", expected_task_count=2, + results=results, native=True, + ) + + run = aggregate(jobs_root, tmp_path / "summaries")["runs"][0] + + assert run["score"] == pytest.approx(0.5) + assert run["nonzero"] == 2 + assert run["run_accepted"] is accepted + assert run["eligible"] is accepted + + +def test_all_openclaw_harness_errors_preserve_rewards_but_reject_run( + tmp_path: Path, +): + jobs_root = tmp_path / "native" + summaries_dir = tmp_path / "summaries" + _write_run( + jobs_root, + "openclaw-gpt55-terminal-evidence-failure", + expected_task_count=2, + results=[ + _result( + "a", + reward=0.25, + exception_type="NonZeroAgentExitCodeError", + exception_message="Agent exited with code 71", + ), + _result( + "b", + reward=0.75, + exception_type="NonZeroAgentExitCodeError", + exception_message="Agent exited with code 71", + ), + ], + harness="openclaw", + ) + + report = aggregate(jobs_root, summaries_dir) + + run = report["runs"][0] + assert run["score"] == pytest.approx(0.5) + assert run["nonzero"] == 2 + assert run["run_accepted"] is False + assert run["run_acceptance_reason"] == ( + "all_trials_harness_or_infrastructure_errors" + ) + assert run["eligible"] is False + assert run["exclusion_reason"] == "infra_dominated" + with (summaries_dir / "per_task_results.csv").open(newline="") as handle: + rows = list(csv.DictReader(handle)) + assert {row["execution_outcome"] for row in rows} == {"harness_error"} + + def test_native_identity_checks_real_agent_exit_trajectory(tmp_path: Path): jobs_root = tmp_path / "native" summaries_dir = tmp_path / "summaries" diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 82b285d..9ad76b3 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -1,12 +1,15 @@ from __future__ import annotations import json +import os import shlex +import shutil import subprocess import tarfile import threading from pathlib import Path from typing import Sequence +from uuid import uuid4 import pytest @@ -80,6 +83,84 @@ def _planned(run: RunSpec) -> dict[str, object]: } +@pytest.mark.parametrize("exit_code", [0, 2]) +def test_remote_run_archives_terminal_status(tmp_path: Path, exit_code: int) -> None: + bash = shutil.which("bash") + assert bash is not None + if subprocess.run( + [ + bash, "-c", + "(( BASH_VERSINFO[0] > 4 || " + "(BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] >= 4) ))", + ], + check=False, + ).returncode: + pytest.skip("remote_run.sh requires modern Bash") + label = f"shellbench-test-{uuid4().hex}" + root = tmp_path / "remote" + (root / "runner").mkdir(parents=True) + toolchain = tmp_path / "toolchain" + proxy = toolchain / "litellm-venv" / "bin" / "litellm" + proxy.parent.mkdir(parents=True) + proxy.write_text("#!/bin/sh\nexec sleep 60\n", encoding="utf-8") + proxy.chmod(0o755) + env_file = tmp_path / "remote.env" + env_file.write_text("", encoding="utf-8") + shell_env = tmp_path / "bash_env" + shell_env.write_text( + '''python3() { + if [[ "$1" == "-" ]]; then cat >/dev/null; return 0; fi + mkdir -p "$TEST_ROOT/results/jobs/$TEST_LABEL/task__trial" + printf '{}\\n' > "$TEST_ROOT/results/jobs/$TEST_LABEL/task__trial/result.json" + printf '{}\\n' > "$TEST_ROOT/results/jobs/$TEST_LABEL/run_manifest.json" + return "$TEST_EXIT_CODE" +} +curl() { return 0; } +sudo() { "$@"; } +''', + encoding="utf-8", + ) + archive = Path("/tmp") / f"{label}-final-artifacts.tar.gz" + state_dir = Path("/tmp/shellbench-runs") / label + metadata_dir = Path("/tmp") / f"shellbench_meta-{label}" + script = Path(__file__).resolve().parents[1] / "scripts/native_eval/remote_run.sh" + try: + process = subprocess.run( + [ + bash, str(script), str(root), str(tmp_path), str(env_file), + label, "openclaw", "gpt55", "1", "1", "tasks-commit", + "20260828", "1", "test", "gpt-5.5", "openai", "sb-gpt55", "", + ], + env={ + **os.environ, + "BASH_ENV": str(shell_env), + "TOOLCHAIN_ROOT": str(toolchain), + "SHELLBENCH_PROXY_KEY": "synthetic-test-key", + "TEST_ROOT": str(root), + "TEST_LABEL": label, + "TEST_EXIT_CODE": str(exit_code), + }, + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert process.returncode == exit_code, process.stderr + assert state_dir.joinpath("exit_status").read_text().strip() == str(exit_code) + config = _config(tmp_path, tmp_path / "run_index.json") + raw = config.local_root / "raw" + raw.mkdir(parents=True) + shutil.copyfile(archive, raw / archive.name) + controller = FleetController( + config, executor=FakeExecutor(config.local_root, expected_counts={label: 1}) + ) + assert controller._archived_exit_status(label) == exit_code + finally: + archive.unlink(missing_ok=True) + shutil.rmtree(state_dir, ignore_errors=True) + shutil.rmtree(metadata_dir, ignore_errors=True) + + def test_provisioning_lease_is_not_ready_before_ssh_details_exist() -> None: with pytest.raises(LeaseNotReadyError, match="provisioning but not ready"): Lease.from_inspect( @@ -1292,7 +1373,7 @@ def test_recovery_required_resumes_existing_remote_run(tmp_path: Path) -> None: assert final["status"] == "completed" -def test_recovery_infers_success_from_verified_full_archive_and_done_log( +def test_recovery_does_not_infer_success_from_result_count( tmp_path: Path, ) -> None: label = "openclaw-gpt55-full-2-r1-20260727" @@ -1306,7 +1387,7 @@ def test_recovery_infers_success_from_verified_full_archive_and_done_log( } run_index = tmp_path / "manifests" / "run_index.json" _write_index(run_index, [run]) - config = _config(tmp_path, run_index) + config = _config(tmp_path, run_index, max_attempts=1) _write_final(config.local_root, label, 2) checkpoint_log = config.local_root / "logs" / f"{label}.checkpoints.log" checkpoint_log.parent.mkdir(parents=True, exist_ok=True) @@ -1328,16 +1409,16 @@ def test_recovery_infers_success_from_verified_full_archive_and_done_log( } executor.active_leases = 1 - assert FleetController(config, executor=executor).run() == 0 + assert FleetController(config, executor=executor).run() == 1 recovered = json.loads(run_index.read_text(encoding="utf-8"))["runs"][0] - assert recovered["status"] == "completed" - assert recovered["run_exit_code"] == 0 - assert recovered["run_exit_code_source"] == "recovered_full_coverage_remote_done" - assert "inferred zero" in recovered["run_exit_code_inference"] + assert recovered["status"] == "failed" + assert recovered.get("run_exit_code") is None + assert recovered["last_error"] == "run exit unknown; result coverage 2/2" -def test_recovery_preserves_archived_nonzero_exit_status(tmp_path: Path) -> None: +@pytest.mark.parametrize("exit_code", [0, 2, 7]) +def test_recovery_preserves_archived_exit_status(tmp_path: Path, exit_code: int) -> None: label = "openclaw-gpt55-full-2-r1-20260727" run = _planned(_run_spec(label)) run["status"] = "recovery_required" @@ -1350,17 +1431,19 @@ def test_recovery_preserves_archived_nonzero_exit_status(tmp_path: Path) -> None run_index = tmp_path / "manifests" / "run_index.json" _write_index(run_index, [run]) config = _config(tmp_path, run_index, max_attempts=1) - _write_final(config.local_root, label, 2, exit_status=7) + _write_final(config.local_root, label, 2, exit_status=exit_code) executor = FakeExecutor(config.local_root, expected_counts={label: 2}) executor.active_leases = 1 - assert FleetController(config, executor=executor).run() == 1 + assert FleetController(config, executor=executor).run() == int(exit_code != 0) recovered = json.loads(run_index.read_text(encoding="utf-8"))["runs"][0] - assert recovered["status"] == "failed" - assert recovered["run_exit_code"] == 7 + assert recovered["status"] == ("completed" if exit_code == 0 else "failed") + assert recovered["run_exit_code"] == exit_code assert recovered["run_exit_code_source"] == "archived_exit_status" - assert recovered["last_error"] == "run exit 7; result coverage 2/2" + assert recovered["last_error"] == ( + None if exit_code == 0 else f"run exit {exit_code}; result coverage 2/2" + ) def test_stop_failure_is_left_pending_without_tight_retry(tmp_path: Path) -> None: diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 6aef78f..280fdd9 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -9,6 +9,8 @@ from pathlib import Path from types import SimpleNamespace +import pytest + from scripts.native_eval.checkpoint_loop import ( count_result_json, next_checkpoint_sequence, @@ -33,6 +35,7 @@ from scripts.native_eval import plan as native_plan from scripts.native_eval.proxy import JUDGE_PROXY_MODEL_NAME, write_proxy_config from scripts.native_eval.run_job import ( + _execution_acceptance, _git_commit, _run_manifest, build_run_spec, @@ -40,8 +43,10 @@ ) from scripts.native_eval.runtime import ( DockerTaskEnvironment, + NonZeroAgentExitCodeError, build_judge_env, collect_agent_metrics, + execution_outcome, read_reward, write_agent_trajectory, ) @@ -59,6 +64,111 @@ def test_matrix_plan_contains_only_requested_models_and_harnesses() -> None: assert {run.repetition for run in plan} == {1, 2, 3} +def test_openclaw_terminal_evidence_exit_is_a_harness_error() -> None: + outcome = execution_outcome( + harness="openclaw", + exception=NonZeroAgentExitCodeError("Agent exited with code 71"), + agent_exit_code=71, + ) + + assert outcome == { + "kind": "harness_error", + "exit_code": 71, + "reason": "terminal_session_evidence_unavailable", + } + + +def test_all_harness_errors_reject_run_but_agent_errors_do_not() -> None: + rejected = _execution_acceptance( + [ + {"execution_outcome": {"kind": "harness_error"}}, + {"execution_outcome": {"kind": "infra_error"}}, + ], + 2, + ) + accepted = _execution_acceptance( + [ + {"execution_outcome": {"kind": "clean"}}, + {"execution_outcome": {"kind": "agent_error"}}, + ], + 2, + ) + + assert rejected == { + "accepted": False, + "reason": "all_trials_harness_or_infrastructure_errors", + "outcome_counts": {"harness_error": 1, "infra_error": 1}, + } + assert accepted == { + "accepted": True, + "reason": None, + "outcome_counts": {"agent_error": 1, "clean": 1}, + } + + +@pytest.mark.parametrize("failure_stage", ["reward", "artifacts", "stop", None]) +def test_trial_does_not_hide_execution_failure_after_agent_exit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_stage: str | None +) -> None: + class Environment: + def __init__(self, *, trial_dir: Path, **_kwargs: object) -> None: + self.trial_dir = trial_dir + + async def start(self): + return native_runtime.CommandResult(0, "start", "end") + + async def copy_instruction(self, _instruction: str) -> None: + pass + + async def exec(self, command: str, **_kwargs: object): + if command.endswith("/tests/test.sh") and failure_stage != "reward": + (self.trial_dir / "verifier" / "reward.txt").write_text("0.5") + return native_runtime.CommandResult(int(command == "run"), "start", "end") + + async def collect_artifacts(self) -> None: + if failure_stage == "artifacts": + raise native_runtime.DockerStartupError("artifact collection failed") + + async def install_tests(self) -> None: + pass + + async def stop(self) -> None: + if failure_stage == "stop": + raise native_runtime.DockerStartupError("container cleanup failed") + + monkeypatch.setattr(native_runtime, "DockerTaskEnvironment", Environment) + monkeypatch.setattr( + native_runtime, + "build_harness_command", + lambda *_args, **_kwargs: SimpleNamespace( + setup_command="setup", run_command="run", cleanup_command="cleanup", env={} + ), + ) + run = RunSpec( + run_label="trial-validity", harness="openclaw", harness_version="test", + model_slug="gpt55", model_id="gpt-5.5", provider="openai", + proxy_model_name="sb-gpt55", repetition=1, expected_task_count=1, + run_date="20260828", + ) + result = asyncio.run( + native_runtime.run_trial( + _trajectory_task(tmp_path, "do the task"), run, + job_dir=tmp_path / "job", toolchain_root=tmp_path, + proxy_url="http://localhost:4000", proxy_key="synthetic-test-key", + ) + ) + + assert result["exception_info"]["exception_type"] == "NonZeroAgentExitCodeError" + expected_kind = ( + "verifier_error" if failure_stage == "reward" + else "infra_error" if failure_stage else "agent_error" + ) + assert result["execution_outcome"]["kind"] == expected_kind + assert _execution_acceptance([result], 1)["accepted"] is (failure_stage is None) + if failure_stage in {None, "stop"}: + assert result["verifier_result"]["rewards"] == {"reward": 0.5} + + def test_run_index_records_agent_and_judge_reasoning( tmp_path: Path, monkeypatch,