From 69702f48dc2bc5e76f2c2dbe9198eaa5a4b7159a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 28 Aug 2026 09:16:37 -0700 Subject: [PATCH] fix(native-eval): resolve reasoning effort before proxy startup Integrate PR #54's native harness effort controls and preserve the planned effort in the fleet RunSpec and run manifest. Resolve CLI/planned effort over environment defaults before generating the proxy configuration. Export that canonical value before starting the proxy or runner, and reject conflicting late CLI overrides. Cover all four harnesses through the real remote launcher with controlled proxy and trial processes. Keep event-driven scheduler test waits bounded without treating host scheduling latency as a performance contract. Fixes #53 Co-authored-by: Vincent Koc --- CHANGELOG.md | 1 + docs/native_reasoning_effort.md | 28 ++++ scripts/native_eval/fleet.py | 5 +- scripts/native_eval/harnesses.py | 20 ++- scripts/native_eval/models.py | 2 + scripts/native_eval/remote_run.sh | 65 ++++---- scripts/native_eval/run_job.py | 33 +++- tests/test_native_eval_effort.py | 240 ++++++++++++++++++++++++++++++ tests/test_native_eval_fleet.py | 16 +- tests/test_native_eval_runner.py | 91 +++++++++++ 10 files changed, 460 insertions(+), 41 deletions(-) create mode 100644 docs/native_reasoning_effort.md create mode 100644 tests/test_native_eval_effort.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ccbe8a7..e6745cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed +- Apply planned native reasoning effort to all four harnesses and resolve proxy, runner, and manifest precedence before proxy startup (#53, thanks @vincentkoc). - Rehydrate replacement native fleet leases instead of trusting stale bootstrap timestamps (#58, thanks @vincentkoc). - Separate native execution validity from diagnostic rewards, reject wholly invalid runs, and preserve terminal exit status in recovery archives (#63, thanks @vincentkoc). - Install only the assigned native harness on each fleet lease and record only its version in the toolchain manifest (#65). diff --git a/docs/native_reasoning_effort.md b/docs/native_reasoning_effort.md new file mode 100644 index 0000000..1ec7239 --- /dev/null +++ b/docs/native_reasoning_effort.md @@ -0,0 +1,28 @@ +# Native reasoning effort + +The native fleet's planned `reasoning_effort` controls both the provider proxy +and the agent harness. Supported values are `low`, `medium`, `high`, and `xhigh`. +OpenClaw receives `--thinking`, Hermes receives `agent.reasoning_effort`, Codex +receives `model_reasoning_effort`, and Claude Code receives `--effort` (`max` for +the canonical `xhigh` value). The run manifest records the resolved run value, +not a later reading of the environment. + +`remote_run.sh` preserves the effort supplied by fleet dispatch before sourcing +the provider environment file. That planned value takes precedence over the +file's default. If dispatch supplies no effort, the environment file supplies +the default instead. Judge reasoning effort remains independent. + +Before starting LiteLLM, the launcher invokes `run_job` with the complete run +arguments and `--prepare-proxy-config PATH`. This preparation mode lets an +explicit `--reasoning-effort` override the environment, validates the result, +writes the proxy configuration, and prints the canonical effort to standard +output. The launcher exports that value as `SHELLBENCH_REASONING_EFFORT` before +starting either the proxy or the job. Invalid values stop startup before the +proxy is launched. + +For manual launches, use the same preparation step before starting the proxy +and export its returned value for the runner. An ordinary `run_job` invocation +rejects conflicting CLI and environment efforts: it cannot safely override a +proxy that is already running. Without either value, direct harness construction +retains each harness's defaults; the managed proxy still requires an explicit +valid effort, as before. diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index b258495..74e0776 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -1479,7 +1479,10 @@ def _matrix_satisfied(self) -> bool: def _run_spec(self, entry: dict[str, Any]) -> RunSpec: try: - return RunSpec(**{field: entry[field] for field in RUN_SPEC_FIELDS}) + return RunSpec( + **{field: entry[field] for field in RUN_SPEC_FIELDS}, + reasoning_effort=entry.get("reasoning_effort"), + ) except KeyError as exc: raise FleetError( f"run entry {entry.get('run_label', '')} lacks {exc.args[0]}" diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index b3c3aa0..03624da 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -532,6 +532,7 @@ def _openclaw( ) -> HarnessCommand: provider = "openai" model = f"{provider}/{run.model_id}" + thinking = run.reasoning_effort or "off" home = "/tmp/shellbench-openclaw" audit_plugin_root = f"{home}/.openclaw/shellbench-audit" gateway_token = secrets.token_urlsafe(32) @@ -616,7 +617,8 @@ def _openclaw( 'kill "$gateway_pid" 2>/dev/null || true; ' 'wait "$gateway_pid" 2>/dev/null || true; ' 'cat "$gateway_log" >&2; exit 70; fi; ' - "openclaw agent --json --agent main --thinking off " + "openclaw agent --json --agent main " + f"--thinking {shlex.quote(thinking)} " f"--model {shlex.quote(model)} " '--message "$(cat /tmp/shellbench-instruction.md)" ' '>"$log" 2>&1 HarnessCommand: home = "/tmp/shellbench-hermes" provider_name = "custom:shellbench" + agent_config: dict[str, object] = {"max_turns": 90} + if run.reasoning_effort: + agent_config["reasoning_effort"] = run.reasoning_effort config: dict[str, object] = { "model": { "default": run.model_id, @@ -741,7 +746,7 @@ def _hermes( } }, "toolsets": ["hermes-cli"], - "agent": {"max_turns": 90}, + "agent": agent_config, "memory": {"memory_enabled": False, "user_profile_enabled": False}, "compression": {"enabled": True, "threshold": 0.85}, "terminal": {"backend": "local", "timeout": 180}, @@ -817,11 +822,17 @@ def _codex( f'printf %s {auth_json} > "$CODEX_HOME/auth.json"; ' f'printf %s {config_text} > "$CODEX_HOME/config.toml"' ) + reasoning_override = ( + f"-c {shlex.quote(f'model_reasoning_effort={json.dumps(run.reasoning_effort)}')} " + if run.reasoning_effort + else "" + ) run_command = ( f"export PATH={_base_path()}; export CODEX_HOME={home}; " "codex exec --dangerously-bypass-approvals-and-sandbox " "--skip-git-repo-check " f"--model {shlex.quote(run.model_id)} " + f"{reasoning_override}" "--json --enable unified_exec -- " '"$(cat /tmp/shellbench-instruction.md)" ' ">/logs/agent/codex.txt 2>/logs/agent/codex-stderr.txt " @@ -871,11 +882,16 @@ def _claude_code( 'mkdir -p "$CLAUDE_CONFIG_DIR/debug" "$CLAUDE_CONFIG_DIR/projects/-app"; ' f'printf %s {mcp_json} > "$CLAUDE_CONFIG_DIR/.claude.json"' ) + # Claude Code names its top effort level "max"; the other native harnesses + # use ShellBench's canonical "xhigh" spelling. + claude_effort = "max" if run.reasoning_effort == "xhigh" else run.reasoning_effort + effort_option = f"--effort {shlex.quote(claude_effort)} " if claude_effort else "" run_command = ( f"export PATH={_base_path()}; export CLAUDE_CONFIG_DIR={home}; " "claude --verbose --output-format=stream-json " "--permission-mode=bypassPermissions --print " f"--model {shlex.quote(run.model_id)} " + f"{effort_option}" '"$(cat /tmp/shellbench-instruction.md)" ' ">/logs/agent/claude-code.txt 2>&1 dict[str, object]: return asdict(self) @@ -162,6 +163,7 @@ def build_matrix_plan( repetition=repetition, expected_task_count=expected_task_count, run_date=stamp, + reasoning_effort=reasoning_effort, ) ) return plan diff --git a/scripts/native_eval/remote_run.sh b/scripts/native_eval/remote_run.sh index 0a4976a..d37fdc1 100755 --- a/scripts/native_eval/remote_run.sh +++ b/scripts/native_eval/remote_run.sh @@ -61,6 +61,8 @@ cleanup() { trap cleanup EXIT umask 077 +# Fleet's planned value takes precedence over defaults in the provider file. +PLANNED_REASONING_EFFORT="${SHELLBENCH_REASONING_EFFORT:-}" set -a # shellcheck disable=SC1090 source "$ENV_FILE" @@ -77,37 +79,6 @@ printf '%s\n' "running" > "$RUN_STATE_DIR/state" export SHELLBENCH_PROXY_KEY="${SHELLBENCH_PROXY_KEY:-$(openssl rand -hex 32)}" cd "$ROOT/runner" -python3 - <"$PROXY_LOG" 2>&1 & -PROXY_PID="$!" - -for _ in $(seq 1 120); do - if curl -fsS \ - -H "Authorization: Bearer $SHELLBENCH_PROXY_KEY" \ - http://127.0.0.1:4000/health/liveliness >/dev/null 2>&1; then - break - fi - if ! kill -0 "$PROXY_PID" 2>/dev/null; then - echo "LiteLLM proxy exited during startup" >&2 - tail -100 "$PROXY_LOG" >&2 || true - exit 1 - fi - sleep 1 -done - -curl -fsS \ - -H "Authorization: Bearer $SHELLBENCH_PROXY_KEY" \ - http://127.0.0.1:4000/health/liveliness >/dev/null - RUN_COMMAND=( python3 -m scripts.native_eval.run_job --tasks-root "$TASKS_ROOT" @@ -128,6 +99,9 @@ RUN_COMMAND=( --proxy-url "http://host.docker.internal:4000" --concurrency "$CONCURRENCY" ) +if [[ -n "$PLANNED_REASONING_EFFORT" ]]; then + RUN_COMMAND+=(--reasoning-effort "$PLANNED_REASONING_EFFORT") +fi if [[ -n "$RERUN_OF_CANONICAL_RUN" ]]; then RUN_COMMAND+=(--rerun-of-canonical-run "$RERUN_OF_CANONICAL_RUN") fi @@ -135,6 +109,35 @@ for task_name in "${TASK_NAMES[@]}"; do RUN_COMMAND+=(--task "$task_name") done +# Resolve once before proxy startup; the runner must see this same value. +SHELLBENCH_REASONING_EFFORT=$("${RUN_COMMAND[@]}" --prepare-proxy-config "$PROXY_CONFIG") +export SHELLBENCH_REASONING_EFFORT + +"$TOOLCHAIN_ROOT/litellm-venv/bin/litellm" \ + --config "$PROXY_CONFIG" \ + --host 0.0.0.0 \ + --port 4000 \ + >"$PROXY_LOG" 2>&1 & +PROXY_PID="$!" + +for _ in $(seq 1 120); do + if curl -fsS \ + -H "Authorization: Bearer $SHELLBENCH_PROXY_KEY" \ + http://127.0.0.1:4000/health/liveliness >/dev/null 2>&1; then + break + fi + if ! kill -0 "$PROXY_PID" 2>/dev/null; then + echo "LiteLLM proxy exited during startup" >&2 + tail -100 "$PROXY_LOG" >&2 || true + exit 1 + fi + sleep 1 +done + +curl -fsS \ + -H "Authorization: Bearer $SHELLBENCH_PROXY_KEY" \ + http://127.0.0.1:4000/health/liveliness >/dev/null + set +e "${RUN_COMMAND[@]}" RUN_STATUS="$?" diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index d1d8ab4..d594b73 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -17,6 +17,7 @@ model_by_slug, trajectory_mode_for_harness, ) +from scripts.native_eval.proxy import REASONING_EFFORTS, write_proxy_config from scripts.native_eval.runtime import atomic_write_json, run_trial, utc_now from scripts.native_eval.tasks import TaskSpec, validate_suite @@ -298,7 +299,7 @@ def _run_manifest( "SHELLBENCH_HARBOR_REFERENCE_COMMIT" ), "judge_model_id": os.environ.get("SHELLBENCH_JUDGE_MODEL_ID"), - "reasoning_effort": os.environ.get("SHELLBENCH_REASONING_EFFORT"), + "reasoning_effort": run.reasoning_effort, "judge_reasoning_effort": os.environ.get( "SHELLBENCH_JUDGE_REASONING_EFFORT" ), @@ -391,6 +392,16 @@ def _runner_patch_hash() -> str: def build_run_spec(args: argparse.Namespace) -> RunSpec: harness = harness_by_name(args.harness) model = model_by_slug(args.model_slug) + cli_effort = getattr(args, "reasoning_effort", None) + environment_effort = os.environ.get("SHELLBENCH_REASONING_EFFORT", "").strip() + if cli_effort and environment_effort and cli_effort != environment_effort: + raise ValueError( + "--reasoning-effort conflicts with SHELLBENCH_REASONING_EFFORT; " + "resolve it with --prepare-proxy-config before starting the proxy" + ) + reasoning_effort = cli_effort or environment_effort or None + if reasoning_effort is not None and reasoning_effort not in REASONING_EFFORTS: + raise ValueError("reasoning effort must be low, medium, high, or xhigh") return RunSpec( run_label=args.run_label, harness=harness.name, @@ -402,6 +413,7 @@ def build_run_spec(args: argparse.Namespace) -> RunSpec: repetition=args.repetition, expected_task_count=args.expected_task_count, run_date=args.run_date, + reasoning_effort=reasoning_effort, ) @@ -428,6 +440,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--public-tasks-commit", required=True) parser.add_argument("--task-suite-path", required=True) parser.add_argument("--run-date", required=True) + parser.add_argument( + "--reasoning-effort", + choices=("low", "medium", "high", "xhigh"), + ) + parser.add_argument( + "--prepare-proxy-config", + type=Path, + help="Resolve effort, write proxy config, print the effort, and exit before running tasks.", + ) parser.add_argument( "--toolchain-root", type=Path, @@ -450,12 +471,20 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: def main() -> None: args = parse_args() + # Overrides are safe only while preparing a proxy that has not started yet. + if args.prepare_proxy_config and args.reasoning_effort: + os.environ["SHELLBENCH_REASONING_EFFORT"] = args.reasoning_effort + run = build_run_spec(args) + if args.prepare_proxy_config: + write_proxy_config(args.prepare_proxy_config) + print(run.reasoning_effort) + return proxy_key = os.environ.get("SHELLBENCH_PROXY_KEY", "") state = asyncio.run( run_job( tasks_root=args.tasks_root, jobs_dir=args.jobs_dir, - run=build_run_spec(args), + run=run, public_tasks_commit=args.public_tasks_commit, task_suite_path=args.task_suite_path, toolchain_root=args.toolchain_root, diff --git a/tests/test_native_eval_effort.py b/tests/test_native_eval_effort.py new file mode 100644 index 0000000..d4e6314 --- /dev/null +++ b/tests/test_native_eval_effort.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from uuid import uuid4 + +import pytest + +from scripts.native_eval.fleet import FleetController +from scripts.native_eval.models import build_matrix_plan +from scripts.native_eval.run_job import _run_manifest, build_run_spec, parse_args + + +def _args() -> list[str]: + return [ + "--tasks-root", + "tasks", + "--jobs-dir", + "jobs", + "--run-label", + "effort-test", + "--harness", + "openclaw", + "--model-slug", + "gpt55", + "--repetition", + "1", + "--expected-task-count", + "1", + "--public-tasks-commit", + "test", + "--task-suite-path", + "tasks", + "--run-date", + "20260828", + ] + + +@pytest.mark.parametrize("environment,cli", [("low", "high"), ("high", "low")]) +def test_runner_rejects_late_effort_override( + monkeypatch: pytest.MonkeyPatch, environment: str, cli: str +) -> None: + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", environment) + args = parse_args([*_args(), "--reasoning-effort", cli]) + with pytest.raises(ValueError, match="conflicts with SHELLBENCH_REASONING_EFFORT"): + build_run_spec(args) + + +@pytest.mark.parametrize("effort", [None, "high"]) +def test_manifest_does_not_reread_effort_from_environment( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, effort: str | None +) -> None: + monkeypatch.delenv("SHELLBENCH_REASONING_EFFORT", raising=False) + run = build_run_spec(parse_args(_args() + (["--reasoning-effort", effort] if effort else []))) + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "low") + manifest = _run_manifest( + run, + public_tasks_commit="test", + task_suite_path="tasks", + concurrency=1, + started_at="test", + tasks_root=tmp_path, + tasks=[], + ) + assert manifest["reasoning_effort"] == effort + + +def test_fleet_keeps_effort_in_run_spec() -> None: + planned = build_matrix_plan(1, reasoning_effort="xhigh")[0] + controller = object.__new__(FleetController) + assert controller._run_spec(planned.to_dict()).reasoning_effort == "xhigh" + + +@pytest.mark.parametrize("harness", ["openclaw", "hermes", "codex", "claude-code"]) +@pytest.mark.parametrize("planned", ["high", "xhigh", None, "invalid"]) +def test_remote_resolves_effort_before_proxy_startup( + tmp_path: Path, harness: str, planned: str | None +) -> None: + bash = shutil.which("bash") + assert bash is not None + if subprocess.run( + [ + bash, + "-c", + "(( BASH_VERSINFO[0] > 4 || (BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] >= 4) ))", + ], + check=False, + ).returncode: + pytest.skip("remote_run.sh requires Bash 4.4+") + repo = Path(__file__).resolve().parents[1] + label = f"shellbench-effort-{uuid4().hex}" + root = tmp_path / "remote" + root.mkdir() + (root / "runner").symlink_to(repo, target_is_directory=True) + tasks = tmp_path / "tasks" + for name, content in { + "task.toml": "", + "instruction.md": "do the task", + "environment/Dockerfile": "FROM scratch\n", + "solution/solve.sh": "true\n", + "tests/test.sh": "true\n", + }.items(): + path = tasks / "example" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + toolchain = tmp_path / "toolchain" + proxy = toolchain / "litellm-venv/bin/litellm" + proxy.parent.mkdir(parents=True) + proxy.write_text( + '#!/bin/sh\ncp "$2" "$PROXY_OBSERVED"\n' + 'printf "%s" "$SHELLBENCH_REASONING_EFFORT" > "$PROXY_ENV_OBSERVED"\n' + "exec sleep 60\n", + encoding="utf-8", + ) + proxy.chmod(0o755) + env_file = tmp_path / "provider.env" + env_file.write_text( + "SHELLBENCH_REASONING_EFFORT=low\nSHELLBENCH_JUDGE_REASONING_EFFORT=medium\n", + encoding="utf-8", + ) + runner = tmp_path / "run_with_fake_trial.py" + runner.write_text( + """import json +import os +import sys +from pathlib import Path +from scripts.native_eval import run_job +from scripts.native_eval.harnesses import build_harness_command + +async def trial(task, run, **kwargs): + command = build_harness_command( + run, proxy_url=kwargs["proxy_url"], proxy_key=kwargs["proxy_key"], + mcp_servers=task.mcp_servers, + ) + observed = {"effort": run.reasoning_effort, + "setup": command.setup_command, "command": command.run_command} + Path(os.environ["HARNESS_OBSERVED"]).write_text(json.dumps(observed)) + result = {"execution_outcome": {"kind": "clean"}, "agent_result": {}} + trial_dir = kwargs["job_dir"] / "example__trial" + trial_dir.mkdir() + (trial_dir / "result.json").write_text(json.dumps(result)) + return result + +run_job.run_trial = trial +sys.argv = ["run_job", *sys.argv[3:]] +run_job.main() +""", + encoding="utf-8", + ) + shell_env = tmp_path / "bash_env" + shell_env.write_text( + """python3() { + if [[ "$1" == "-" || "$*" == *--prepare-proxy-config* ]]; then + command "$TEST_PYTHON" "$@" + else + command "$TEST_PYTHON" "$TEST_RUNNER" "$@" + fi +} +curl() { test -f "$PROXY_ENV_OBSERVED"; } +sudo() { "$@"; } +""", + encoding="utf-8", + ) + env = { + **os.environ, + "BASH_ENV": str(shell_env), + "PYTHONPATH": str(repo), + "TEST_PYTHON": sys.executable, + "TEST_RUNNER": str(runner), + "TOOLCHAIN_ROOT": str(toolchain), + "SHELLBENCH_PROXY_KEY": "synthetic-test-key", + "SHELLBENCH_JUDGE_MODEL_ID": "gpt-5.5", + "PROXY_OBSERVED": str(tmp_path / "proxy-observed.json"), + "PROXY_ENV_OBSERVED": str(tmp_path / "proxy-env.txt"), + "HARNESS_OBSERVED": str(tmp_path / "harness-observed.json"), + } + env.pop("SHELLBENCH_REASONING_EFFORT", None) + if planned is not None: + env["SHELLBENCH_REASONING_EFFORT"] = planned + archive = Path("/tmp") / f"{label}-final-artifacts.tar.gz" + try: + process = subprocess.run( + [ + bash, + str(repo / "scripts/native_eval/remote_run.sh"), + str(root), + str(tasks), + str(env_file), + label, + harness, + "gpt55", + "1", + "1", + "test", + "20260828", + "1", + "test", + "gpt-5.5", + "openai", + "gpt-5.5", + "", + ], + env=env, + capture_output=True, + text=True, + timeout=45, + check=False, + ) + if planned == "invalid": + assert process.returncode != 0 + assert not Path(env["PROXY_OBSERVED"]).exists() + assert not Path(env["HARNESS_OBSERVED"]).exists() + return + assert process.returncode == 0, process.stderr + expected = planned or "low" + config = json.loads(Path(env["PROXY_OBSERVED"]).read_text()) + observed = json.loads(Path(env["HARNESS_OBSERVED"]).read_text()) + manifest = json.loads((root / "results/jobs" / label / "run_manifest.json").read_text()) + assert config["shellbench_native"]["reasoning_effort"] == expected + assert Path(env["PROXY_ENV_OBSERVED"]).read_text() == expected + models = {item["model_name"]: item for item in config["model_list"]} + assert models["gpt-5.5"]["litellm_params"]["reasoning_effort"] == expected + assert models["shellbench-judge"]["litellm_params"]["reasoning_effort"] == "medium" + assert observed["effort"] == manifest["reasoning_effort"] == expected + if harness == "openclaw": + assert f"--thinking {expected}" in observed["command"] + elif harness == "hermes": + assert f'"reasoning_effort":"{expected}"' in observed["setup"] + elif harness == "codex": + assert f'model_reasoning_effort="{expected}"' in observed["command"] + else: + assert f"--effort {'max' if expected == 'xhigh' else expected}" in observed["command"] + finally: + archive.unlink(missing_ok=True) + shutil.rmtree(Path("/tmp/shellbench-runs") / label, ignore_errors=True) + shutil.rmtree(Path("/tmp") / f"shellbench_meta-{label}", ignore_errors=True) diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index aee94cf..2304ff7 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -26,6 +26,10 @@ from scripts.native_eval.models import RunSpec +# Bound deadlocks, not scheduler latency; only the tests release blocked jobs. +SCHEDULER_TEST_TIMEOUT = 30 + + def _run_spec( label: str, *, @@ -109,7 +113,7 @@ def test_remote_run_archives_terminal_status(tmp_path: Path, exit_code: int) -> shell_env = tmp_path / "bash_env" shell_env.write_text( '''python3() { - if [[ "$1" == "-" ]]; then cat >/dev/null; return 0; fi + if [[ "$*" == *--prepare-proxy-config* ]]; then printf 'high\\n'; return 0; fi mkdir -p "$TEST_ROOT/results/jobs/$TEST_LABEL/task__trial" printf '{}\\n' > "$TEST_ROOT/results/jobs/$TEST_LABEL/task__trial/result.json" printf '{}\\n' > "$TEST_ROOT/results/jobs/$TEST_LABEL/run_manifest.json" @@ -1067,12 +1071,12 @@ def test_slow_capped_model_does_not_block_refilling_eligible_slot( ) controller.start() try: - assert executor.wait_for_dispatch(later_gpt, timeout=2) + assert executor.wait_for_dispatch(later_gpt, timeout=SCHEDULER_TEST_TIMEOUT) assert capped_fable not in executor.dispatches assert not release_slow.is_set() finally: release_slow.set() - controller.join(timeout=5) + controller.join(timeout=SCHEDULER_TEST_TIMEOUT) assert not controller.is_alive() assert result == [0] @@ -1143,7 +1147,9 @@ def test_recovery_pending_entries_respect_capacity_behind_owned_runs( ) controller.start() try: - assert executor.wait_for_dispatch(pending_labels[0], timeout=2) + assert executor.wait_for_dispatch( + pending_labels[0], timeout=SCHEDULER_TEST_TIMEOUT + ) assert executor.dispatches == [pending_labels[0]] index = json.loads(run_index.read_text(encoding="utf-8")) pending_statuses = { @@ -1155,7 +1161,7 @@ def test_recovery_pending_entries_respect_capacity_behind_owned_runs( assert executor.active_leases == 10 finally: release_runs.set() - controller.join(timeout=10) + controller.join(timeout=SCHEDULER_TEST_TIMEOUT) assert not controller.is_alive() assert result == [0] diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 280fdd9..2c59fbe 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -495,6 +495,7 @@ def test_run_manifest_records_native_audit_metadata( repetition=1, expected_task_count=2, run_date="20260727", + reasoning_effort="high", ) manifest = _run_manifest( @@ -789,6 +790,96 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: assert "cat /logs/agent/codex-stderr.txt >&2" in command.run_command +@pytest.mark.parametrize("effort", ("low", "medium", "high", "xhigh")) +@pytest.mark.parametrize("harness", ("openclaw", "hermes", "codex", "claude-code")) +def test_harness_commands_apply_native_reasoning_effort( + harness: str, + effort: str, +) -> None: + run = RunSpec( + run_label=f"{harness}-{effort}", + harness=harness, + harness_version="test", + model_slug="gpt56-sol", + model_id="gpt-5.6-sol", + provider="openai", + proxy_model_name="gpt-5.6-sol", + repetition=1, + expected_task_count=116, + run_date="20260729", + reasoning_effort=effort, + ) + + command = build_harness_command( + run, + proxy_url="http://host.docker.internal:4000", + proxy_key="local-proxy-key", + mcp_servers=(), + ) + + if harness == "openclaw": + assert f"--thinking {effort}" in command.run_command + elif harness == "hermes": + assert f'"reasoning_effort":"{effort}"' in command.setup_command + elif harness == "codex": + assert f'model_reasoning_effort="{effort}"' in command.run_command + else: + expected_effort = "max" if effort == "xhigh" else effort + assert f"--effort {expected_effort}" in command.run_command + + +def test_harness_commands_preserve_defaults_without_reasoning_effort() -> None: + commands = {} + for harness in ("openclaw", "hermes", "codex", "claude-code"): + run = RunSpec( + run_label=f"{harness}-default", + harness=harness, + harness_version="test", + model_slug="gpt55", + model_id="gpt-5.5", + provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=116, + run_date="20260729", + ) + commands[harness] = build_harness_command( + run, + proxy_url="http://host.docker.internal:4000", + proxy_key="local-proxy-key", + mcp_servers=(), + ) + + assert "--thinking off" in commands["openclaw"].run_command + assert '"reasoning_effort"' not in commands["hermes"].setup_command + assert "model_reasoning_effort" not in commands["codex"].run_command + assert "--effort" not in commands["claude-code"].run_command + + +def test_build_run_spec_records_reasoning_effort_from_environment( + monkeypatch, +) -> None: + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "xhigh") + + run = build_run_spec( + Namespace( + run_label="openclaw-gpt56-sol-xhigh", + harness="openclaw", + harness_version="test", + model_slug="gpt56-sol", + model_id=None, + model_provider=None, + proxy_model_name=None, + repetition=1, + expected_task_count=116, + run_date="20260729", + reasoning_effort=None, + ) + ) + + assert run.reasoning_effort == "xhigh" + + def test_openclaw_completion_probe_accepts_markerless_final_envelope( tmp_path: Path, ) -> None: