diff --git a/CHANGELOG.md b/CHANGELOG.md index ccbe8a7..e6745cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed +- Apply planned native reasoning effort to all four harnesses and resolve proxy, runner, and manifest precedence before proxy startup (#53, thanks @vincentkoc). - Rehydrate replacement native fleet leases instead of trusting stale bootstrap timestamps (#58, thanks @vincentkoc). - Separate native execution validity from diagnostic rewards, reject wholly invalid runs, and preserve terminal exit status in recovery archives (#63, thanks @vincentkoc). - Install only the assigned native harness on each fleet lease and record only its version in the toolchain manifest (#65). diff --git a/docs/native_reasoning_effort.md b/docs/native_reasoning_effort.md new file mode 100644 index 0000000..1ec7239 --- /dev/null +++ b/docs/native_reasoning_effort.md @@ -0,0 +1,28 @@ +# Native reasoning effort + +The native fleet's planned `reasoning_effort` controls both the provider proxy +and the agent harness. Supported values are `low`, `medium`, `high`, and `xhigh`. +OpenClaw receives `--thinking`, Hermes receives `agent.reasoning_effort`, Codex +receives `model_reasoning_effort`, and Claude Code receives `--effort` (`max` for +the canonical `xhigh` value). The run manifest records the resolved run value, +not a later reading of the environment. + +`remote_run.sh` preserves the effort supplied by fleet dispatch before sourcing +the provider environment file. That planned value takes precedence over the +file's default. If dispatch supplies no effort, the environment file supplies +the default instead. Judge reasoning effort remains independent. + +Before starting LiteLLM, the launcher invokes `run_job` with the complete run +arguments and `--prepare-proxy-config PATH`. This preparation mode lets an +explicit `--reasoning-effort` override the environment, validates the result, +writes the proxy configuration, and prints the canonical effort to standard +output. The launcher exports that value as `SHELLBENCH_REASONING_EFFORT` before +starting either the proxy or the job. Invalid values stop startup before the +proxy is launched. + +For manual launches, use the same preparation step before starting the proxy +and export its returned value for the runner. An ordinary `run_job` invocation +rejects conflicting CLI and environment efforts: it cannot safely override a +proxy that is already running. Without either value, direct harness construction +retains each harness's defaults; the managed proxy still requires an explicit +valid effort, as before. diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index b258495..74e0776 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -1479,7 +1479,10 @@ def _matrix_satisfied(self) -> bool: def _run_spec(self, entry: dict[str, Any]) -> RunSpec: try: - return RunSpec(**{field: entry[field] for field in RUN_SPEC_FIELDS}) + return RunSpec( + **{field: entry[field] for field in RUN_SPEC_FIELDS}, + reasoning_effort=entry.get("reasoning_effort"), + ) except KeyError as exc: raise FleetError( f"run entry {entry.get('run_label', '')} lacks {exc.args[0]}" diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index b3c3aa0..03624da 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -532,6 +532,7 @@ def _openclaw( ) -> HarnessCommand: provider = "openai" model = f"{provider}/{run.model_id}" + thinking = run.reasoning_effort or "off" home = "/tmp/shellbench-openclaw" audit_plugin_root = f"{home}/.openclaw/shellbench-audit" gateway_token = secrets.token_urlsafe(32) @@ -616,7 +617,8 @@ def _openclaw( 'kill "$gateway_pid" 2>/dev/null || true; ' 'wait "$gateway_pid" 2>/dev/null || true; ' 'cat "$gateway_log" >&2; exit 70; fi; ' - "openclaw agent --json --agent main --thinking off " + "openclaw agent --json --agent main " + f"--thinking {shlex.quote(thinking)} " f"--model {shlex.quote(model)} " '--message "$(cat /tmp/shellbench-instruction.md)" ' '>"$log" 2>&1 HarnessCommand: home = "/tmp/shellbench-hermes" provider_name = "custom:shellbench" + agent_config: dict[str, object] = {"max_turns": 90} + if run.reasoning_effort: + agent_config["reasoning_effort"] = run.reasoning_effort config: dict[str, object] = { "model": { "default": run.model_id, @@ -741,7 +746,7 @@ def _hermes( } }, "toolsets": ["hermes-cli"], - "agent": {"max_turns": 90}, + "agent": agent_config, "memory": {"memory_enabled": False, "user_profile_enabled": False}, "compression": {"enabled": True, "threshold": 0.85}, "terminal": {"backend": "local", "timeout": 180}, @@ -817,11 +822,17 @@ def _codex( f'printf %s {auth_json} > "$CODEX_HOME/auth.json"; ' f'printf %s {config_text} > "$CODEX_HOME/config.toml"' ) + reasoning_override = ( + f"-c {shlex.quote(f'model_reasoning_effort={json.dumps(run.reasoning_effort)}')} " + if run.reasoning_effort + else "" + ) run_command = ( f"export PATH={_base_path()}; export CODEX_HOME={home}; " "codex exec --dangerously-bypass-approvals-and-sandbox " "--skip-git-repo-check " f"--model {shlex.quote(run.model_id)} " + f"{reasoning_override}" "--json --enable unified_exec -- " '"$(cat /tmp/shellbench-instruction.md)" ' ">/logs/agent/codex.txt 2>/logs/agent/codex-stderr.txt " @@ -871,11 +882,16 @@ def _claude_code( 'mkdir -p "$CLAUDE_CONFIG_DIR/debug" "$CLAUDE_CONFIG_DIR/projects/-app"; ' f'printf %s {mcp_json} > "$CLAUDE_CONFIG_DIR/.claude.json"' ) + # Claude Code names its top effort level "max"; the other native harnesses + # use ShellBench's canonical "xhigh" spelling. + claude_effort = "max" if run.reasoning_effort == "xhigh" else run.reasoning_effort + effort_option = f"--effort {shlex.quote(claude_effort)} " if claude_effort else "" run_command = ( f"export PATH={_base_path()}; export CLAUDE_CONFIG_DIR={home}; " "claude --verbose --output-format=stream-json " "--permission-mode=bypassPermissions --print " f"--model {shlex.quote(run.model_id)} " + f"{effort_option}" '"$(cat /tmp/shellbench-instruction.md)" ' ">/logs/agent/claude-code.txt 2>&1 dict[str, object]: return asdict(self) @@ -162,6 +163,7 @@ def build_matrix_plan( repetition=repetition, expected_task_count=expected_task_count, run_date=stamp, + reasoning_effort=reasoning_effort, ) ) return plan diff --git a/scripts/native_eval/remote_run.sh b/scripts/native_eval/remote_run.sh index 0a4976a..d37fdc1 100755 --- a/scripts/native_eval/remote_run.sh +++ b/scripts/native_eval/remote_run.sh @@ -61,6 +61,8 @@ cleanup() { trap cleanup EXIT umask 077 +# Fleet's planned value takes precedence over defaults in the provider file. +PLANNED_REASONING_EFFORT="${SHELLBENCH_REASONING_EFFORT:-}" set -a # shellcheck disable=SC1090 source "$ENV_FILE" @@ -77,37 +79,6 @@ printf '%s\n' "running" > "$RUN_STATE_DIR/state" export SHELLBENCH_PROXY_KEY="${SHELLBENCH_PROXY_KEY:-$(openssl rand -hex 32)}" cd "$ROOT/runner" -python3 - <"$PROXY_LOG" 2>&1 & -PROXY_PID="$!" - -for _ in $(seq 1 120); do - if curl -fsS \ - -H "Authorization: Bearer $SHELLBENCH_PROXY_KEY" \ - http://127.0.0.1:4000/health/liveliness >/dev/null 2>&1; then - break - fi - if ! kill -0 "$PROXY_PID" 2>/dev/null; then - echo "LiteLLM proxy exited during startup" >&2 - tail -100 "$PROXY_LOG" >&2 || true - exit 1 - fi - sleep 1 -done - -curl -fsS \ - -H "Authorization: Bearer $SHELLBENCH_PROXY_KEY" \ - http://127.0.0.1:4000/health/liveliness >/dev/null - RUN_COMMAND=( python3 -m scripts.native_eval.run_job --tasks-root "$TASKS_ROOT" @@ -128,6 +99,9 @@ RUN_COMMAND=( --proxy-url "http://host.docker.internal:4000" --concurrency "$CONCURRENCY" ) +if [[ -n "$PLANNED_REASONING_EFFORT" ]]; then + RUN_COMMAND+=(--reasoning-effort "$PLANNED_REASONING_EFFORT") +fi if [[ -n "$RERUN_OF_CANONICAL_RUN" ]]; then RUN_COMMAND+=(--rerun-of-canonical-run "$RERUN_OF_CANONICAL_RUN") fi @@ -135,6 +109,35 @@ for task_name in "${TASK_NAMES[@]}"; do RUN_COMMAND+=(--task "$task_name") done +# Resolve once before proxy startup; the runner must see this same value. +SHELLBENCH_REASONING_EFFORT=$("${RUN_COMMAND[@]}" --prepare-proxy-config "$PROXY_CONFIG") +export SHELLBENCH_REASONING_EFFORT + +"$TOOLCHAIN_ROOT/litellm-venv/bin/litellm" \ + --config "$PROXY_CONFIG" \ + --host 0.0.0.0 \ + --port 4000 \ + >"$PROXY_LOG" 2>&1 & +PROXY_PID="$!" + +for _ in $(seq 1 120); do + if curl -fsS \ + -H "Authorization: Bearer $SHELLBENCH_PROXY_KEY" \ + http://127.0.0.1:4000/health/liveliness >/dev/null 2>&1; then + break + fi + if ! kill -0 "$PROXY_PID" 2>/dev/null; then + echo "LiteLLM proxy exited during startup" >&2 + tail -100 "$PROXY_LOG" >&2 || true + exit 1 + fi + sleep 1 +done + +curl -fsS \ + -H "Authorization: Bearer $SHELLBENCH_PROXY_KEY" \ + http://127.0.0.1:4000/health/liveliness >/dev/null + set +e "${RUN_COMMAND[@]}" RUN_STATUS="$?" diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index d1d8ab4..d594b73 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -17,6 +17,7 @@ model_by_slug, trajectory_mode_for_harness, ) +from scripts.native_eval.proxy import REASONING_EFFORTS, write_proxy_config from scripts.native_eval.runtime import atomic_write_json, run_trial, utc_now from scripts.native_eval.tasks import TaskSpec, validate_suite @@ -298,7 +299,7 @@ def _run_manifest( "SHELLBENCH_HARBOR_REFERENCE_COMMIT" ), "judge_model_id": os.environ.get("SHELLBENCH_JUDGE_MODEL_ID"), - "reasoning_effort": os.environ.get("SHELLBENCH_REASONING_EFFORT"), + "reasoning_effort": run.reasoning_effort, "judge_reasoning_effort": os.environ.get( "SHELLBENCH_JUDGE_REASONING_EFFORT" ), @@ -391,6 +392,16 @@ def _runner_patch_hash() -> str: def build_run_spec(args: argparse.Namespace) -> RunSpec: harness = harness_by_name(args.harness) model = model_by_slug(args.model_slug) + cli_effort = getattr(args, "reasoning_effort", None) + environment_effort = os.environ.get("SHELLBENCH_REASONING_EFFORT", "").strip() + if cli_effort and environment_effort and cli_effort != environment_effort: + raise ValueError( + "--reasoning-effort conflicts with SHELLBENCH_REASONING_EFFORT; " + "resolve it with --prepare-proxy-config before starting the proxy" + ) + reasoning_effort = cli_effort or environment_effort or None + if reasoning_effort is not None and reasoning_effort not in REASONING_EFFORTS: + raise ValueError("reasoning effort must be low, medium, high, or xhigh") return RunSpec( run_label=args.run_label, harness=harness.name, @@ -402,6 +413,7 @@ def build_run_spec(args: argparse.Namespace) -> RunSpec: repetition=args.repetition, expected_task_count=args.expected_task_count, run_date=args.run_date, + reasoning_effort=reasoning_effort, ) @@ -428,6 +440,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--public-tasks-commit", required=True) parser.add_argument("--task-suite-path", required=True) parser.add_argument("--run-date", required=True) + parser.add_argument( + "--reasoning-effort", + choices=("low", "medium", "high", "xhigh"), + ) + parser.add_argument( + "--prepare-proxy-config", + type=Path, + help="Resolve effort, write proxy config, print the effort, and exit before running tasks.", + ) parser.add_argument( "--toolchain-root", type=Path, @@ -450,12 +471,20 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: def main() -> None: args = parse_args() + # Overrides are safe only while preparing a proxy that has not started yet. + if args.prepare_proxy_config and args.reasoning_effort: + os.environ["SHELLBENCH_REASONING_EFFORT"] = args.reasoning_effort + run = build_run_spec(args) + if args.prepare_proxy_config: + write_proxy_config(args.prepare_proxy_config) + print(run.reasoning_effort) + return proxy_key = os.environ.get("SHELLBENCH_PROXY_KEY", "") state = asyncio.run( run_job( tasks_root=args.tasks_root, jobs_dir=args.jobs_dir, - run=build_run_spec(args), + run=run, public_tasks_commit=args.public_tasks_commit, task_suite_path=args.task_suite_path, toolchain_root=args.toolchain_root, diff --git a/tests/test_native_eval_effort.py b/tests/test_native_eval_effort.py new file mode 100644 index 0000000..d4e6314 --- /dev/null +++ b/tests/test_native_eval_effort.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from uuid import uuid4 + +import pytest + +from scripts.native_eval.fleet import FleetController +from scripts.native_eval.models import build_matrix_plan +from scripts.native_eval.run_job import _run_manifest, build_run_spec, parse_args + + +def _args() -> list[str]: + return [ + "--tasks-root", + "tasks", + "--jobs-dir", + "jobs", + "--run-label", + "effort-test", + "--harness", + "openclaw", + "--model-slug", + "gpt55", + "--repetition", + "1", + "--expected-task-count", + "1", + "--public-tasks-commit", + "test", + "--task-suite-path", + "tasks", + "--run-date", + "20260828", + ] + + +@pytest.mark.parametrize("environment,cli", [("low", "high"), ("high", "low")]) +def test_runner_rejects_late_effort_override( + monkeypatch: pytest.MonkeyPatch, environment: str, cli: str +) -> None: + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", environment) + args = parse_args([*_args(), "--reasoning-effort", cli]) + with pytest.raises(ValueError, match="conflicts with SHELLBENCH_REASONING_EFFORT"): + build_run_spec(args) + + +@pytest.mark.parametrize("effort", [None, "high"]) +def test_manifest_does_not_reread_effort_from_environment( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, effort: str | None +) -> None: + monkeypatch.delenv("SHELLBENCH_REASONING_EFFORT", raising=False) + run = build_run_spec(parse_args(_args() + (["--reasoning-effort", effort] if effort else []))) + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "low") + manifest = _run_manifest( + run, + public_tasks_commit="test", + task_suite_path="tasks", + concurrency=1, + started_at="test", + tasks_root=tmp_path, + tasks=[], + ) + assert manifest["reasoning_effort"] == effort + + +def test_fleet_keeps_effort_in_run_spec() -> None: + planned = build_matrix_plan(1, reasoning_effort="xhigh")[0] + controller = object.__new__(FleetController) + assert controller._run_spec(planned.to_dict()).reasoning_effort == "xhigh" + + +@pytest.mark.parametrize("harness", ["openclaw", "hermes", "codex", "claude-code"]) +@pytest.mark.parametrize("planned", ["high", "xhigh", None, "invalid"]) +def test_remote_resolves_effort_before_proxy_startup( + tmp_path: Path, harness: str, planned: str | None +) -> None: + bash = shutil.which("bash") + assert bash is not None + if subprocess.run( + [ + bash, + "-c", + "(( BASH_VERSINFO[0] > 4 || (BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] >= 4) ))", + ], + check=False, + ).returncode: + pytest.skip("remote_run.sh requires Bash 4.4+") + repo = Path(__file__).resolve().parents[1] + label = f"shellbench-effort-{uuid4().hex}" + root = tmp_path / "remote" + root.mkdir() + (root / "runner").symlink_to(repo, target_is_directory=True) + tasks = tmp_path / "tasks" + for name, content in { + "task.toml": "", + "instruction.md": "do the task", + "environment/Dockerfile": "FROM scratch\n", + "solution/solve.sh": "true\n", + "tests/test.sh": "true\n", + }.items(): + path = tasks / "example" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + toolchain = tmp_path / "toolchain" + proxy = toolchain / "litellm-venv/bin/litellm" + proxy.parent.mkdir(parents=True) + proxy.write_text( + '#!/bin/sh\ncp "$2" "$PROXY_OBSERVED"\n' + 'printf "%s" "$SHELLBENCH_REASONING_EFFORT" > "$PROXY_ENV_OBSERVED"\n' + "exec sleep 60\n", + encoding="utf-8", + ) + proxy.chmod(0o755) + env_file = tmp_path / "provider.env" + env_file.write_text( + "SHELLBENCH_REASONING_EFFORT=low\nSHELLBENCH_JUDGE_REASONING_EFFORT=medium\n", + encoding="utf-8", + ) + runner = tmp_path / "run_with_fake_trial.py" + runner.write_text( + """import json +import os +import sys +from pathlib import Path +from scripts.native_eval import run_job +from scripts.native_eval.harnesses import build_harness_command + +async def trial(task, run, **kwargs): + command = build_harness_command( + run, proxy_url=kwargs["proxy_url"], proxy_key=kwargs["proxy_key"], + mcp_servers=task.mcp_servers, + ) + observed = {"effort": run.reasoning_effort, + "setup": command.setup_command, "command": command.run_command} + Path(os.environ["HARNESS_OBSERVED"]).write_text(json.dumps(observed)) + result = {"execution_outcome": {"kind": "clean"}, "agent_result": {}} + trial_dir = kwargs["job_dir"] / "example__trial" + trial_dir.mkdir() + (trial_dir / "result.json").write_text(json.dumps(result)) + return result + +run_job.run_trial = trial +sys.argv = ["run_job", *sys.argv[3:]] +run_job.main() +""", + encoding="utf-8", + ) + shell_env = tmp_path / "bash_env" + shell_env.write_text( + """python3() { + if [[ "$1" == "-" || "$*" == *--prepare-proxy-config* ]]; then + command "$TEST_PYTHON" "$@" + else + command "$TEST_PYTHON" "$TEST_RUNNER" "$@" + fi +} +curl() { test -f "$PROXY_ENV_OBSERVED"; } +sudo() { "$@"; } +""", + encoding="utf-8", + ) + env = { + **os.environ, + "BASH_ENV": str(shell_env), + "PYTHONPATH": str(repo), + "TEST_PYTHON": sys.executable, + "TEST_RUNNER": str(runner), + "TOOLCHAIN_ROOT": str(toolchain), + "SHELLBENCH_PROXY_KEY": "synthetic-test-key", + "SHELLBENCH_JUDGE_MODEL_ID": "gpt-5.5", + "PROXY_OBSERVED": str(tmp_path / "proxy-observed.json"), + "PROXY_ENV_OBSERVED": str(tmp_path / "proxy-env.txt"), + "HARNESS_OBSERVED": str(tmp_path / "harness-observed.json"), + } + env.pop("SHELLBENCH_REASONING_EFFORT", None) + if planned is not None: + env["SHELLBENCH_REASONING_EFFORT"] = planned + archive = Path("/tmp") / f"{label}-final-artifacts.tar.gz" + try: + process = subprocess.run( + [ + bash, + str(repo / "scripts/native_eval/remote_run.sh"), + str(root), + str(tasks), + str(env_file), + label, + harness, + "gpt55", + "1", + "1", + "test", + "20260828", + "1", + "test", + "gpt-5.5", + "openai", + "gpt-5.5", + "", + ], + env=env, + capture_output=True, + text=True, + timeout=45, + check=False, + ) + if planned == "invalid": + assert process.returncode != 0 + assert not Path(env["PROXY_OBSERVED"]).exists() + assert not Path(env["HARNESS_OBSERVED"]).exists() + return + assert process.returncode == 0, process.stderr + expected = planned or "low" + config = json.loads(Path(env["PROXY_OBSERVED"]).read_text()) + observed = json.loads(Path(env["HARNESS_OBSERVED"]).read_text()) + manifest = json.loads((root / "results/jobs" / label / "run_manifest.json").read_text()) + assert config["shellbench_native"]["reasoning_effort"] == expected + assert Path(env["PROXY_ENV_OBSERVED"]).read_text() == expected + models = {item["model_name"]: item for item in config["model_list"]} + assert models["gpt-5.5"]["litellm_params"]["reasoning_effort"] == expected + assert models["shellbench-judge"]["litellm_params"]["reasoning_effort"] == "medium" + assert observed["effort"] == manifest["reasoning_effort"] == expected + if harness == "openclaw": + assert f"--thinking {expected}" in observed["command"] + elif harness == "hermes": + assert f'"reasoning_effort":"{expected}"' in observed["setup"] + elif harness == "codex": + assert f'model_reasoning_effort="{expected}"' in observed["command"] + else: + assert f"--effort {'max' if expected == 'xhigh' else expected}" in observed["command"] + finally: + archive.unlink(missing_ok=True) + shutil.rmtree(Path("/tmp/shellbench-runs") / label, ignore_errors=True) + shutil.rmtree(Path("/tmp") / f"shellbench_meta-{label}", ignore_errors=True) diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index aee94cf..2304ff7 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -26,6 +26,10 @@ from scripts.native_eval.models import RunSpec +# Bound deadlocks, not scheduler latency; only the tests release blocked jobs. +SCHEDULER_TEST_TIMEOUT = 30 + + def _run_spec( label: str, *, @@ -109,7 +113,7 @@ def test_remote_run_archives_terminal_status(tmp_path: Path, exit_code: int) -> shell_env = tmp_path / "bash_env" shell_env.write_text( '''python3() { - if [[ "$1" == "-" ]]; then cat >/dev/null; return 0; fi + if [[ "$*" == *--prepare-proxy-config* ]]; then printf 'high\\n'; return 0; fi mkdir -p "$TEST_ROOT/results/jobs/$TEST_LABEL/task__trial" printf '{}\\n' > "$TEST_ROOT/results/jobs/$TEST_LABEL/task__trial/result.json" printf '{}\\n' > "$TEST_ROOT/results/jobs/$TEST_LABEL/run_manifest.json" @@ -1067,12 +1071,12 @@ def test_slow_capped_model_does_not_block_refilling_eligible_slot( ) controller.start() try: - assert executor.wait_for_dispatch(later_gpt, timeout=2) + assert executor.wait_for_dispatch(later_gpt, timeout=SCHEDULER_TEST_TIMEOUT) assert capped_fable not in executor.dispatches assert not release_slow.is_set() finally: release_slow.set() - controller.join(timeout=5) + controller.join(timeout=SCHEDULER_TEST_TIMEOUT) assert not controller.is_alive() assert result == [0] @@ -1143,7 +1147,9 @@ def test_recovery_pending_entries_respect_capacity_behind_owned_runs( ) controller.start() try: - assert executor.wait_for_dispatch(pending_labels[0], timeout=2) + assert executor.wait_for_dispatch( + pending_labels[0], timeout=SCHEDULER_TEST_TIMEOUT + ) assert executor.dispatches == [pending_labels[0]] index = json.loads(run_index.read_text(encoding="utf-8")) pending_statuses = { @@ -1155,7 +1161,7 @@ def test_recovery_pending_entries_respect_capacity_behind_owned_runs( assert executor.active_leases == 10 finally: release_runs.set() - controller.join(timeout=10) + controller.join(timeout=SCHEDULER_TEST_TIMEOUT) assert not controller.is_alive() assert result == [0] diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 280fdd9..2c59fbe 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -495,6 +495,7 @@ def test_run_manifest_records_native_audit_metadata( repetition=1, expected_task_count=2, run_date="20260727", + reasoning_effort="high", ) manifest = _run_manifest( @@ -789,6 +790,96 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: assert "cat /logs/agent/codex-stderr.txt >&2" in command.run_command +@pytest.mark.parametrize("effort", ("low", "medium", "high", "xhigh")) +@pytest.mark.parametrize("harness", ("openclaw", "hermes", "codex", "claude-code")) +def test_harness_commands_apply_native_reasoning_effort( + harness: str, + effort: str, +) -> None: + run = RunSpec( + run_label=f"{harness}-{effort}", + harness=harness, + harness_version="test", + model_slug="gpt56-sol", + model_id="gpt-5.6-sol", + provider="openai", + proxy_model_name="gpt-5.6-sol", + repetition=1, + expected_task_count=116, + run_date="20260729", + reasoning_effort=effort, + ) + + command = build_harness_command( + run, + proxy_url="http://host.docker.internal:4000", + proxy_key="local-proxy-key", + mcp_servers=(), + ) + + if harness == "openclaw": + assert f"--thinking {effort}" in command.run_command + elif harness == "hermes": + assert f'"reasoning_effort":"{effort}"' in command.setup_command + elif harness == "codex": + assert f'model_reasoning_effort="{effort}"' in command.run_command + else: + expected_effort = "max" if effort == "xhigh" else effort + assert f"--effort {expected_effort}" in command.run_command + + +def test_harness_commands_preserve_defaults_without_reasoning_effort() -> None: + commands = {} + for harness in ("openclaw", "hermes", "codex", "claude-code"): + run = RunSpec( + run_label=f"{harness}-default", + harness=harness, + harness_version="test", + model_slug="gpt55", + model_id="gpt-5.5", + provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=116, + run_date="20260729", + ) + commands[harness] = build_harness_command( + run, + proxy_url="http://host.docker.internal:4000", + proxy_key="local-proxy-key", + mcp_servers=(), + ) + + assert "--thinking off" in commands["openclaw"].run_command + assert '"reasoning_effort"' not in commands["hermes"].setup_command + assert "model_reasoning_effort" not in commands["codex"].run_command + assert "--effort" not in commands["claude-code"].run_command + + +def test_build_run_spec_records_reasoning_effort_from_environment( + monkeypatch, +) -> None: + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "xhigh") + + run = build_run_spec( + Namespace( + run_label="openclaw-gpt56-sol-xhigh", + harness="openclaw", + harness_version="test", + model_slug="gpt56-sol", + model_id=None, + model_provider=None, + proxy_model_name=None, + repetition=1, + expected_task_count=116, + run_date="20260729", + reasoning_effort=None, + ) + ) + + assert run.reasoning_effort == "xhigh" + + def test_openclaw_completion_probe_accepts_markerless_final_envelope( tmp_path: Path, ) -> None: