fix(native-eval): resolve planned reasoning effort before proxy startup

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Fixed
 
+- Apply planned native reasoning effort to all four harnesses and resolve proxy, runner, and manifest precedence before proxy startup (#53, thanks @vincentkoc).
 - Rehydrate replacement native fleet leases instead of trusting stale bootstrap timestamps (#58, thanks @vincentkoc).
 - Separate native execution validity from diagnostic rewards, reject wholly invalid runs, and preserve terminal exit status in recovery archives (#63, thanks @vincentkoc).
 - Install only the assigned native harness on each fleet lease and record only its version in the toolchain manifest (#65).

docs/native_reasoning_effort.md

@@ -0,0 +1,28 @@
+# Native reasoning effort
+
+The native fleet's planned `reasoning_effort` controls both the provider proxy
+and the agent harness. Supported values are `low`, `medium`, `high`, and `xhigh`.
+OpenClaw receives `--thinking`, Hermes receives `agent.reasoning_effort`, Codex
+receives `model_reasoning_effort`, and Claude Code receives `--effort` (`max` for
+the canonical `xhigh` value). The run manifest records the resolved run value,
+not a later reading of the environment.
+
+`remote_run.sh` preserves the effort supplied by fleet dispatch before sourcing
+the provider environment file. That planned value takes precedence over the
+file's default. If dispatch supplies no effort, the environment file supplies
+the default instead. Judge reasoning effort remains independent.
+
+Before starting LiteLLM, the launcher invokes `run_job` with the complete run
+arguments and `--prepare-proxy-config PATH`. This preparation mode lets an
+explicit `--reasoning-effort` override the environment, validates the result,
+writes the proxy configuration, and prints the canonical effort to standard
+output. The launcher exports that value as `SHELLBENCH_REASONING_EFFORT` before
+starting either the proxy or the job. Invalid values stop startup before the
+proxy is launched.
+
+For manual launches, use the same preparation step before starting the proxy
+and export its returned value for the runner. An ordinary `run_job` invocation
+rejects conflicting CLI and environment efforts: it cannot safely override a
+proxy that is already running. Without either value, direct harness construction
+retains each harness's defaults; the managed proxy still requires an explicit
+valid effort, as before.

scripts/native_eval/fleet.py

@@ -1479,7 +1479,10 @@ def _matrix_satisfied(self) -> bool:
 
     def _run_spec(self, entry: dict[str, Any]) -> RunSpec:
         try:
-            return RunSpec(**{field: entry[field] for field in RUN_SPEC_FIELDS})
+            return RunSpec(
+                **{field: entry[field] for field in RUN_SPEC_FIELDS},
+                reasoning_effort=entry.get("reasoning_effort"),
+            )
         except KeyError as exc:
             raise FleetError(
                 f"run entry {entry.get('run_label', '<unknown>')} lacks {exc.args[0]}"

scripts/native_eval/harnesses.py

@@ -532,6 +532,7 @@ def _openclaw(
 ) -> HarnessCommand:
     provider = "openai"
     model = f"{provider}/{run.model_id}"
+    thinking = run.reasoning_effort or "off"
     home = "/tmp/shellbench-openclaw"
     audit_plugin_root = f"{home}/.openclaw/shellbench-audit"
     gateway_token = secrets.token_urlsafe(32)
@@ -616,7 +617,8 @@ def _openclaw(
         'kill "$gateway_pid" 2>/dev/null || true; '
         'wait "$gateway_pid" 2>/dev/null || true; '
         'cat "$gateway_log" >&2; exit 70; fi; '
-        "openclaw agent --json --agent main --thinking off "
+        "openclaw agent --json --agent main "
+        f"--thinking {shlex.quote(thinking)} "
         f"--model {shlex.quote(model)} "
         '--message "$(cat /tmp/shellbench-instruction.md)" '
         '>"$log" 2>&1 </dev/null & pid=$!; '
@@ -725,6 +727,9 @@ def _hermes(
 ) -> HarnessCommand:
     home = "/tmp/shellbench-hermes"
     provider_name = "custom:shellbench"
+    agent_config: dict[str, object] = {"max_turns": 90}
+    if run.reasoning_effort:
+        agent_config["reasoning_effort"] = run.reasoning_effort
     config: dict[str, object] = {
         "model": {
             "default": run.model_id,
@@ -741,7 +746,7 @@ def _hermes(
             }
         },
         "toolsets": ["hermes-cli"],
-        "agent": {"max_turns": 90},
+        "agent": agent_config,
         "memory": {"memory_enabled": False, "user_profile_enabled": False},
         "compression": {"enabled": True, "threshold": 0.85},
         "terminal": {"backend": "local", "timeout": 180},
@@ -817,11 +822,17 @@ def _codex(
         f'printf %s {auth_json} > "$CODEX_HOME/auth.json"; '
         f'printf %s {config_text} > "$CODEX_HOME/config.toml"'
     )
+    reasoning_override = (
+        f"-c {shlex.quote(f'model_reasoning_effort={json.dumps(run.reasoning_effort)}')} "
+        if run.reasoning_effort
+        else ""
+    )
     run_command = (
         f"export PATH={_base_path()}; export CODEX_HOME={home}; "
         "codex exec --dangerously-bypass-approvals-and-sandbox "
         "--skip-git-repo-check "
         f"--model {shlex.quote(run.model_id)} "
+        f"{reasoning_override}"
         "--json --enable unified_exec -- "
         '"$(cat /tmp/shellbench-instruction.md)" '
         ">/logs/agent/codex.txt 2>/logs/agent/codex-stderr.txt "
@@ -871,11 +882,16 @@ def _claude_code(
         'mkdir -p "$CLAUDE_CONFIG_DIR/debug" "$CLAUDE_CONFIG_DIR/projects/-app"; '
         f'printf %s {mcp_json} > "$CLAUDE_CONFIG_DIR/.claude.json"'
     )
+    # Claude Code names its top effort level "max"; the other native harnesses
+    # use ShellBench's canonical "xhigh" spelling.
+    claude_effort = "max" if run.reasoning_effort == "xhigh" else run.reasoning_effort
+    effort_option = f"--effort {shlex.quote(claude_effort)} " if claude_effort else ""
     run_command = (
         f"export PATH={_base_path()}; export CLAUDE_CONFIG_DIR={home}; "
         "claude --verbose --output-format=stream-json "
         "--permission-mode=bypassPermissions --print "
         f"--model {shlex.quote(run.model_id)} "
+        f"{effort_option}"
         '"$(cat /tmp/shellbench-instruction.md)" '
         ">/logs/agent/claude-code.txt 2>&1 </dev/null; status=$?; "
         'cat /logs/agent/claude-code.txt; exit "$status"'

scripts/native_eval/models.py

@@ -32,6 +32,7 @@ class RunSpec:
     repetition: int
     expected_task_count: int
     run_date: str
+    reasoning_effort: str | None = None
 
     def to_dict(self) -> dict[str, object]:
         return asdict(self)
@@ -162,6 +163,7 @@ def build_matrix_plan(
                         repetition=repetition,
                         expected_task_count=expected_task_count,
                         run_date=stamp,
+                        reasoning_effort=reasoning_effort,
                     )
                 )
     return plan

scripts/native_eval/remote_run.sh

@@ -61,6 +61,8 @@ cleanup() {
 trap cleanup EXIT
 
 umask 077
+# Fleet's planned value takes precedence over defaults in the provider file.
+PLANNED_REASONING_EFFORT="${SHELLBENCH_REASONING_EFFORT:-}"
 set -a
 # shellcheck disable=SC1090
 source "$ENV_FILE"
@@ -77,37 +79,6 @@ printf '%s\n' "running" > "$RUN_STATE_DIR/state"
 export SHELLBENCH_PROXY_KEY="${SHELLBENCH_PROXY_KEY:-$(openssl rand -hex 32)}"
 
 cd "$ROOT/runner"
-python3 - <<PY
-from pathlib import Path
-from scripts.native_eval.proxy import write_proxy_config
-write_proxy_config(Path(${PROXY_CONFIG@Q}))
-PY
-
-"$TOOLCHAIN_ROOT/litellm-venv/bin/litellm" \
-  --config "$PROXY_CONFIG" \
-  --host 0.0.0.0 \
-  --port 4000 \
-  >"$PROXY_LOG" 2>&1 &
-PROXY_PID="$!"
-
-for _ in $(seq 1 120); do
-  if curl -fsS \
-    -H "Authorization: Bearer $SHELLBENCH_PROXY_KEY" \
-    http://127.0.0.1:4000/health/liveliness >/dev/null 2>&1; then
-    break
-  fi
-  if ! kill -0 "$PROXY_PID" 2>/dev/null; then
-    echo "LiteLLM proxy exited during startup" >&2
-    tail -100 "$PROXY_LOG" >&2 || true
-    exit 1
-  fi
-  sleep 1
-done
-
-curl -fsS \
-  -H "Authorization: Bearer $SHELLBENCH_PROXY_KEY" \
-  http://127.0.0.1:4000/health/liveliness >/dev/null
-
 RUN_COMMAND=(
   python3 -m scripts.native_eval.run_job
   --tasks-root "$TASKS_ROOT"
@@ -128,13 +99,45 @@ RUN_COMMAND=(
   --proxy-url "http://host.docker.internal:4000"
   --concurrency "$CONCURRENCY"
 )
+if [[ -n "$PLANNED_REASONING_EFFORT" ]]; then
+  RUN_COMMAND+=(--reasoning-effort "$PLANNED_REASONING_EFFORT")
+fi
 if [[ -n "$RERUN_OF_CANONICAL_RUN" ]]; then
   RUN_COMMAND+=(--rerun-of-canonical-run "$RERUN_OF_CANONICAL_RUN")
 fi
 for task_name in "${TASK_NAMES[@]}"; do
   RUN_COMMAND+=(--task "$task_name")
 done
 
+# Resolve once before proxy startup; the runner must see this same value.
+SHELLBENCH_REASONING_EFFORT=$("${RUN_COMMAND[@]}" --prepare-proxy-config "$PROXY_CONFIG")
+export SHELLBENCH_REASONING_EFFORT
+
+"$TOOLCHAIN_ROOT/litellm-venv/bin/litellm" \
+  --config "$PROXY_CONFIG" \
+  --host 0.0.0.0 \
+  --port 4000 \
+  >"$PROXY_LOG" 2>&1 &
+PROXY_PID="$!"
+
+for _ in $(seq 1 120); do
+  if curl -fsS \
+    -H "Authorization: Bearer $SHELLBENCH_PROXY_KEY" \
+    http://127.0.0.1:4000/health/liveliness >/dev/null 2>&1; then
+    break
+  fi
+  if ! kill -0 "$PROXY_PID" 2>/dev/null; then
+    echo "LiteLLM proxy exited during startup" >&2
+    tail -100 "$PROXY_LOG" >&2 || true
+    exit 1
+  fi
+  sleep 1
+done
+
+curl -fsS \
+  -H "Authorization: Bearer $SHELLBENCH_PROXY_KEY" \
+  http://127.0.0.1:4000/health/liveliness >/dev/null
+
 set +e
 "${RUN_COMMAND[@]}"
 RUN_STATUS="$?"

scripts/native_eval/run_job.py

@@ -17,6 +17,7 @@
     model_by_slug,
     trajectory_mode_for_harness,
 )
+from scripts.native_eval.proxy import REASONING_EFFORTS, write_proxy_config
 from scripts.native_eval.runtime import atomic_write_json, run_trial, utc_now
 from scripts.native_eval.tasks import TaskSpec, validate_suite
 
@@ -298,7 +299,7 @@ def _run_manifest(
             "SHELLBENCH_HARBOR_REFERENCE_COMMIT"
         ),
         "judge_model_id": os.environ.get("SHELLBENCH_JUDGE_MODEL_ID"),
-        "reasoning_effort": os.environ.get("SHELLBENCH_REASONING_EFFORT"),
+        "reasoning_effort": run.reasoning_effort,
         "judge_reasoning_effort": os.environ.get(
             "SHELLBENCH_JUDGE_REASONING_EFFORT"
         ),
@@ -391,6 +392,16 @@ def _runner_patch_hash() -> str:
 def build_run_spec(args: argparse.Namespace) -> RunSpec:
     harness = harness_by_name(args.harness)
     model = model_by_slug(args.model_slug)
+    cli_effort = getattr(args, "reasoning_effort", None)
+    environment_effort = os.environ.get("SHELLBENCH_REASONING_EFFORT", "").strip()
+    if cli_effort and environment_effort and cli_effort != environment_effort:
+        raise ValueError(
+            "--reasoning-effort conflicts with SHELLBENCH_REASONING_EFFORT; "
+            "resolve it with --prepare-proxy-config before starting the proxy"
+        )
+    reasoning_effort = cli_effort or environment_effort or None
+    if reasoning_effort is not None and reasoning_effort not in REASONING_EFFORTS:
+        raise ValueError("reasoning effort must be low, medium, high, or xhigh")
     return RunSpec(
         run_label=args.run_label,
         harness=harness.name,
@@ -402,6 +413,7 @@ def build_run_spec(args: argparse.Namespace) -> RunSpec:
         repetition=args.repetition,
         expected_task_count=args.expected_task_count,
         run_date=args.run_date,
+        reasoning_effort=reasoning_effort,
     )
 
 
@@ -428,6 +440,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser.add_argument("--public-tasks-commit", required=True)
     parser.add_argument("--task-suite-path", required=True)
     parser.add_argument("--run-date", required=True)
+    parser.add_argument(
+        "--reasoning-effort",
+        choices=("low", "medium", "high", "xhigh"),
+    )
+    parser.add_argument(
+        "--prepare-proxy-config",
+        type=Path,
+        help="Resolve effort, write proxy config, print the effort, and exit before running tasks.",
+    )
     parser.add_argument(
         "--toolchain-root",
         type=Path,
@@ -450,12 +471,20 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
 
 def main() -> None:
     args = parse_args()
+    # Overrides are safe only while preparing a proxy that has not started yet.
+    if args.prepare_proxy_config and args.reasoning_effort:
+        os.environ["SHELLBENCH_REASONING_EFFORT"] = args.reasoning_effort
+    run = build_run_spec(args)
+    if args.prepare_proxy_config:
+        write_proxy_config(args.prepare_proxy_config)
+        print(run.reasoning_effort)
+        return
     proxy_key = os.environ.get("SHELLBENCH_PROXY_KEY", "")
     state = asyncio.run(
         run_job(
             tasks_root=args.tasks_root,
             jobs_dir=args.jobs_dir,
-            run=build_run_spec(args),
+            run=run,
             public_tasks_commit=args.public_tasks_commit,
             task_suite_path=args.task_suite_path,
             toolchain_root=args.toolchain_root,