From d6c5cfb54acd5916d8b91232531d756a438bfa7b Mon Sep 17 00:00:00 2001
From: Edbert Chan <edbert.chan@yahoo.com>
Date: Sat, 12 Sep 2026 19:58:09 -0700
Subject: [PATCH] test(runner): the full test run isolates the judge's state
 and runners

run_all_tests.sh exports a temporary judge state folder and a local stub
runner for every suite, so a test that skips the shared base still cannot
reach a model or the user's judge queue.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013smxPCr4XeE7R77zVtN1u4
Change-Id: I8429f17d7c0c75cb8b2217e81e689a6c501206f9
---
 scripts/run_all_tests.sh | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/scripts/run_all_tests.sh b/scripts/run_all_tests.sh
index be5ea2b8..c5d39457 100755
--- a/scripts/run_all_tests.sh
+++ b/scripts/run_all_tests.sh
@@ -7,6 +7,17 @@ set -euo pipefail
 REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 cd "$REPO_DIR"
 
+JUDGE_STATE_DIR="$(python3 -c 'import tempfile; print(tempfile.mkdtemp(prefix="catstack-llm-judge-tests-"))')"
+trap 'python3 -c '\''import os, shutil; shutil.rmtree(os.environ["CATSTACK_LLM_JUDGE_STATE_DIR"])'\''' EXIT
+export CATSTACK_LLM_JUDGE_STATE_DIR="$JUDGE_STATE_DIR"
+export CATSTACK_LLM_JUDGE_RUNNERS="$(python3 - <<'PY'
+import json
+import sys
+
+print(json.dumps([["stub", [sys.executable, "-c", 'print(\'{"match": false}\')', "{prompt}"]]]))
+PY
+)"
+
 missing_node_deps() {
   python3 - <<'PY'
 import json