From 86705c2649a8abd1cda46f7d7a688ea5d35448d5 Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Sat, 12 Sep 2026 19:57:02 -0700 Subject: [PATCH] test(llm-judge): the shared test base runs only a local stub judge Tests on the base can no longer fall through to codex/claude/cursor, and enqueue is pinned to the temporary state folder. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013smxPCr4XeE7R77zVtN1u4 Change-Id: I7b2356d2fda21cbe253900540139827fcc4d8827 --- engine/hooks/llm-judge/judge_test_base.py | 9 +++++++-- engine/hooks/llm-judge/tests/test_judge.py | 21 ++++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/engine/hooks/llm-judge/judge_test_base.py b/engine/hooks/llm-judge/judge_test_base.py index 01249d18..9bb41eb8 100644 --- a/engine/hooks/llm-judge/judge_test_base.py +++ b/engine/hooks/llm-judge/judge_test_base.py @@ -2,6 +2,7 @@ import json import os +import sys import tempfile import unittest from unittest.mock import patch @@ -13,10 +14,14 @@ class JudgeTestCase(unittest.TestCase): def setUp(self): super().setUp() self.state = tempfile.TemporaryDirectory() - self.judge_env = patch.dict(os.environ, {judge.STATE_ENV: self.state.name}) + self.judge_env = patch.dict(os.environ, { + judge.STATE_ENV: self.state.name, + judge.RUNNERS_ENV: json.dumps([ + ["stub", [sys.executable, "-c", "print('{\"match\": false}')", judge.PROMPT_SLOT]], + ]), + }) self.judge_env.start() os.environ.pop(judge.CHILD_ENV, None) - os.environ.pop(judge.RUNNERS_ENV, None) def tearDown(self): self.judge_env.stop() diff --git a/engine/hooks/llm-judge/tests/test_judge.py b/engine/hooks/llm-judge/tests/test_judge.py index c186e8ee..526598ba 100644 --- a/engine/hooks/llm-judge/tests/test_judge.py +++ b/engine/hooks/llm-judge/tests/test_judge.py @@ -6,6 +6,7 @@ import json import os import sys +import tempfile import time import unittest import warnings @@ -107,8 +108,14 @@ def test_malformed_runners_env_refuses_instead_of_running_defaults(self): with self.assertRaises(ValueError): judge.ask("x") + def test_test_base_runs_only_the_local_stub(self): + self.assertEqual([name for name, _ in judge.runners()], ["stub"]) + self.assertEqual(judge.ask("x")["answer"], {"match": False}) + def test_default_runner_order_is_codex_then_claude_then_cursor(self): - self.assertEqual([name for name, _ in judge.runners()], ["codex", "claude", "cursor"]) + with patch.dict(os.environ): + os.environ.pop(judge.RUNNERS_ENV) + self.assertEqual([name for name, _ in judge.runners()], ["codex", "claude", "cursor"]) class TestVerdict(JudgeBehaviorTestCase): @@ -135,6 +142,18 @@ def test_unchecked_when_ask_was_unchecked(self): class TestBackground(JudgeBehaviorTestCase): + def test_enqueue_writes_only_to_temporary_state_directory(self): + with tempfile.TemporaryDirectory() as home: + with patch.dict(os.environ, {"HOME": home}): + with patch.dict(os.environ): + os.environ.pop(judge.STATE_ENV) + default_state = judge.state_root() + os.makedirs(default_state) + with patch.object(judge.subprocess, "Popen"): + self.assertEqual(judge.enqueue(self.job(id="isolated-job")), "isolated-job") + self.assertTrue(os.path.isfile(os.path.join(self.state.name, "jobs", "isolated-job.json"))) + self.assertEqual(os.listdir(default_state), []) + def test_enqueue_as_judge_child_returns_none_and_starts_nothing(self): os.environ[judge.CHILD_ENV] = "1" self.use_runners(ANSWER_MATCH)