From 8a202456e24d6ed921206c3a6b1793d10fb12b2f Mon Sep 17 00:00:00 2001 From: akushonkamen Date: Sat, 10 Oct 2026 16:52:04 +0800 Subject: [PATCH 1/2] Add --evaluator-args flag forwarding kwargs to evaluate() via signature sniffing - New evaluator.evaluator_args config field (Optional[Dict[str, Any]]), settable from YAML or the new --evaluator-args '{"weight": 2.0}' CLI flag (invalid JSON or a non-object value exits 1 with an actionable message) - Evaluator inspects evaluate() with inspect.signature: configured kwargs are forwarded via functools.partial when the function accepts them (named parameters or **kwargs); unsupported keys - or a kwarg colliding with the positional program path - fail at startup with an error naming the offending keys and the actual signature instead of failing on every evaluation; single-argument evaluators without evaluator_args keep their existing behavior unchanged - Log kwarg key names only, never values (they may contain secrets) - Document the flag in README, replacing the env-var workaround for per-run evaluator settings - Add tests/test_evaluator_args.py: kwargs forwarding (named + **kwargs), backward compatibility, startup validation, YAML load, worker config dict round-trip, CLI JSON parsing and invalid-JSON exit code Cascade evaluate_stage1/2/3 functions are intentionally out of scope; issue #474 only covers evaluate(). Fixes #474 --- README.md | 35 +++++ openevolve/cli.py | 48 ++++++- openevolve/config.py | 5 + openevolve/evaluator.py | 102 +++++++++++++- tests/test_evaluator_args.py | 258 +++++++++++++++++++++++++++++++++++ 5 files changed, 443 insertions(+), 5 deletions(-) create mode 100644 tests/test_evaluator_args.py diff --git a/README.md b/README.md index 7c0d246a51..575303712a 100644 --- a/README.md +++ b/README.md @@ -575,6 +575,8 @@ evaluator: enable_artifacts: true # Error feedback to LLM cascade_evaluation: true # Multi-stage testing use_llm_feedback: true # AI code quality assessment + evaluator_args: # Keyword arguments forwarded to evaluate() + weight: 2.0 prompt: # Sophisticated inspiration system @@ -613,6 +615,39 @@ database: +
+📞 Passing Arguments to Your Evaluator + +If your `evaluate()` function takes extra keyword arguments beyond the program +path, pass them with the `--evaluator-args` CLI flag (a JSON object): + +```bash +openevolve-run initial_program.py evaluator.py \ + --evaluator-args '{"weight": 2.0, "label": "experiment-1"}' +``` + +or configure them in YAML: + +```yaml +evaluator: + evaluator_args: + weight: 2.0 + label: "experiment-1" +``` + +OpenEvolve inspects the signature of your `evaluate()` function: arguments are +forwarded only when the function accepts them (named parameters or `**kwargs`), +and a startup error lists the offending keys and the actual signature if it +does not — so typos fail fast instead of silently doing nothing. Evaluators +with a single `program_path` argument are unaffected. + +This replaces the earlier workaround of smuggling per-run settings to the +evaluator through environment variables. (Note: custom top-level keys in +`config.yaml` are dropped silently by the config parser — use +`evaluator.evaluator_args` or the CLI flag instead.) + +
+
🎨 Custom Prompt Templates diff --git a/openevolve/cli.py b/openevolve/cli.py index d1da28137c..d3eb61d3d7 100644 --- a/openevolve/cli.py +++ b/openevolve/cli.py @@ -4,10 +4,11 @@ import argparse import asyncio +import json import logging import os import sys -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from openevolve import OpenEvolve from openevolve.config import Config, load_config @@ -53,6 +54,15 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--api-base", help="Base URL for the LLM API", default=None) + parser.add_argument( + "--evaluator-args", + help=( + "JSON object of keyword arguments forwarded to the 'evaluate' function of the " + "evaluation file, e.g. '{\"weight\": 2.0}' (the function must accept them)" + ), + default=None, + ) + parser.add_argument("--primary-model", help="Primary LLM model name", default=None) parser.add_argument("--secondary-model", help="Secondary LLM model name", default=None) @@ -60,6 +70,32 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() +def parse_evaluator_args(raw: str) -> Dict[str, Any]: + """ + Parse the --evaluator-args JSON string into a kwargs dict + + Args: + raw: JSON object literal of keyword arguments + + Returns: + Dict of keyword arguments for the evaluation function + + Raises: + ValueError: If the string is not valid JSON or not a JSON object + """ + try: + parsed = json.loads(raw) + except json.JSONDecodeError as e: + raise ValueError(f"--evaluator-args is not valid JSON: {e}") from e + + if not isinstance(parsed, dict): + type_name = type(parsed).__name__ + raise ValueError( + f"--evaluator-args must be a JSON object of keyword arguments, got {type_name}" + ) + return parsed + + async def main_async() -> int: """ Main asynchronous entry point @@ -81,6 +117,16 @@ async def main_async() -> int: # Load base config from file or defaults config = load_config(args.config) + # Apply evaluator args CLI override (kwargs forwarded to evaluate(), issue #474) + if args.evaluator_args is not None: + try: + config.evaluator.evaluator_args = parse_evaluator_args(args.evaluator_args) + except ValueError as e: + print(f"Error: {str(e)}") + return 1 + # Print keys only - values may contain secrets and end up in logs + print(f"Using evaluator args keys: {sorted(config.evaluator.evaluator_args.keys())}") + # Create config object with command-line overrides if args.api_base or args.primary_model or args.secondary_model: # Apply command-line overrides diff --git a/openevolve/config.py b/openevolve/config.py index d68b701f84..2844e7f779 100644 --- a/openevolve/config.py +++ b/openevolve/config.py @@ -409,6 +409,11 @@ class EvaluatorConfig: enable_artifacts: bool = True max_artifact_storage: int = 100 * 1024 * 1024 # 100MB per program + # Extra keyword arguments forwarded to the user's evaluate() function + # (only applied when its signature accepts them; see openevolve.evaluator). + # Settable via YAML ('evaluator.evaluator_args') or the --evaluator-args CLI flag. + evaluator_args: Optional[Dict[str, Any]] = None + @dataclass class EvolutionTraceConfig: diff --git a/openevolve/evaluator.py b/openevolve/evaluator.py index 30cb3bda70..74c49af49e 100644 --- a/openevolve/evaluator.py +++ b/openevolve/evaluator.py @@ -3,7 +3,9 @@ """ import asyncio +import functools import importlib.util +import inspect import json import logging import os @@ -16,15 +18,13 @@ from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Tuple, Union -import traceback from openevolve.config import EvaluatorConfig from openevolve.database import ProgramDatabase from openevolve.evaluation_result import EvaluationResult -from openevolve.database import ProgramDatabase from openevolve.llm.ensemble import LLMEnsemble -from openevolve.utils.async_utils import TaskPool, run_in_executor from openevolve.prompt.sampler import PromptSampler +from openevolve.utils.async_utils import TaskPool, run_in_executor from openevolve.utils.format_utils import format_metrics_safe logger = logging.getLogger(__name__) @@ -54,6 +54,11 @@ def __init__( self.prompt_sampler = prompt_sampler self.database = database + # Keyword arguments forwarded to the user's evaluate() function; populated + # by _load_evaluation_function() based on config.evaluator.evaluator_args + # and the signature of the loaded function + self._evaluator_kwargs: Dict[str, Any] = {} + # Create a task pool for parallel evaluation self.task_pool = TaskPool(max_concurrency=config.parallel_evaluations) @@ -98,6 +103,7 @@ def _load_evaluation_function(self) -> None: ) self.evaluate_function = module.evaluate + self._evaluator_kwargs = self._resolve_evaluator_kwargs(module.evaluate) logger.info(f"Successfully loaded evaluation function from {self.evaluation_file}") # Validate cascade configuration @@ -106,6 +112,90 @@ def _load_evaluation_function(self) -> None: logger.error(f"Error loading evaluation function: {str(e)}") raise + def _resolve_evaluator_kwargs(self, evaluate_function: Callable) -> Dict[str, Any]: + """ + Resolve the keyword arguments to forward to the user's evaluate() function. + + Uses inspect.signature to verify that evaluate() actually accepts the + configured evaluator_args (GitHub issue #474): + + - Evaluators accepting **kwargs get all configured kwargs. + - Evaluators whose named parameters cover the configured keys get them + forwarded on every call. + - Evaluators that cannot accept a configured key fail here, at startup, + with an actionable error instead of failing on every evaluation. + + Evaluators with no configured evaluator_args (the common single + `program_path` argument case) are unaffected and keep their existing + behavior. + + Args: + evaluate_function: The user-provided evaluate() function + + Returns: + Dict of keyword arguments to forward (empty when nothing configured) + + Raises: + ValueError: If evaluate() cannot accept a configured evaluator_arg + """ + evaluator_args = self.config.evaluator_args or {} + if not evaluator_args: + return {} + + function_name = getattr(evaluate_function, "__name__", "evaluate") + try: + signature = inspect.signature(evaluate_function) + except (TypeError, ValueError): + # Some callables (e.g. certain builtins) expose no signature; forward + # as-is and let the call surface any mismatch + logger.warning( + f"Could not inspect the signature of '{function_name}'; forwarding " + f"evaluator_args keys: {sorted(evaluator_args.keys())}" + ) + return dict(evaluator_args) + + if any( + param.kind == inspect.Parameter.VAR_KEYWORD for param in signature.parameters.values() + ): + # **kwargs swallows everything + logger.info( + f"Forwarding evaluator_args keys to '{function_name}': " + f"{sorted(evaluator_args.keys())}" + ) + return dict(evaluator_args) + + named_params = { + name + for name, param in signature.parameters.items() + if param.kind + in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY) + } + unsupported = sorted(set(evaluator_args.keys()) - named_params) + if unsupported: + raise ValueError( + f"Evaluation function '{function_name}' in {self.evaluation_file} does not " + f"accept the configured evaluator_args keys: {unsupported}. Its signature is " + f"'{function_name}{signature}'. Add the parameters to evaluate(), accept " + f"**kwargs, or remove the keys from evaluator.evaluator_args / --evaluator-args." + ) + + # Guard against kwargs colliding with the program path passed positionally + # (e.g. evaluator_args={"program_path": ...} for evaluate(program_path)) + try: + signature.bind(object(), **evaluator_args) + except TypeError as e: + raise ValueError( + f"Configured evaluator_args {sorted(evaluator_args.keys())} cannot be " + f"forwarded to evaluation function '{function_name}' in " + f"{self.evaluation_file}: {e}. Its signature is '{function_name}{signature}'." + ) from e + + logger.info( + f"Forwarding evaluator_args keys to '{function_name}': " + f"{sorted(evaluator_args.keys())}" + ) + return dict(evaluator_args) + def _validate_cascade_configuration(self, module) -> None: """ Validate cascade evaluation configuration and warn about potential issues @@ -356,7 +446,11 @@ async def _direct_evaluate( # Create a coroutine that runs the evaluation function in an executor async def run_evaluation(): loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self.evaluate_function, program_path) + return await loop.run_in_executor( + self._executor, + functools.partial(self.evaluate_function, **self._evaluator_kwargs), + program_path, + ) # Run the evaluation with timeout - let exceptions bubble up for retry handling result = await asyncio.wait_for(run_evaluation(), timeout=self.config.timeout) diff --git a/tests/test_evaluator_args.py b/tests/test_evaluator_args.py new file mode 100644 index 0000000000..2d540a9ee2 --- /dev/null +++ b/tests/test_evaluator_args.py @@ -0,0 +1,258 @@ +""" +Tests for forwarding evaluator_args kwargs to the evaluation function +(GitHub issue #474: --evaluator-args CLI flag + config.evaluator.evaluator_args) +""" + +import asyncio +import os +import subprocess +import sys +import tempfile +import unittest + +from openevolve.cli import parse_evaluator_args +from openevolve.config import Config, EvaluatorConfig +from openevolve.evaluator import Evaluator + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +class EvaluatorArgsTestBase(unittest.TestCase): + """Shared helpers for writing temporary evaluation files""" + + def setUp(self): + self._temp_files = [] + + def tearDown(self): + for path in self._temp_files: + if os.path.exists(path): + os.unlink(path) + + def _write_eval_file(self, source: str) -> str: + temp_file = tempfile.NamedTemporaryFile( + mode="w", suffix=".py", delete=False, encoding="utf-8" + ) + temp_file.write(source) + temp_file.close() + self._temp_files.append(temp_file.name) + return temp_file.name + + def _create_evaluator(self, eval_file: str, evaluator_args=None) -> Evaluator: + config = EvaluatorConfig() + config.timeout = 10 + config.max_retries = 1 + config.cascade_evaluation = False + config.evaluator_args = evaluator_args + return Evaluator( + config=config, + evaluation_file=eval_file, + llm_ensemble=None, + prompt_sampler=None, + ) + + +class TestEvaluatorKwargsForwarding(EvaluatorArgsTestBase): + """Test that configured kwargs reach the evaluation function""" + + def test_multi_param_evaluator_receives_kwargs(self): + """evaluate(program_path, weight=...) receives the configured kwarg""" + eval_file = self._write_eval_file(""" +def evaluate(program_path, weight=1.0): + return {"weighted_score": weight} +""") + evaluator = self._create_evaluator(eval_file, evaluator_args={"weight": 2.5}) + + async def run_test(): + return await evaluator.evaluate_program("x = 1", "test_weighted") + + result = asyncio.run(run_test()) + self.assertEqual(result["weighted_score"], 2.5) + + def test_var_keyword_evaluator_receives_kwargs(self): + """evaluate(program_path, **kwargs) receives all configured kwargs""" + eval_file = self._write_eval_file(""" +def evaluate(program_path, **kwargs): + return {"score": 1.0 if kwargs.get("factor") == 3.0 and kwargs.get("shift") == -1 else 0.0} +""") + evaluator = self._create_evaluator(eval_file, evaluator_args={"factor": 3.0, "shift": -1}) + + async def run_test(): + return await evaluator.evaluate_program("x = 1", "test_var_keyword") + + result = asyncio.run(run_test()) + self.assertEqual(result["score"], 1.0) + + def test_single_param_evaluator_unchanged_without_args(self): + """No evaluator_args configured: single-argument evaluators keep working""" + eval_file = self._write_eval_file(""" +def evaluate(program_path): + return {"score": 0.5} +""") + evaluator = self._create_evaluator(eval_file) + self.assertEqual(evaluator._evaluator_kwargs, {}) + + async def run_test(): + return await evaluator.evaluate_program("x = 1", "test_plain") + + result = asyncio.run(run_test()) + self.assertEqual(result["score"], 0.5) + + def test_configured_empty_args_unchanged(self): + """Explicitly empty evaluator_args dict behaves like no args""" + eval_file = self._write_eval_file(""" +def evaluate(program_path): + return {"score": 0.7} +""") + evaluator = self._create_evaluator(eval_file, evaluator_args={}) + self.assertEqual(evaluator._evaluator_kwargs, {}) + + async def run_test(): + return await evaluator.evaluate_program("x = 1", "test_empty") + + result = asyncio.run(run_test()) + self.assertEqual(result["score"], 0.7) + + +class TestEvaluatorKwargsValidation(EvaluatorArgsTestBase): + """Test startup validation of evaluator_args against the evaluate() signature""" + + def test_unsupported_key_raises_at_startup(self): + """Configured key not accepted by evaluate() fails at load time with the key named""" + eval_file = self._write_eval_file(""" +def evaluate(program_path): + return {"score": 1.0} +""") + with self.assertRaises(ValueError) as ctx: + self._create_evaluator(eval_file, evaluator_args={"weight": 1.0}) + + message = str(ctx.exception) + self.assertIn("weight", message) + self.assertIn("evaluate", message) + self.assertIn(eval_file, message) + + def test_program_path_collision_raises(self): + """A kwarg named like the positional program path parameter is rejected""" + eval_file = self._write_eval_file(""" +def evaluate(program_path): + return {"score": 1.0} +""") + with self.assertRaises(ValueError) as ctx: + self._create_evaluator(eval_file, evaluator_args={"program_path": "other.py"}) + + self.assertIn("program_path", str(ctx.exception)) + + def test_error_lists_actual_signature(self): + """The actionable error includes the evaluator's actual signature""" + eval_file = self._write_eval_file(""" +def evaluate(program_path): + return {"score": 1.0} +""") + with self.assertRaises(ValueError) as ctx: + self._create_evaluator(eval_file, evaluator_args={"a": 1, "b": 2}) + + message = str(ctx.exception) + self.assertIn("evaluate(program_path)", message) + self.assertIn("'a'", message) + self.assertIn("'b'", message) + + +class TestEvaluatorArgsConfig(unittest.TestCase): + """Test config plumbing (YAML load + worker dict round-trip)""" + + def test_yaml_load_evaluator_args(self): + """evaluator.evaluator_args loads from YAML via dacite""" + import tempfile + + config_content = """ +evaluator: + timeout: 60 + evaluator_args: + weight: 2.0 + label: "test" +""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write(config_content) + config_path = f.name + try: + from openevolve.config import load_config + + config = load_config(config_path) + self.assertEqual(config.evaluator.evaluator_args, {"weight": 2.0, "label": "test"}) + finally: + os.unlink(config_path) + + def test_worker_round_trip_preserves_evaluator_args(self): + """to_dict() -> EvaluatorConfig(**d['evaluator']) keeps the field (process pool path)""" + config = Config(language="python") + config.evaluator.evaluator_args = {"weight": 3.5, "mode": "fast"} + + config_dict = config.to_dict() + reconstructed = EvaluatorConfig(**config_dict["evaluator"]) + + self.assertEqual(reconstructed.evaluator_args, {"weight": 3.5, "mode": "fast"}) + + def test_default_is_none(self): + """Default config has no evaluator_args (backward compatible)""" + self.assertIsNone(EvaluatorConfig().evaluator_args) + + +class TestParseEvaluatorArgs(unittest.TestCase): + """Test the CLI --evaluator-args JSON parsing helper""" + + def test_valid_json_object(self): + self.assertEqual( + parse_evaluator_args('{"weight": 2.0, "label": "x"}'), {"weight": 2.0, "label": "x"} + ) + + def test_empty_object(self): + self.assertEqual(parse_evaluator_args("{}"), {}) + + def test_invalid_json_raises(self): + with self.assertRaises(ValueError) as ctx: + parse_evaluator_args("{'weight': 2.0}") # single quotes are not JSON + self.assertIn("not valid JSON", str(ctx.exception)) + + def test_non_object_json_raises(self): + with self.assertRaises(ValueError) as ctx: + parse_evaluator_args("[1, 2, 3]") + self.assertIn("JSON object", str(ctx.exception)) + + +class TestCLIEvaluatorArgsExitCode(unittest.TestCase): + """End-to-end CLI check: invalid --evaluator-args JSON exits with code 1""" + + def test_invalid_json_exits_1(self): + program_file = tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) + program_file.write("x = 1\n") + program_file.close() + eval_file = tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) + eval_file.write("def evaluate(program_path):\n return {'score': 1.0}\n") + eval_file.close() + try: + env = dict(os.environ) + env.setdefault("OPENAI_API_KEY", "test-key-for-unit-tests") + result = subprocess.run( + [ + sys.executable, + "-m", + "openevolve.cli", + program_file.name, + eval_file.name, + "--evaluator-args", + "{invalid json", + ], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=60, + ) + self.assertEqual(result.returncode, 1) + self.assertIn("not valid JSON", result.stdout + result.stderr) + finally: + os.unlink(program_file.name) + os.unlink(eval_file.name) + + +if __name__ == "__main__": + unittest.main() From dac0e2187632404d7c0517c4bd802d934b6c95d1 Mon Sep 17 00:00:00 2001 From: akushonkamen Date: Sat, 10 Oct 2026 17:24:09 +0800 Subject: [PATCH 2/2] fix(evaluator): reject evaluator_args shadowing positional params of **kwargs evaluators at startup Move the signature.bind guard ahead of the VAR_KEYWORD shortcut so a configured key colliding with a positional parameter (e.g. evaluator_args={"program_path": ...} for evaluate(program_path, **kwargs)) fails at load time with an actionable message instead of raising TypeError on every evaluation. The bind check mirrors the actual call functools.partial(evaluate, **args)(program_path), so it can only reject configurations that would fail at call time anyway. Adds two tests: collision rejection for **kwargs evaluators and continued forwarding of non-colliding keys. --- openevolve/evaluator.py | 26 +++++++++++++++----------- tests/test_evaluator_args.py | 27 +++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/openevolve/evaluator.py b/openevolve/evaluator.py index 74c49af49e..5c91c39f4a 100644 --- a/openevolve/evaluator.py +++ b/openevolve/evaluator.py @@ -154,6 +154,21 @@ def _resolve_evaluator_kwargs(self, evaluate_function: Callable) -> Dict[str, An ) return dict(evaluator_args) + # Guard against kwargs that cannot be forwarded together with the program + # path passed positionally (e.g. evaluator_args={"program_path": ...} for + # evaluate(program_path)). Runs before the **kwargs shortcut below: binding + # mirrors the actual call functools.partial(evaluate, **args)(program_path), + # so it also rejects a configured key shadowing a positional parameter of a + # **kwargs evaluator (TypeError: multiple values for argument). + try: + signature.bind(object(), **evaluator_args) + except TypeError as e: + raise ValueError( + f"Configured evaluator_args {sorted(evaluator_args.keys())} cannot be " + f"forwarded to evaluation function '{function_name}' in " + f"{self.evaluation_file}: {e}. Its signature is '{function_name}{signature}'." + ) from e + if any( param.kind == inspect.Parameter.VAR_KEYWORD for param in signature.parameters.values() ): @@ -179,17 +194,6 @@ def _resolve_evaluator_kwargs(self, evaluate_function: Callable) -> Dict[str, An f"**kwargs, or remove the keys from evaluator.evaluator_args / --evaluator-args." ) - # Guard against kwargs colliding with the program path passed positionally - # (e.g. evaluator_args={"program_path": ...} for evaluate(program_path)) - try: - signature.bind(object(), **evaluator_args) - except TypeError as e: - raise ValueError( - f"Configured evaluator_args {sorted(evaluator_args.keys())} cannot be " - f"forwarded to evaluation function '{function_name}' in " - f"{self.evaluation_file}: {e}. Its signature is '{function_name}{signature}'." - ) from e - logger.info( f"Forwarding evaluator_args keys to '{function_name}': " f"{sorted(evaluator_args.keys())}" diff --git a/tests/test_evaluator_args.py b/tests/test_evaluator_args.py index 2d540a9ee2..067f9e7e9e 100644 --- a/tests/test_evaluator_args.py +++ b/tests/test_evaluator_args.py @@ -141,6 +141,33 @@ def evaluate(program_path): self.assertIn("program_path", str(ctx.exception)) + def test_program_path_collision_raises_for_var_keyword_evaluator(self): + """A configured key shadowing the positional param of a **kwargs evaluator is rejected""" + eval_file = self._write_eval_file(""" +def evaluate(program_path, **kwargs): + return {"score": kwargs.get("weight", 1.0)} +""") + with self.assertRaises(ValueError) as ctx: + self._create_evaluator(eval_file, evaluator_args={"program_path": "other.py"}) + + message = str(ctx.exception) + self.assertIn("program_path", message) + self.assertIn("multiple values", message) + + def test_var_keyword_evaluator_still_accepts_non_colliding_kwargs(self): + """**kwargs evaluators keep receiving configured keys that do not shadow parameters""" + eval_file = self._write_eval_file(""" +def evaluate(program_path, **kwargs): + return {"score": kwargs.get("weight", 1.0)} +""") + evaluator = self._create_evaluator(eval_file, evaluator_args={"weight": 0.25}) + + async def run_test(): + return await evaluator.evaluate_program("x = 1", "test_vk_no_collision") + + result = asyncio.run(run_test()) + self.assertEqual(result["score"], 0.25) + def test_error_lists_actual_signature(self): """The actionable error includes the evaluator's actual signature""" eval_file = self._write_eval_file("""