diff --git a/README.md b/README.md
index 7c0d246a5..575303712 100644
--- a/README.md
+++ b/README.md
@@ -575,6 +575,8 @@ evaluator:
enable_artifacts: true # Error feedback to LLM
cascade_evaluation: true # Multi-stage testing
use_llm_feedback: true # AI code quality assessment
+ evaluator_args: # Keyword arguments forwarded to evaluate()
+ weight: 2.0
prompt:
# Sophisticated inspiration system
@@ -613,6 +615,39 @@ database:
+
+📞 Passing Arguments to Your Evaluator
+
+If your `evaluate()` function takes extra keyword arguments beyond the program
+path, pass them with the `--evaluator-args` CLI flag (a JSON object):
+
+```bash
+openevolve-run initial_program.py evaluator.py \
+ --evaluator-args '{"weight": 2.0, "label": "experiment-1"}'
+```
+
+or configure them in YAML:
+
+```yaml
+evaluator:
+ evaluator_args:
+ weight: 2.0
+ label: "experiment-1"
+```
+
+OpenEvolve inspects the signature of your `evaluate()` function: arguments are
+forwarded only when the function accepts them (named parameters or `**kwargs`),
+and a startup error lists the offending keys and the actual signature if it
+does not — so typos fail fast instead of silently doing nothing. Evaluators
+with a single `program_path` argument are unaffected.
+
+This replaces the earlier workaround of smuggling per-run settings to the
+evaluator through environment variables. (Note: custom top-level keys in
+`config.yaml` are dropped silently by the config parser — use
+`evaluator.evaluator_args` or the CLI flag instead.)
+
+
+
🎨 Custom Prompt Templates
diff --git a/openevolve/cli.py b/openevolve/cli.py
index d1da28137..d3eb61d3d 100644
--- a/openevolve/cli.py
+++ b/openevolve/cli.py
@@ -4,10 +4,11 @@
import argparse
import asyncio
+import json
import logging
import os
import sys
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
from openevolve import OpenEvolve
from openevolve.config import Config, load_config
@@ -53,6 +54,15 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--api-base", help="Base URL for the LLM API", default=None)
+ parser.add_argument(
+ "--evaluator-args",
+ help=(
+ "JSON object of keyword arguments forwarded to the 'evaluate' function of the "
+ "evaluation file, e.g. '{\"weight\": 2.0}' (the function must accept them)"
+ ),
+ default=None,
+ )
+
parser.add_argument("--primary-model", help="Primary LLM model name", default=None)
parser.add_argument("--secondary-model", help="Secondary LLM model name", default=None)
@@ -60,6 +70,32 @@ def parse_args() -> argparse.Namespace:
return parser.parse_args()
+def parse_evaluator_args(raw: str) -> Dict[str, Any]:
+ """
+ Parse the --evaluator-args JSON string into a kwargs dict
+
+ Args:
+ raw: JSON object literal of keyword arguments
+
+ Returns:
+ Dict of keyword arguments for the evaluation function
+
+ Raises:
+ ValueError: If the string is not valid JSON or not a JSON object
+ """
+ try:
+ parsed = json.loads(raw)
+ except json.JSONDecodeError as e:
+ raise ValueError(f"--evaluator-args is not valid JSON: {e}") from e
+
+ if not isinstance(parsed, dict):
+ type_name = type(parsed).__name__
+ raise ValueError(
+ f"--evaluator-args must be a JSON object of keyword arguments, got {type_name}"
+ )
+ return parsed
+
+
async def main_async() -> int:
"""
Main asynchronous entry point
@@ -81,6 +117,16 @@ async def main_async() -> int:
# Load base config from file or defaults
config = load_config(args.config)
+ # Apply evaluator args CLI override (kwargs forwarded to evaluate(), issue #474)
+ if args.evaluator_args is not None:
+ try:
+ config.evaluator.evaluator_args = parse_evaluator_args(args.evaluator_args)
+ except ValueError as e:
+ print(f"Error: {str(e)}")
+ return 1
+ # Print keys only - values may contain secrets and end up in logs
+ print(f"Using evaluator args keys: {sorted(config.evaluator.evaluator_args.keys())}")
+
# Create config object with command-line overrides
if args.api_base or args.primary_model or args.secondary_model:
# Apply command-line overrides
diff --git a/openevolve/config.py b/openevolve/config.py
index d68b701f8..2844e7f77 100644
--- a/openevolve/config.py
+++ b/openevolve/config.py
@@ -409,6 +409,11 @@ class EvaluatorConfig:
enable_artifacts: bool = True
max_artifact_storage: int = 100 * 1024 * 1024 # 100MB per program
+ # Extra keyword arguments forwarded to the user's evaluate() function
+ # (only applied when its signature accepts them; see openevolve.evaluator).
+ # Settable via YAML ('evaluator.evaluator_args') or the --evaluator-args CLI flag.
+ evaluator_args: Optional[Dict[str, Any]] = None
+
@dataclass
class EvolutionTraceConfig:
diff --git a/openevolve/evaluator.py b/openevolve/evaluator.py
index 30cb3bda7..5c91c39f4 100644
--- a/openevolve/evaluator.py
+++ b/openevolve/evaluator.py
@@ -3,7 +3,9 @@
"""
import asyncio
+import functools
import importlib.util
+import inspect
import json
import logging
import os
@@ -16,15 +18,13 @@
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
-import traceback
from openevolve.config import EvaluatorConfig
from openevolve.database import ProgramDatabase
from openevolve.evaluation_result import EvaluationResult
-from openevolve.database import ProgramDatabase
from openevolve.llm.ensemble import LLMEnsemble
-from openevolve.utils.async_utils import TaskPool, run_in_executor
from openevolve.prompt.sampler import PromptSampler
+from openevolve.utils.async_utils import TaskPool, run_in_executor
from openevolve.utils.format_utils import format_metrics_safe
logger = logging.getLogger(__name__)
@@ -54,6 +54,11 @@ def __init__(
self.prompt_sampler = prompt_sampler
self.database = database
+ # Keyword arguments forwarded to the user's evaluate() function; populated
+ # by _load_evaluation_function() based on config.evaluator.evaluator_args
+ # and the signature of the loaded function
+ self._evaluator_kwargs: Dict[str, Any] = {}
+
# Create a task pool for parallel evaluation
self.task_pool = TaskPool(max_concurrency=config.parallel_evaluations)
@@ -98,6 +103,7 @@ def _load_evaluation_function(self) -> None:
)
self.evaluate_function = module.evaluate
+ self._evaluator_kwargs = self._resolve_evaluator_kwargs(module.evaluate)
logger.info(f"Successfully loaded evaluation function from {self.evaluation_file}")
# Validate cascade configuration
@@ -106,6 +112,94 @@ def _load_evaluation_function(self) -> None:
logger.error(f"Error loading evaluation function: {str(e)}")
raise
+ def _resolve_evaluator_kwargs(self, evaluate_function: Callable) -> Dict[str, Any]:
+ """
+ Resolve the keyword arguments to forward to the user's evaluate() function.
+
+ Uses inspect.signature to verify that evaluate() actually accepts the
+ configured evaluator_args (GitHub issue #474):
+
+ - Evaluators accepting **kwargs get all configured kwargs.
+ - Evaluators whose named parameters cover the configured keys get them
+ forwarded on every call.
+ - Evaluators that cannot accept a configured key fail here, at startup,
+ with an actionable error instead of failing on every evaluation.
+
+ Evaluators with no configured evaluator_args (the common single
+ `program_path` argument case) are unaffected and keep their existing
+ behavior.
+
+ Args:
+ evaluate_function: The user-provided evaluate() function
+
+ Returns:
+ Dict of keyword arguments to forward (empty when nothing configured)
+
+ Raises:
+ ValueError: If evaluate() cannot accept a configured evaluator_arg
+ """
+ evaluator_args = self.config.evaluator_args or {}
+ if not evaluator_args:
+ return {}
+
+ function_name = getattr(evaluate_function, "__name__", "evaluate")
+ try:
+ signature = inspect.signature(evaluate_function)
+ except (TypeError, ValueError):
+ # Some callables (e.g. certain builtins) expose no signature; forward
+ # as-is and let the call surface any mismatch
+ logger.warning(
+ f"Could not inspect the signature of '{function_name}'; forwarding "
+ f"evaluator_args keys: {sorted(evaluator_args.keys())}"
+ )
+ return dict(evaluator_args)
+
+ # Guard against kwargs that cannot be forwarded together with the program
+ # path passed positionally (e.g. evaluator_args={"program_path": ...} for
+ # evaluate(program_path)). Runs before the **kwargs shortcut below: binding
+ # mirrors the actual call functools.partial(evaluate, **args)(program_path),
+ # so it also rejects a configured key shadowing a positional parameter of a
+ # **kwargs evaluator (TypeError: multiple values for argument).
+ try:
+ signature.bind(object(), **evaluator_args)
+ except TypeError as e:
+ raise ValueError(
+ f"Configured evaluator_args {sorted(evaluator_args.keys())} cannot be "
+ f"forwarded to evaluation function '{function_name}' in "
+ f"{self.evaluation_file}: {e}. Its signature is '{function_name}{signature}'."
+ ) from e
+
+ if any(
+ param.kind == inspect.Parameter.VAR_KEYWORD for param in signature.parameters.values()
+ ):
+ # **kwargs swallows everything
+ logger.info(
+ f"Forwarding evaluator_args keys to '{function_name}': "
+ f"{sorted(evaluator_args.keys())}"
+ )
+ return dict(evaluator_args)
+
+ named_params = {
+ name
+ for name, param in signature.parameters.items()
+ if param.kind
+ in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY)
+ }
+ unsupported = sorted(set(evaluator_args.keys()) - named_params)
+ if unsupported:
+ raise ValueError(
+ f"Evaluation function '{function_name}' in {self.evaluation_file} does not "
+ f"accept the configured evaluator_args keys: {unsupported}. Its signature is "
+ f"'{function_name}{signature}'. Add the parameters to evaluate(), accept "
+ f"**kwargs, or remove the keys from evaluator.evaluator_args / --evaluator-args."
+ )
+
+ logger.info(
+ f"Forwarding evaluator_args keys to '{function_name}': "
+ f"{sorted(evaluator_args.keys())}"
+ )
+ return dict(evaluator_args)
+
def _validate_cascade_configuration(self, module) -> None:
"""
Validate cascade evaluation configuration and warn about potential issues
@@ -356,7 +450,11 @@ async def _direct_evaluate(
# Create a coroutine that runs the evaluation function in an executor
async def run_evaluation():
loop = asyncio.get_event_loop()
- return await loop.run_in_executor(self._executor, self.evaluate_function, program_path)
+ return await loop.run_in_executor(
+ self._executor,
+ functools.partial(self.evaluate_function, **self._evaluator_kwargs),
+ program_path,
+ )
# Run the evaluation with timeout - let exceptions bubble up for retry handling
result = await asyncio.wait_for(run_evaluation(), timeout=self.config.timeout)
diff --git a/tests/test_evaluator_args.py b/tests/test_evaluator_args.py
new file mode 100644
index 000000000..067f9e7e9
--- /dev/null
+++ b/tests/test_evaluator_args.py
@@ -0,0 +1,285 @@
+"""
+Tests for forwarding evaluator_args kwargs to the evaluation function
+(GitHub issue #474: --evaluator-args CLI flag + config.evaluator.evaluator_args)
+"""
+
+import asyncio
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+
+from openevolve.cli import parse_evaluator_args
+from openevolve.config import Config, EvaluatorConfig
+from openevolve.evaluator import Evaluator
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+class EvaluatorArgsTestBase(unittest.TestCase):
+ """Shared helpers for writing temporary evaluation files"""
+
+ def setUp(self):
+ self._temp_files = []
+
+ def tearDown(self):
+ for path in self._temp_files:
+ if os.path.exists(path):
+ os.unlink(path)
+
+ def _write_eval_file(self, source: str) -> str:
+ temp_file = tempfile.NamedTemporaryFile(
+ mode="w", suffix=".py", delete=False, encoding="utf-8"
+ )
+ temp_file.write(source)
+ temp_file.close()
+ self._temp_files.append(temp_file.name)
+ return temp_file.name
+
+ def _create_evaluator(self, eval_file: str, evaluator_args=None) -> Evaluator:
+ config = EvaluatorConfig()
+ config.timeout = 10
+ config.max_retries = 1
+ config.cascade_evaluation = False
+ config.evaluator_args = evaluator_args
+ return Evaluator(
+ config=config,
+ evaluation_file=eval_file,
+ llm_ensemble=None,
+ prompt_sampler=None,
+ )
+
+
+class TestEvaluatorKwargsForwarding(EvaluatorArgsTestBase):
+ """Test that configured kwargs reach the evaluation function"""
+
+ def test_multi_param_evaluator_receives_kwargs(self):
+ """evaluate(program_path, weight=...) receives the configured kwarg"""
+ eval_file = self._write_eval_file("""
+def evaluate(program_path, weight=1.0):
+ return {"weighted_score": weight}
+""")
+ evaluator = self._create_evaluator(eval_file, evaluator_args={"weight": 2.5})
+
+ async def run_test():
+ return await evaluator.evaluate_program("x = 1", "test_weighted")
+
+ result = asyncio.run(run_test())
+ self.assertEqual(result["weighted_score"], 2.5)
+
+ def test_var_keyword_evaluator_receives_kwargs(self):
+ """evaluate(program_path, **kwargs) receives all configured kwargs"""
+ eval_file = self._write_eval_file("""
+def evaluate(program_path, **kwargs):
+ return {"score": 1.0 if kwargs.get("factor") == 3.0 and kwargs.get("shift") == -1 else 0.0}
+""")
+ evaluator = self._create_evaluator(eval_file, evaluator_args={"factor": 3.0, "shift": -1})
+
+ async def run_test():
+ return await evaluator.evaluate_program("x = 1", "test_var_keyword")
+
+ result = asyncio.run(run_test())
+ self.assertEqual(result["score"], 1.0)
+
+ def test_single_param_evaluator_unchanged_without_args(self):
+ """No evaluator_args configured: single-argument evaluators keep working"""
+ eval_file = self._write_eval_file("""
+def evaluate(program_path):
+ return {"score": 0.5}
+""")
+ evaluator = self._create_evaluator(eval_file)
+ self.assertEqual(evaluator._evaluator_kwargs, {})
+
+ async def run_test():
+ return await evaluator.evaluate_program("x = 1", "test_plain")
+
+ result = asyncio.run(run_test())
+ self.assertEqual(result["score"], 0.5)
+
+ def test_configured_empty_args_unchanged(self):
+ """Explicitly empty evaluator_args dict behaves like no args"""
+ eval_file = self._write_eval_file("""
+def evaluate(program_path):
+ return {"score": 0.7}
+""")
+ evaluator = self._create_evaluator(eval_file, evaluator_args={})
+ self.assertEqual(evaluator._evaluator_kwargs, {})
+
+ async def run_test():
+ return await evaluator.evaluate_program("x = 1", "test_empty")
+
+ result = asyncio.run(run_test())
+ self.assertEqual(result["score"], 0.7)
+
+
+class TestEvaluatorKwargsValidation(EvaluatorArgsTestBase):
+ """Test startup validation of evaluator_args against the evaluate() signature"""
+
+ def test_unsupported_key_raises_at_startup(self):
+ """Configured key not accepted by evaluate() fails at load time with the key named"""
+ eval_file = self._write_eval_file("""
+def evaluate(program_path):
+ return {"score": 1.0}
+""")
+ with self.assertRaises(ValueError) as ctx:
+ self._create_evaluator(eval_file, evaluator_args={"weight": 1.0})
+
+ message = str(ctx.exception)
+ self.assertIn("weight", message)
+ self.assertIn("evaluate", message)
+ self.assertIn(eval_file, message)
+
+ def test_program_path_collision_raises(self):
+ """A kwarg named like the positional program path parameter is rejected"""
+ eval_file = self._write_eval_file("""
+def evaluate(program_path):
+ return {"score": 1.0}
+""")
+ with self.assertRaises(ValueError) as ctx:
+ self._create_evaluator(eval_file, evaluator_args={"program_path": "other.py"})
+
+ self.assertIn("program_path", str(ctx.exception))
+
+ def test_program_path_collision_raises_for_var_keyword_evaluator(self):
+ """A configured key shadowing the positional param of a **kwargs evaluator is rejected"""
+ eval_file = self._write_eval_file("""
+def evaluate(program_path, **kwargs):
+ return {"score": kwargs.get("weight", 1.0)}
+""")
+ with self.assertRaises(ValueError) as ctx:
+ self._create_evaluator(eval_file, evaluator_args={"program_path": "other.py"})
+
+ message = str(ctx.exception)
+ self.assertIn("program_path", message)
+ self.assertIn("multiple values", message)
+
+ def test_var_keyword_evaluator_still_accepts_non_colliding_kwargs(self):
+ """**kwargs evaluators keep receiving configured keys that do not shadow parameters"""
+ eval_file = self._write_eval_file("""
+def evaluate(program_path, **kwargs):
+ return {"score": kwargs.get("weight", 1.0)}
+""")
+ evaluator = self._create_evaluator(eval_file, evaluator_args={"weight": 0.25})
+
+ async def run_test():
+ return await evaluator.evaluate_program("x = 1", "test_vk_no_collision")
+
+ result = asyncio.run(run_test())
+ self.assertEqual(result["score"], 0.25)
+
+ def test_error_lists_actual_signature(self):
+ """The actionable error includes the evaluator's actual signature"""
+ eval_file = self._write_eval_file("""
+def evaluate(program_path):
+ return {"score": 1.0}
+""")
+ with self.assertRaises(ValueError) as ctx:
+ self._create_evaluator(eval_file, evaluator_args={"a": 1, "b": 2})
+
+ message = str(ctx.exception)
+ self.assertIn("evaluate(program_path)", message)
+ self.assertIn("'a'", message)
+ self.assertIn("'b'", message)
+
+
+class TestEvaluatorArgsConfig(unittest.TestCase):
+ """Test config plumbing (YAML load + worker dict round-trip)"""
+
+ def test_yaml_load_evaluator_args(self):
+ """evaluator.evaluator_args loads from YAML via dacite"""
+ import tempfile
+
+ config_content = """
+evaluator:
+ timeout: 60
+ evaluator_args:
+ weight: 2.0
+ label: "test"
+"""
+ with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
+ f.write(config_content)
+ config_path = f.name
+ try:
+ from openevolve.config import load_config
+
+ config = load_config(config_path)
+ self.assertEqual(config.evaluator.evaluator_args, {"weight": 2.0, "label": "test"})
+ finally:
+ os.unlink(config_path)
+
+ def test_worker_round_trip_preserves_evaluator_args(self):
+ """to_dict() -> EvaluatorConfig(**d['evaluator']) keeps the field (process pool path)"""
+ config = Config(language="python")
+ config.evaluator.evaluator_args = {"weight": 3.5, "mode": "fast"}
+
+ config_dict = config.to_dict()
+ reconstructed = EvaluatorConfig(**config_dict["evaluator"])
+
+ self.assertEqual(reconstructed.evaluator_args, {"weight": 3.5, "mode": "fast"})
+
+ def test_default_is_none(self):
+ """Default config has no evaluator_args (backward compatible)"""
+ self.assertIsNone(EvaluatorConfig().evaluator_args)
+
+
+class TestParseEvaluatorArgs(unittest.TestCase):
+ """Test the CLI --evaluator-args JSON parsing helper"""
+
+ def test_valid_json_object(self):
+ self.assertEqual(
+ parse_evaluator_args('{"weight": 2.0, "label": "x"}'), {"weight": 2.0, "label": "x"}
+ )
+
+ def test_empty_object(self):
+ self.assertEqual(parse_evaluator_args("{}"), {})
+
+ def test_invalid_json_raises(self):
+ with self.assertRaises(ValueError) as ctx:
+ parse_evaluator_args("{'weight': 2.0}") # single quotes are not JSON
+ self.assertIn("not valid JSON", str(ctx.exception))
+
+ def test_non_object_json_raises(self):
+ with self.assertRaises(ValueError) as ctx:
+ parse_evaluator_args("[1, 2, 3]")
+ self.assertIn("JSON object", str(ctx.exception))
+
+
+class TestCLIEvaluatorArgsExitCode(unittest.TestCase):
+ """End-to-end CLI check: invalid --evaluator-args JSON exits with code 1"""
+
+ def test_invalid_json_exits_1(self):
+ program_file = tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False)
+ program_file.write("x = 1\n")
+ program_file.close()
+ eval_file = tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False)
+ eval_file.write("def evaluate(program_path):\n return {'score': 1.0}\n")
+ eval_file.close()
+ try:
+ env = dict(os.environ)
+ env.setdefault("OPENAI_API_KEY", "test-key-for-unit-tests")
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-m",
+ "openevolve.cli",
+ program_file.name,
+ eval_file.name,
+ "--evaluator-args",
+ "{invalid json",
+ ],
+ cwd=REPO_ROOT,
+ env=env,
+ capture_output=True,
+ text=True,
+ timeout=60,
+ )
+ self.assertEqual(result.returncode, 1)
+ self.assertIn("not valid JSON", result.stdout + result.stderr)
+ finally:
+ os.unlink(program_file.name)
+ os.unlink(eval_file.name)
+
+
+if __name__ == "__main__":
+ unittest.main()