Add --evaluator-args flag forwarding kwargs to evaluate() via signature sniffing

README.md

@@ -575,6 +575,8 @@ evaluator:
   enable_artifacts: true      # Error feedback to LLM
   cascade_evaluation: true    # Multi-stage testing
   use_llm_feedback: true      # AI code quality assessment
+  evaluator_args:             # Keyword arguments forwarded to evaluate()
+    weight: 2.0
 
 prompt:
   # Sophisticated inspiration system
@@ -613,6 +615,39 @@ database:
 
 </details>
 
+<details>
+<summary><b>📞 Passing Arguments to Your Evaluator</b></summary>
+
+If your `evaluate()` function takes extra keyword arguments beyond the program
+path, pass them with the `--evaluator-args` CLI flag (a JSON object):
+
+```bash
+openevolve-run initial_program.py evaluator.py \
+  --evaluator-args '{"weight": 2.0, "label": "experiment-1"}'
+```
+
+or configure them in YAML:
+
+```yaml
+evaluator:
+  evaluator_args:
+    weight: 2.0
+    label: "experiment-1"
+```
+
+OpenEvolve inspects the signature of your `evaluate()` function: arguments are
+forwarded only when the function accepts them (named parameters or `**kwargs`),
+and a startup error lists the offending keys and the actual signature if it
+does not — so typos fail fast instead of silently doing nothing. Evaluators
+with a single `program_path` argument are unaffected.
+
+This replaces the earlier workaround of smuggling per-run settings to the
+evaluator through environment variables. (Note: custom top-level keys in
+`config.yaml` are dropped silently by the config parser — use
+`evaluator.evaluator_args` or the CLI flag instead.)
+
+</details>
+
 <details>
 <summary><b>🎨 Custom Prompt Templates</b></summary>
 

openevolve/cli.py

@@ -4,10 +4,11 @@
 
 import argparse
 import asyncio
+import json
 import logging
 import os
 import sys
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 from openevolve import OpenEvolve
 from openevolve.config import Config, load_config
@@ -53,13 +54,48 @@ def parse_args() -> argparse.Namespace:
 
     parser.add_argument("--api-base", help="Base URL for the LLM API", default=None)
 
+    parser.add_argument(
+        "--evaluator-args",
+        help=(
+            "JSON object of keyword arguments forwarded to the 'evaluate' function of the "
+            "evaluation file, e.g. '{\"weight\": 2.0}' (the function must accept them)"
+        ),
+        default=None,
+    )
+
     parser.add_argument("--primary-model", help="Primary LLM model name", default=None)
 
     parser.add_argument("--secondary-model", help="Secondary LLM model name", default=None)
 
     return parser.parse_args()
 
 
+def parse_evaluator_args(raw: str) -> Dict[str, Any]:
+    """
+    Parse the --evaluator-args JSON string into a kwargs dict
+
+    Args:
+        raw: JSON object literal of keyword arguments
+
+    Returns:
+        Dict of keyword arguments for the evaluation function
+
+    Raises:
+        ValueError: If the string is not valid JSON or not a JSON object
+    """
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"--evaluator-args is not valid JSON: {e}") from e
+
+    if not isinstance(parsed, dict):
+        type_name = type(parsed).__name__
+        raise ValueError(
+            f"--evaluator-args must be a JSON object of keyword arguments, got {type_name}"
+        )
+    return parsed
+
+
 async def main_async() -> int:
     """
     Main asynchronous entry point
@@ -81,6 +117,16 @@ async def main_async() -> int:
     # Load base config from file or defaults
     config = load_config(args.config)
 
+    # Apply evaluator args CLI override (kwargs forwarded to evaluate(), issue #474)
+    if args.evaluator_args is not None:
+        try:
+            config.evaluator.evaluator_args = parse_evaluator_args(args.evaluator_args)
+        except ValueError as e:
+            print(f"Error: {str(e)}")
+            return 1
+        # Print keys only - values may contain secrets and end up in logs
+        print(f"Using evaluator args keys: {sorted(config.evaluator.evaluator_args.keys())}")
+
     # Create config object with command-line overrides
     if args.api_base or args.primary_model or args.secondary_model:
         # Apply command-line overrides

openevolve/config.py

@@ -409,6 +409,11 @@ class EvaluatorConfig:
     enable_artifacts: bool = True
     max_artifact_storage: int = 100 * 1024 * 1024  # 100MB per program
 
+    # Extra keyword arguments forwarded to the user's evaluate() function
+    # (only applied when its signature accepts them; see openevolve.evaluator).
+    # Settable via YAML ('evaluator.evaluator_args') or the --evaluator-args CLI flag.
+    evaluator_args: Optional[Dict[str, Any]] = None
+
 
 @dataclass
 class EvolutionTraceConfig:

openevolve/evaluator.py

@@ -3,7 +3,9 @@
 """
 
 import asyncio
+import functools
 import importlib.util
+import inspect
 import json
 import logging
 import os
@@ -16,15 +18,13 @@
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
-import traceback
 
 from openevolve.config import EvaluatorConfig
 from openevolve.database import ProgramDatabase
 from openevolve.evaluation_result import EvaluationResult
-from openevolve.database import ProgramDatabase
 from openevolve.llm.ensemble import LLMEnsemble
-from openevolve.utils.async_utils import TaskPool, run_in_executor
 from openevolve.prompt.sampler import PromptSampler
+from openevolve.utils.async_utils import TaskPool, run_in_executor
 from openevolve.utils.format_utils import format_metrics_safe
 
 logger = logging.getLogger(__name__)
@@ -54,6 +54,11 @@ def __init__(
         self.prompt_sampler = prompt_sampler
         self.database = database
 
+        # Keyword arguments forwarded to the user's evaluate() function; populated
+        # by _load_evaluation_function() based on config.evaluator.evaluator_args
+        # and the signature of the loaded function
+        self._evaluator_kwargs: Dict[str, Any] = {}
+
         # Create a task pool for parallel evaluation
         self.task_pool = TaskPool(max_concurrency=config.parallel_evaluations)
 
@@ -98,6 +103,7 @@ def _load_evaluation_function(self) -> None:
                 )
 
             self.evaluate_function = module.evaluate
+            self._evaluator_kwargs = self._resolve_evaluator_kwargs(module.evaluate)
             logger.info(f"Successfully loaded evaluation function from {self.evaluation_file}")
 
             # Validate cascade configuration
@@ -106,6 +112,94 @@ def _load_evaluation_function(self) -> None:
             logger.error(f"Error loading evaluation function: {str(e)}")
             raise
 
+    def _resolve_evaluator_kwargs(self, evaluate_function: Callable) -> Dict[str, Any]:
+        """
+        Resolve the keyword arguments to forward to the user's evaluate() function.
+
+        Uses inspect.signature to verify that evaluate() actually accepts the
+        configured evaluator_args (GitHub issue #474):
+
+        - Evaluators accepting **kwargs get all configured kwargs.
+        - Evaluators whose named parameters cover the configured keys get them
+          forwarded on every call.
+        - Evaluators that cannot accept a configured key fail here, at startup,
+          with an actionable error instead of failing on every evaluation.
+
+        Evaluators with no configured evaluator_args (the common single
+        `program_path` argument case) are unaffected and keep their existing
+        behavior.
+
+        Args:
+            evaluate_function: The user-provided evaluate() function
+
+        Returns:
+            Dict of keyword arguments to forward (empty when nothing configured)
+
+        Raises:
+            ValueError: If evaluate() cannot accept a configured evaluator_arg
+        """
+        evaluator_args = self.config.evaluator_args or {}
+        if not evaluator_args:
+            return {}
+
+        function_name = getattr(evaluate_function, "__name__", "evaluate")
+        try:
+            signature = inspect.signature(evaluate_function)
+        except (TypeError, ValueError):
+            # Some callables (e.g. certain builtins) expose no signature; forward
+            # as-is and let the call surface any mismatch
+            logger.warning(
+                f"Could not inspect the signature of '{function_name}'; forwarding "
+                f"evaluator_args keys: {sorted(evaluator_args.keys())}"
+            )
+            return dict(evaluator_args)
+
+        # Guard against kwargs that cannot be forwarded together with the program
+        # path passed positionally (e.g. evaluator_args={"program_path": ...} for
+        # evaluate(program_path)). Runs before the **kwargs shortcut below: binding
+        # mirrors the actual call functools.partial(evaluate, **args)(program_path),
+        # so it also rejects a configured key shadowing a positional parameter of a
+        # **kwargs evaluator (TypeError: multiple values for argument).
+        try:
+            signature.bind(object(), **evaluator_args)
+        except TypeError as e:
+            raise ValueError(
+                f"Configured evaluator_args {sorted(evaluator_args.keys())} cannot be "
+                f"forwarded to evaluation function '{function_name}' in "
+                f"{self.evaluation_file}: {e}. Its signature is '{function_name}{signature}'."
+            ) from e
+
+        if any(
+            param.kind == inspect.Parameter.VAR_KEYWORD for param in signature.parameters.values()
+        ):
+            # **kwargs swallows everything
+            logger.info(
+                f"Forwarding evaluator_args keys to '{function_name}': "
+                f"{sorted(evaluator_args.keys())}"
+            )
+            return dict(evaluator_args)
+
+        named_params = {
+            name
+            for name, param in signature.parameters.items()
+            if param.kind
+            in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY)
+        }
+        unsupported = sorted(set(evaluator_args.keys()) - named_params)
+        if unsupported:
+            raise ValueError(
+                f"Evaluation function '{function_name}' in {self.evaluation_file} does not "
+                f"accept the configured evaluator_args keys: {unsupported}. Its signature is "
+                f"'{function_name}{signature}'. Add the parameters to evaluate(), accept "
+                f"**kwargs, or remove the keys from evaluator.evaluator_args / --evaluator-args."
+            )
+
+        logger.info(
+            f"Forwarding evaluator_args keys to '{function_name}': "
+            f"{sorted(evaluator_args.keys())}"
+        )
+        return dict(evaluator_args)
+
     def _validate_cascade_configuration(self, module) -> None:
         """
         Validate cascade evaluation configuration and warn about potential issues
@@ -356,7 +450,11 @@ async def _direct_evaluate(
         # Create a coroutine that runs the evaluation function in an executor
         async def run_evaluation():
             loop = asyncio.get_event_loop()
-            return await loop.run_in_executor(self._executor, self.evaluate_function, program_path)
+            return await loop.run_in_executor(
+                self._executor,
+                functools.partial(self.evaluate_function, **self._evaluator_kwargs),
+                program_path,
+            )
 
         # Run the evaluation with timeout - let exceptions bubble up for retry handling
         result = await asyncio.wait_for(run_evaluation(), timeout=self.config.timeout)