From 3c74a362187fe7161572848c5d5b618b18c914df Mon Sep 17 00:00:00 2001 From: ZY Liu <236001964+liuzhengyang699@users.noreply.github.com> Date: Sun, 11 Oct 2026 11:02:34 +0800 Subject: [PATCH] fix(evaluator): retain valid feedback from partial judge ensembles --- openevolve/evaluator.py | 66 ++++++++++++++------------- tests/test_llm_evaluation_ensemble.py | 66 +++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 31 deletions(-) create mode 100644 tests/test_llm_evaluation_ensemble.py diff --git a/openevolve/evaluator.py b/openevolve/evaluator.py index 5c91c39f4..084d1c04c 100644 --- a/openevolve/evaluator.py +++ b/openevolve/evaluator.py @@ -703,41 +703,45 @@ async def _llm_evaluate(self, program_code: str, program_id: str = "") -> Dict[s import re artifacts = {} - avg_metrics = {} + weighted_metrics = {} + metric_weights = {} for i, response in enumerate(responses): - json_match = re.search(json_pattern, response, re.DOTALL) - - if json_match: - json_str = json_match.group(1) - else: - # Try to extract JSON directly - json_str = response - # Remove non-JSON parts - start_idx = json_str.find("{") - end_idx = json_str.rfind("}") + 1 - if start_idx >= 0 and end_idx > start_idx: - json_str = json_str[start_idx:end_idx] - - # Parse JSON - result = json.loads(json_str) - - # All non-numeric values are artifacts, all numeric values are metrics - metrics = {} - for key, value in result.items(): - if not isinstance(value, (int, float)): - artifacts[key] = value - else: - metrics[key] = float(value) - - # Weight of the model in the ensemble weight = self.llm_ensemble.weights[i] if self.llm_ensemble.weights else 1.0 + if weight <= 0: + continue - # Average the metrics - for name, value in metrics.items(): - if name in avg_metrics: - avg_metrics[name] += value * weight + try: + json_match = re.search(json_pattern, response, re.DOTALL) + if json_match: + json_str = json_match.group(1) else: - avg_metrics[name] = value * weight + json_str = response + start_idx = json_str.find("{") + end_idx = json_str.rfind("}") + 1 + if start_idx >= 0 and end_idx > start_idx: + json_str = json_str[start_idx:end_idx] + + result = json.loads(json_str) + if not isinstance(result, dict): + raise ValueError("LLM evaluation must return a JSON object") + except (ValueError, TypeError) as error: + logger.warning("Skipping invalid LLM judge response %d: %s", i, error) + continue + + for name, value in result.items(): + if not isinstance(value, (int, float)): + artifacts[name] = value + continue + weighted_metrics[name] = ( + weighted_metrics.get(name, 0.0) + float(value) * weight + ) + metric_weights[name] = metric_weights.get(name, 0.0) + weight + + # A missing or invalid response is not a zero score. Normalize + # each metric using only the judges that actually supplied it. + avg_metrics = { + name: value / metric_weights[name] for name, value in weighted_metrics.items() + } return EvaluationResult( metrics=avg_metrics, diff --git a/tests/test_llm_evaluation_ensemble.py b/tests/test_llm_evaluation_ensemble.py new file mode 100644 index 000000000..948b4a13f --- /dev/null +++ b/tests/test_llm_evaluation_ensemble.py @@ -0,0 +1,66 @@ +"""Keep valid LLM judge feedback when an ensemble response is incomplete.""" + +import tempfile +import unittest +from pathlib import Path +from unittest.mock import AsyncMock, Mock + +from openevolve.config import EvaluatorConfig +from openevolve.evaluator import Evaluator + + +class TestLLMEvaluationEnsemble(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + evaluation_file = Path(self.directory.name) / "evaluate.py" + evaluation_file.write_text('def evaluate(path):\n return {"combined_score": 0.5}\n') + self.ensemble = Mock(weights=[0.25, 0.75]) + self.sampler = Mock() + self.sampler.build_prompt.return_value = {"system": "judge", "user": "code"} + self.evaluator = Evaluator( + EvaluatorConfig(cascade_evaluation=False, use_llm_feedback=True), + str(evaluation_file), + llm_ensemble=self.ensemble, + prompt_sampler=self.sampler, + ) + self.addCleanup(self.evaluator._executor.shutdown) + + async def evaluate_responses(self, responses): + self.ensemble.generate_all_with_context = AsyncMock(return_value=responses) + result = await self.evaluator._llm_evaluate("x = 1") + return self.evaluator._process_evaluation_result(result) + + async def test_invalid_judge_does_not_discard_valid_feedback(self): + for invalid in ("not JSON", "[]", "null"): + for responses in ([invalid, '{"quality": 0.8}'], ['{"quality": 0.8}', invalid]): + with self.subTest(responses=responses): + result = await self.evaluate_responses(responses) + self.assertAlmostEqual(result.metrics["quality"], 0.8) + + async def test_missing_metric_uses_weights_of_judges_that_report_it(self): + result = await self.evaluate_responses( + ['{"quality": 0.4, "clarity": 0.6}', '{"quality": 0.8}'] + ) + self.assertAlmostEqual(result.metrics["quality"], 0.7) + self.assertAlmostEqual(result.metrics["clarity"], 0.6) + + async def test_zero_weight_judge_does_not_create_a_metric(self): + self.ensemble.weights = [0.0, 1.0] + result = await self.evaluate_responses(['{"quality": 0.9}', '{"clarity": 0.8}']) + self.assertEqual(result.metrics, {"clarity": 0.8}) + + async def test_all_invalid_judges_leave_measured_fitness_unchanged(self): + self.ensemble.generate_all_with_context = AsyncMock(return_value=["invalid", "null"]) + metrics = await self.evaluator.evaluate_program("x = 1", "all-invalid") + self.assertEqual(metrics, {"combined_score": 0.5}) + + async def test_valid_feedback_still_contributes_to_combined_fitness(self): + self.ensemble.generate_all_with_context = AsyncMock( + return_value=["invalid", '{"quality": 0.8, "comment": "clear"}'] + ) + metrics = await self.evaluator.evaluate_program("x = 1", "one-valid") + self.assertAlmostEqual(metrics["combined_score"], 0.59) + self.assertAlmostEqual( + metrics["llm_average"], 0.8 * self.evaluator.config.llm_feedback_weight + )