From 6256828b6f3d5cd3375613bab216220790771eb0 Mon Sep 17 00:00:00 2001 From: Jiang Wu <245355730+JiangLLM@users.noreply.github.com> Date: Sat, 19 Sep 2026 13:53:17 -0700 Subject: [PATCH] fix(eval): classify PrefEval judge answers like the reference benchmark classify_error_type compared judge answers with exact 'Yes'/'No' strings and sent every unmatched case to the 'Personalized Response' bucket, including empty answers from failed judge calls and responses that were both violating and unhelpful. Move the classifier to evaluation/scripts/utils/pref_classify.py, mirror the amazon-science/PrefEval semantics, report unrecognized answers as 'Judge Failure', and add unit tests. --- evaluation/scripts/PrefEval/pref_eval.py | 23 ++--- evaluation/scripts/utils/pref_classify.py | 71 +++++++++++++++ tests/evaluation/test_pref_classify.py | 103 ++++++++++++++++++++++ 3 files changed, 179 insertions(+), 18 deletions(-) create mode 100644 evaluation/scripts/utils/pref_classify.py create mode 100644 tests/evaluation/test_pref_classify.py diff --git a/evaluation/scripts/PrefEval/pref_eval.py b/evaluation/scripts/PrefEval/pref_eval.py index 9da3c9438..0f782fae7 100644 --- a/evaluation/scripts/PrefEval/pref_eval.py +++ b/evaluation/scripts/PrefEval/pref_eval.py @@ -3,6 +3,7 @@ import json import os import re +import sys from collections import Counter from typing import Any @@ -14,6 +15,10 @@ from tqdm.asyncio import tqdm +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from utils.pref_classify import classify_error_type + + load_dotenv() API_KEY = os.getenv("OPENAI_API_KEY") @@ -178,24 +183,6 @@ async def evaluate_helpful_response_async( } -def classify_error_type(evaluation_results: dict[str, Any]) -> str: - violate = evaluation_results["violate_preference"]["answer"] - acknowledge = evaluation_results["acknowledge_preference"]["answer"] - hallucinate = evaluation_results["hallucinate_preference"]["answer"] - helpful = evaluation_results["helpful_response"]["answer"] - - if violate == "Yes" and acknowledge == "No" and helpful == "Yes": - return "Preference-Unaware Violation" - elif violate == "Yes" and acknowledge == "Yes" and hallucinate == "Yes" and helpful == "Yes": - return "Preference Hallucination Violation" - elif violate == "Yes" and acknowledge == "Yes" and hallucinate == "No" and helpful == "Yes": - return "Inconsistency Violation" - elif violate == "No" and helpful == "No": - return "Unhelpful Response" - else: - return "Personalized Response" - - async def process_line(line: str, client: OpenAI, semaphore: asyncio.Semaphore) -> dict[str, Any]: async with semaphore: data = json.loads(line.strip()) diff --git a/evaluation/scripts/utils/pref_classify.py b/evaluation/scripts/utils/pref_classify.py new file mode 100644 index 000000000..7e587de70 --- /dev/null +++ b/evaluation/scripts/utils/pref_classify.py @@ -0,0 +1,71 @@ +"""Error-type classification for PrefEval judge answers. + +Kept free of third-party imports so it can be unit tested without the +evaluation dependency group. +""" + +from typing import Any + + +JUDGE_FAILURE = "Judge Failure" +PERSONALIZED_RESPONSE = "Personalized Response" +UNHELPFUL_RESPONSE = "Unhelpful Response" +PREFERENCE_UNAWARE_VIOLATION = "Preference-Unaware Violation" +PREFERENCE_HALLUCINATION_VIOLATION = "Preference Hallucination Violation" +INCONSISTENCY_VIOLATION = "Inconsistency Violation" + +_JUDGE_KEYS = ( + "violate_preference", + "acknowledge_preference", + "hallucinate_preference", + "helpful_response", +) + + +def parse_yes_no(answer: str | None) -> bool | None: + """Map a judge answer to True (yes) or False (no). + + Returns None when the answer is missing or is neither yes nor no, for + example the empty string that the judge call returns after an API error. + """ + if answer is None: + return None + normalized = answer.strip().lower() + if normalized.startswith("yes"): + return True + if normalized.startswith("no"): + return False + return None + + +def classify_error_type(evaluation_results: dict[str, Any]) -> str: + """Classify one PrefEval sample from the four judge answers. + + Follows the reference implementation in amazon-science/PrefEval + (generation_task/get_preference_following_accuracy_generation_task.py): + an unhelpful response is an error on its own, the three violation + buckets require a helpful response, and a hallucinated preference only + counts when the preference was acknowledged. A missing or unrecognized + judge answer is reported as a judge failure instead of being counted as a + personalized response. + """ + answers = { + key: parse_yes_no(evaluation_results.get(key, {}).get("answer")) for key in _JUDGE_KEYS + } + if any(value is None for value in answers.values()): + return JUDGE_FAILURE + + violate = answers["violate_preference"] + acknowledge = answers["acknowledge_preference"] + hallucinate = acknowledge and answers["hallucinate_preference"] + unhelpful = not answers["helpful_response"] + + if unhelpful: + return UNHELPFUL_RESPONSE + if violate and not acknowledge: + return PREFERENCE_UNAWARE_VIOLATION + if violate and hallucinate: + return PREFERENCE_HALLUCINATION_VIOLATION + if violate: + return INCONSISTENCY_VIOLATION + return PERSONALIZED_RESPONSE diff --git a/tests/evaluation/test_pref_classify.py b/tests/evaluation/test_pref_classify.py new file mode 100644 index 000000000..45b885044 --- /dev/null +++ b/tests/evaluation/test_pref_classify.py @@ -0,0 +1,103 @@ +import importlib.util +import itertools + +from pathlib import Path + +import pytest + + +MODULE_PATH = ( + Path(__file__).resolve().parents[2] / "evaluation" / "scripts" / "utils" / "pref_classify.py" +) + + +def _load_module(): + spec = importlib.util.spec_from_file_location("pref_classify", MODULE_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +pref_classify = _load_module() + + +def _judge(violate, acknowledge, hallucinate, helpful): + return { + "violate_preference": {"answer": violate}, + "acknowledge_preference": {"answer": acknowledge}, + "hallucinate_preference": {"answer": hallucinate}, + "helpful_response": {"answer": helpful}, + } + + +def _reference_bucket(violate, acknowledge, hallucinate, helpful): + """The bucket implied by amazon-science/PrefEval analyze_errors().""" + is_ack = "yes" in acknowledge.lower() + is_hal = is_ack and "yes" in hallucinate.lower() + is_vio = "yes" in violate.lower() + is_unhelpful = "no" in helpful.lower() + if is_unhelpful: + return pref_classify.UNHELPFUL_RESPONSE + if is_ack and not is_hal and is_vio: + return pref_classify.INCONSISTENCY_VIOLATION + if is_ack and is_hal and is_vio: + return pref_classify.PREFERENCE_HALLUCINATION_VIOLATION + if not is_ack and is_vio: + return pref_classify.PREFERENCE_UNAWARE_VIOLATION + return pref_classify.PERSONALIZED_RESPONSE + + +@pytest.mark.parametrize("answers", list(itertools.product(["Yes", "No"], repeat=4))) +def test_matches_reference_on_well_formed_answers(answers): + assert pref_classify.classify_error_type(_judge(*answers)) == _reference_bucket(*answers) + + +@pytest.mark.parametrize( + ("answers", "expected"), + [ + (("yes", "no", "No", "yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + (("Yes.", "No", "No", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + ((" YES ", "No", "No", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + (("Yes", "No", "No", "No"), pref_classify.UNHELPFUL_RESPONSE), + (("Yes", "Yes", "Yes", "No"), pref_classify.UNHELPFUL_RESPONSE), + (("Yes", "No", "Yes", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + (("No", "Yes", "Yes", "Yes"), pref_classify.PERSONALIZED_RESPONSE), + ], +) +def test_case_and_precedence(answers, expected): + assert pref_classify.classify_error_type(_judge(*answers)) == expected + + +@pytest.mark.parametrize( + "answers", + [ + ("", "", "", ""), + ("Yes", "No", "No", ""), + ("Maybe", "No", "No", "Yes"), + ("Yes", "No", "No", None), + ], +) +def test_unrecognized_answers_are_judge_failures(answers): + assert pref_classify.classify_error_type(_judge(*answers)) == pref_classify.JUDGE_FAILURE + + +def test_missing_judge_key_is_a_failure(): + partial = {"violate_preference": {"answer": "Yes"}} + assert pref_classify.classify_error_type(partial) == pref_classify.JUDGE_FAILURE + + +@pytest.mark.parametrize( + ("answer", "expected"), + [ + ("Yes", True), + ("yes.", True), + ("No", False), + (" no", False), + ("", None), + (None, None), + ("Unknown", None), + ], +) +def test_parse_yes_no(answer, expected): + assert pref_classify.parse_yes_no(answer) is expected