fix(eval): classify PrefEval judge answers like the reference benchmark

evaluation/scripts/PrefEval/pref_eval.py

@@ -3,6 +3,7 @@
 import json
 import os
 import re
+import sys
 
 from collections import Counter
 from typing import Any
@@ -14,6 +15,10 @@
 from tqdm.asyncio import tqdm
 
 
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from utils.pref_classify import classify_error_type
+
+
 load_dotenv()
 
 API_KEY = os.getenv("OPENAI_API_KEY")
@@ -178,24 +183,6 @@ async def evaluate_helpful_response_async(
     }
 
 
-def classify_error_type(evaluation_results: dict[str, Any]) -> str:
-    violate = evaluation_results["violate_preference"]["answer"]
-    acknowledge = evaluation_results["acknowledge_preference"]["answer"]
-    hallucinate = evaluation_results["hallucinate_preference"]["answer"]
-    helpful = evaluation_results["helpful_response"]["answer"]
-
-    if violate == "Yes" and acknowledge == "No" and helpful == "Yes":
-        return "Preference-Unaware Violation"
-    elif violate == "Yes" and acknowledge == "Yes" and hallucinate == "Yes" and helpful == "Yes":
-        return "Preference Hallucination Violation"
-    elif violate == "Yes" and acknowledge == "Yes" and hallucinate == "No" and helpful == "Yes":
-        return "Inconsistency Violation"
-    elif violate == "No" and helpful == "No":
-        return "Unhelpful Response"
-    else:
-        return "Personalized Response"
-
-
 async def process_line(line: str, client: OpenAI, semaphore: asyncio.Semaphore) -> dict[str, Any]:
     async with semaphore:
         data = json.loads(line.strip())

evaluation/scripts/utils/pref_classify.py

@@ -0,0 +1,71 @@
+"""Error-type classification for PrefEval judge answers.
+
+Kept free of third-party imports so it can be unit tested without the
+evaluation dependency group.
+"""
+
+from typing import Any
+
+
+JUDGE_FAILURE = "Judge Failure"
+PERSONALIZED_RESPONSE = "Personalized Response"
+UNHELPFUL_RESPONSE = "Unhelpful Response"
+PREFERENCE_UNAWARE_VIOLATION = "Preference-Unaware Violation"
+PREFERENCE_HALLUCINATION_VIOLATION = "Preference Hallucination Violation"
+INCONSISTENCY_VIOLATION = "Inconsistency Violation"
+
+_JUDGE_KEYS = (
+    "violate_preference",
+    "acknowledge_preference",
+    "hallucinate_preference",
+    "helpful_response",
+)
+
+
+def parse_yes_no(answer: str | None) -> bool | None:
+    """Map a judge answer to True (yes) or False (no).
+
+    Returns None when the answer is missing or is neither yes nor no, for
+    example the empty string that the judge call returns after an API error.
+    """
+    if answer is None:
+        return None
+    normalized = answer.strip().lower()
+    if normalized.startswith("yes"):
+        return True
+    if normalized.startswith("no"):
+        return False
+    return None
+
+
+def classify_error_type(evaluation_results: dict[str, Any]) -> str:
+    """Classify one PrefEval sample from the four judge answers.
+
+    Follows the reference implementation in amazon-science/PrefEval
+    (generation_task/get_preference_following_accuracy_generation_task.py):
+    an unhelpful response is an error on its own, the three violation
+    buckets require a helpful response, and a hallucinated preference only
+    counts when the preference was acknowledged. A missing or unrecognized
+    judge answer is reported as a judge failure instead of being counted as a
+    personalized response.
+    """
+    answers = {
+        key: parse_yes_no(evaluation_results.get(key, {}).get("answer")) for key in _JUDGE_KEYS
+    }
+    if any(value is None for value in answers.values()):
+        return JUDGE_FAILURE
+
+    violate = answers["violate_preference"]
+    acknowledge = answers["acknowledge_preference"]
+    hallucinate = acknowledge and answers["hallucinate_preference"]
+    unhelpful = not answers["helpful_response"]
+
+    if unhelpful:
+        return UNHELPFUL_RESPONSE
+    if violate and not acknowledge:
+        return PREFERENCE_UNAWARE_VIOLATION
+    if violate and hallucinate:
+        return PREFERENCE_HALLUCINATION_VIOLATION
+    if violate:
+        return INCONSISTENCY_VIOLATION
+    return PERSONALIZED_RESPONSE

tests/evaluation/test_pref_classify.py

@@ -0,0 +1,103 @@
+import importlib.util
+import itertools
+
+from pathlib import Path
+
+import pytest
+
+
+MODULE_PATH = (
+    Path(__file__).resolve().parents[2] / "evaluation" / "scripts" / "utils" / "pref_classify.py"
+)
+
+
+def _load_module():
+    spec = importlib.util.spec_from_file_location("pref_classify", MODULE_PATH)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+pref_classify = _load_module()
+
+
+def _judge(violate, acknowledge, hallucinate, helpful):
+    return {
+        "violate_preference": {"answer": violate},
+        "acknowledge_preference": {"answer": acknowledge},
+        "hallucinate_preference": {"answer": hallucinate},
+        "helpful_response": {"answer": helpful},
+    }
+
+
+def _reference_bucket(violate, acknowledge, hallucinate, helpful):
+    """The bucket implied by amazon-science/PrefEval analyze_errors()."""
+    is_ack = "yes" in acknowledge.lower()
+    is_hal = is_ack and "yes" in hallucinate.lower()
+    is_vio = "yes" in violate.lower()
+    is_unhelpful = "no" in helpful.lower()
+    if is_unhelpful:
+        return pref_classify.UNHELPFUL_RESPONSE
+    if is_ack and not is_hal and is_vio:
+        return pref_classify.INCONSISTENCY_VIOLATION
+    if is_ack and is_hal and is_vio:
+        return pref_classify.PREFERENCE_HALLUCINATION_VIOLATION
+    if not is_ack and is_vio:
+        return pref_classify.PREFERENCE_UNAWARE_VIOLATION
+    return pref_classify.PERSONALIZED_RESPONSE
+
+
+@pytest.mark.parametrize("answers", list(itertools.product(["Yes", "No"], repeat=4)))
+def test_matches_reference_on_well_formed_answers(answers):
+    assert pref_classify.classify_error_type(_judge(*answers)) == _reference_bucket(*answers)
+
+
+@pytest.mark.parametrize(
+    ("answers", "expected"),
+    [
+        (("yes", "no", "No", "yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION),
+        (("Yes.", "No", "No", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION),
+        ((" YES ", "No", "No", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION),
+        (("Yes", "No", "No", "No"), pref_classify.UNHELPFUL_RESPONSE),
+        (("Yes", "Yes", "Yes", "No"), pref_classify.UNHELPFUL_RESPONSE),
+        (("Yes", "No", "Yes", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION),
+        (("No", "Yes", "Yes", "Yes"), pref_classify.PERSONALIZED_RESPONSE),
+    ],
+)
+def test_case_and_precedence(answers, expected):
+    assert pref_classify.classify_error_type(_judge(*answers)) == expected
+
+
+@pytest.mark.parametrize(
+    "answers",
+    [
+        ("", "", "", ""),
+        ("Yes", "No", "No", ""),
+        ("Maybe", "No", "No", "Yes"),
+        ("Yes", "No", "No", None),
+    ],
+)
+def test_unrecognized_answers_are_judge_failures(answers):
+    assert pref_classify.classify_error_type(_judge(*answers)) == pref_classify.JUDGE_FAILURE
+
+
+def test_missing_judge_key_is_a_failure():
+    partial = {"violate_preference": {"answer": "Yes"}}
+    assert pref_classify.classify_error_type(partial) == pref_classify.JUDGE_FAILURE
+
+
+@pytest.mark.parametrize(
+    ("answer", "expected"),
+    [
+        ("Yes", True),
+        ("yes.", True),
+        ("No", False),
+        (" no", False),
+        ("", None),
+        (None, None),
+        ("Unknown", None),
+    ],
+)
+def test_parse_yes_no(answer, expected):
+    assert pref_classify.parse_yes_no(answer) is expected