diff --git a/evaluation/scripts/PrefEval/pref_eval.py b/evaluation/scripts/PrefEval/pref_eval.py index 9da3c9438..0f782fae7 100644 --- a/evaluation/scripts/PrefEval/pref_eval.py +++ b/evaluation/scripts/PrefEval/pref_eval.py @@ -3,6 +3,7 @@ import json import os import re +import sys from collections import Counter from typing import Any @@ -14,6 +15,10 @@ from tqdm.asyncio import tqdm +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from utils.pref_classify import classify_error_type + + load_dotenv() API_KEY = os.getenv("OPENAI_API_KEY") @@ -178,24 +183,6 @@ async def evaluate_helpful_response_async( } -def classify_error_type(evaluation_results: dict[str, Any]) -> str: - violate = evaluation_results["violate_preference"]["answer"] - acknowledge = evaluation_results["acknowledge_preference"]["answer"] - hallucinate = evaluation_results["hallucinate_preference"]["answer"] - helpful = evaluation_results["helpful_response"]["answer"] - - if violate == "Yes" and acknowledge == "No" and helpful == "Yes": - return "Preference-Unaware Violation" - elif violate == "Yes" and acknowledge == "Yes" and hallucinate == "Yes" and helpful == "Yes": - return "Preference Hallucination Violation" - elif violate == "Yes" and acknowledge == "Yes" and hallucinate == "No" and helpful == "Yes": - return "Inconsistency Violation" - elif violate == "No" and helpful == "No": - return "Unhelpful Response" - else: - return "Personalized Response" - - async def process_line(line: str, client: OpenAI, semaphore: asyncio.Semaphore) -> dict[str, Any]: async with semaphore: data = json.loads(line.strip()) diff --git a/evaluation/scripts/utils/pref_classify.py b/evaluation/scripts/utils/pref_classify.py new file mode 100644 index 000000000..7e587de70 --- /dev/null +++ b/evaluation/scripts/utils/pref_classify.py @@ -0,0 +1,71 @@ +"""Error-type classification for PrefEval judge answers. + +Kept free of third-party imports so it can be unit tested without the +evaluation dependency group. +""" + +from typing import Any + + +JUDGE_FAILURE = "Judge Failure" +PERSONALIZED_RESPONSE = "Personalized Response" +UNHELPFUL_RESPONSE = "Unhelpful Response" +PREFERENCE_UNAWARE_VIOLATION = "Preference-Unaware Violation" +PREFERENCE_HALLUCINATION_VIOLATION = "Preference Hallucination Violation" +INCONSISTENCY_VIOLATION = "Inconsistency Violation" + +_JUDGE_KEYS = ( + "violate_preference", + "acknowledge_preference", + "hallucinate_preference", + "helpful_response", +) + + +def parse_yes_no(answer: str | None) -> bool | None: + """Map a judge answer to True (yes) or False (no). + + Returns None when the answer is missing or is neither yes nor no, for + example the empty string that the judge call returns after an API error. + """ + if answer is None: + return None + normalized = answer.strip().lower() + if normalized.startswith("yes"): + return True + if normalized.startswith("no"): + return False + return None + + +def classify_error_type(evaluation_results: dict[str, Any]) -> str: + """Classify one PrefEval sample from the four judge answers. + + Follows the reference implementation in amazon-science/PrefEval + (generation_task/get_preference_following_accuracy_generation_task.py): + an unhelpful response is an error on its own, the three violation + buckets require a helpful response, and a hallucinated preference only + counts when the preference was acknowledged. A missing or unrecognized + judge answer is reported as a judge failure instead of being counted as a + personalized response. + """ + answers = { + key: parse_yes_no(evaluation_results.get(key, {}).get("answer")) for key in _JUDGE_KEYS + } + if any(value is None for value in answers.values()): + return JUDGE_FAILURE + + violate = answers["violate_preference"] + acknowledge = answers["acknowledge_preference"] + hallucinate = acknowledge and answers["hallucinate_preference"] + unhelpful = not answers["helpful_response"] + + if unhelpful: + return UNHELPFUL_RESPONSE + if violate and not acknowledge: + return PREFERENCE_UNAWARE_VIOLATION + if violate and hallucinate: + return PREFERENCE_HALLUCINATION_VIOLATION + if violate: + return INCONSISTENCY_VIOLATION + return PERSONALIZED_RESPONSE diff --git a/tests/evaluation/test_pref_classify.py b/tests/evaluation/test_pref_classify.py new file mode 100644 index 000000000..45b885044 --- /dev/null +++ b/tests/evaluation/test_pref_classify.py @@ -0,0 +1,103 @@ +import importlib.util +import itertools + +from pathlib import Path + +import pytest + + +MODULE_PATH = ( + Path(__file__).resolve().parents[2] / "evaluation" / "scripts" / "utils" / "pref_classify.py" +) + + +def _load_module(): + spec = importlib.util.spec_from_file_location("pref_classify", MODULE_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +pref_classify = _load_module() + + +def _judge(violate, acknowledge, hallucinate, helpful): + return { + "violate_preference": {"answer": violate}, + "acknowledge_preference": {"answer": acknowledge}, + "hallucinate_preference": {"answer": hallucinate}, + "helpful_response": {"answer": helpful}, + } + + +def _reference_bucket(violate, acknowledge, hallucinate, helpful): + """The bucket implied by amazon-science/PrefEval analyze_errors().""" + is_ack = "yes" in acknowledge.lower() + is_hal = is_ack and "yes" in hallucinate.lower() + is_vio = "yes" in violate.lower() + is_unhelpful = "no" in helpful.lower() + if is_unhelpful: + return pref_classify.UNHELPFUL_RESPONSE + if is_ack and not is_hal and is_vio: + return pref_classify.INCONSISTENCY_VIOLATION + if is_ack and is_hal and is_vio: + return pref_classify.PREFERENCE_HALLUCINATION_VIOLATION + if not is_ack and is_vio: + return pref_classify.PREFERENCE_UNAWARE_VIOLATION + return pref_classify.PERSONALIZED_RESPONSE + + +@pytest.mark.parametrize("answers", list(itertools.product(["Yes", "No"], repeat=4))) +def test_matches_reference_on_well_formed_answers(answers): + assert pref_classify.classify_error_type(_judge(*answers)) == _reference_bucket(*answers) + + +@pytest.mark.parametrize( + ("answers", "expected"), + [ + (("yes", "no", "No", "yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + (("Yes.", "No", "No", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + ((" YES ", "No", "No", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + (("Yes", "No", "No", "No"), pref_classify.UNHELPFUL_RESPONSE), + (("Yes", "Yes", "Yes", "No"), pref_classify.UNHELPFUL_RESPONSE), + (("Yes", "No", "Yes", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + (("No", "Yes", "Yes", "Yes"), pref_classify.PERSONALIZED_RESPONSE), + ], +) +def test_case_and_precedence(answers, expected): + assert pref_classify.classify_error_type(_judge(*answers)) == expected + + +@pytest.mark.parametrize( + "answers", + [ + ("", "", "", ""), + ("Yes", "No", "No", ""), + ("Maybe", "No", "No", "Yes"), + ("Yes", "No", "No", None), + ], +) +def test_unrecognized_answers_are_judge_failures(answers): + assert pref_classify.classify_error_type(_judge(*answers)) == pref_classify.JUDGE_FAILURE + + +def test_missing_judge_key_is_a_failure(): + partial = {"violate_preference": {"answer": "Yes"}} + assert pref_classify.classify_error_type(partial) == pref_classify.JUDGE_FAILURE + + +@pytest.mark.parametrize( + ("answer", "expected"), + [ + ("Yes", True), + ("yes.", True), + ("No", False), + (" no", False), + ("", None), + (None, None), + ("Unknown", None), + ], +) +def test_parse_yes_no(answer, expected): + assert pref_classify.parse_yes_no(answer) is expected