From 5dae19741e473408d47717bd63d58381e43ae7ef Mon Sep 17 00:00:00 2001 From: Chen Yufeiyang Date: Sun, 18 Oct 2026 01:49:01 +0800 Subject: [PATCH 1/3] FIX: Use the canonical leading-comment rule when loading classifier CSVs The two local classifier scorers hand-rolled how they skip a dataset's leading '#' comment lines, and neither matched HumanLabeledDataset._read_leading_comment_block: - LocalRefusalClassifierScorer dropped every line starting with '#', including data rows and lines inside quoted cells; the packaged refusal.csv already has one training response truncated by this (4161 vs 4257 characters). - LocalViolenceClassifierScorer skipped only the first line, so a file with several leading '#' lines (or a blank line first) quietly loaded zero rows and then failed with a misleading not-enough-training-data error. Both now use a shared strip_leading_comment_block() helper with the canonical scan: only the leading comment/blank block is removed, everything from the first content line on is data. --- pyrit/common/csv_helper.py | 31 +++++++++++++++++++ .../local_violence_classifier_scorer.py | 6 ++-- .../local_refusal_classifier_scorer.py | 3 +- .../test_local_refusal_classifier_scorer.py | 23 ++++++++++++++ .../test_local_violence_classifier_scorer.py | 25 +++++++++++++++ 5 files changed, 84 insertions(+), 4 deletions(-) diff --git a/pyrit/common/csv_helper.py b/pyrit/common/csv_helper.py index 51633c123b..167c9cf03f 100644 --- a/pyrit/common/csv_helper.py +++ b/pyrit/common/csv_helper.py @@ -2,9 +2,40 @@ # Licensed under the MIT license. import csv +import io from typing import IO, Any +def strip_leading_comment_block(text: str) -> str: + """ + Drop the leading "#" comment and blank lines of CSV text. + + Mirrors the scan in ``HumanLabeledDataset._read_leading_comment_block``: only the + leading block is a comment, so a "#" that starts a data row or sits inside a cell + is preserved as data. Callers that hand the result to ``csv.DictReader`` must use + this instead of dropping every "#" line, which silently deletes data. + + Lines are read the way ``open(..., newline=None)`` reads them; a ``StringIO`` + opened with the same newline setting keeps the block boundary identical to the + canonical scan instead of also splitting on separators such as "\\v". + + Args: + text (str): The full CSV text. + + Returns: + str: The text from the first non-comment, non-blank line onward. + """ + stream = io.StringIO(text, newline=None) + remaining: list[str] = [] + for line in stream: + stripped = line.strip() + if stripped and not stripped.startswith("#"): + remaining.append(line) + break + remaining.append(stream.read()) + return "".join(remaining) + + def read_csv(file: IO[Any]) -> list[dict[str, str]]: """ Read a CSV file and return its rows as dictionaries. diff --git a/pyrit/score/float_scale/local_violence_classifier_scorer.py b/pyrit/score/float_scale/local_violence_classifier_scorer.py index f211eee2f5..dbd2d790d8 100644 --- a/pyrit/score/float_scale/local_violence_classifier_scorer.py +++ b/pyrit/score/float_scale/local_violence_classifier_scorer.py @@ -16,6 +16,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar +from pyrit.common.csv_helper import strip_leading_comment_block from pyrit.common.path import SCORER_EVALS_HARM_PATH from pyrit.models import ComponentIdentifier, MessagePiece, Score, ScoreStatus from pyrit.score.float_scale.float_scale_scorer import MessageFloatScaleScorer @@ -307,9 +308,8 @@ def _load_training_rows() -> tuple[list[str], list[int]]: "If the dataset was updated deliberately, re-validate the scorer and update " "LocalViolenceClassifierScorer's pinned hashes rather than scoring with an unchecked model." ) - lines = raw.decode("utf-8").splitlines() - start = 1 if lines and lines[0].startswith("#") else 0 - for row in csv.DictReader(io.StringIO("\n".join(lines[start:]))): + body = strip_leading_comment_block(raw.decode("utf-8")) + for row in csv.DictReader(io.StringIO(body)): raw_scores = [row.get(key) for key in ("human_score", "human_score_1", "human_score_2", "human_score_3")] scores = [float(value) for value in raw_scores if value not in (None, "")] response = (row.get("assistant_response") or "").strip() diff --git a/pyrit/score/true_false/local_refusal_classifier_scorer.py b/pyrit/score/true_false/local_refusal_classifier_scorer.py index edc7a46d90..2a7879f27b 100644 --- a/pyrit/score/true_false/local_refusal_classifier_scorer.py +++ b/pyrit/score/true_false/local_refusal_classifier_scorer.py @@ -16,6 +16,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar +from pyrit.common.csv_helper import strip_leading_comment_block from pyrit.common.path import SCORER_EVALS_REFUSAL_SCORER_PATH from pyrit.models import ComponentIdentifier, Message, MessagePiece, Score, ScoreStatus from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -366,7 +367,7 @@ def _load_training_rows() -> tuple[list[tuple[str, str]], list[int]]: f"({expected_hash}, found {actual_hash}). Re-validate the scorer and update the pin." ) text = raw.decode("utf-8", errors="replace") - body = "".join(line for line in text.splitlines(keepends=True) if not line.startswith("#")) + body = strip_leading_comment_block(text) for row in csv.DictReader(io.StringIO(body)): if row.get("data_type", "text") != "text" or row["human_score"] not in ("0", "1"): continue diff --git a/tests/unit/score/test_local_refusal_classifier_scorer.py b/tests/unit/score/test_local_refusal_classifier_scorer.py index 900bae4f9c..86e5d3294f 100644 --- a/tests/unit/score/test_local_refusal_classifier_scorer.py +++ b/tests/unit/score/test_local_refusal_classifier_scorer.py @@ -71,6 +71,29 @@ def test_training_rejects_changed_data(tmp_path: Path) -> None: assert _load_training_rows() == ([(objective, response)] * 2, [1, 1]) +def test_load_training_rows_keeps_hash_lines_outside_the_leading_comments(tmp_path: Path) -> None: + """Only leading '#' lines are comments; a '#' starting a row or inside a cell is data (#2974).""" + bodies = { + "refusal.csv": ( + "# dataset_version=1.0\n" + "objective,assistant_response,human_score,data_type\n" + "# Hash-prefixed objective,refusal response,1,text\n" + 'plain objective,"line one\n### in-cell heading\nline three",0,text\n' + ), + "refusal_extra.csv": "# dataset_version=1.0\nobjective,assistant_response,human_score,data_type\n", + } + pins = {} + for name, body in bodies.items(): + data = body.encode() + (tmp_path / name).write_bytes(data) + pins[name] = hashlib.sha256(data).hexdigest() + with patch.object(module, "_REFUSAL_EVALS_PATH", tmp_path), patch.object(module, "_TRAINING_DATASETS", pins): + pairs, labels = _load_training_rows() + assert [objective for objective, _ in pairs] == ["# Hash-prefixed objective", "plain objective"] + assert "### in-cell heading" in pairs[1][1] + assert labels == [1, 0] + + @requires_torch def test_training_is_deterministic_and_preserves_rng() -> None: import torch diff --git a/tests/unit/score/test_local_violence_classifier_scorer.py b/tests/unit/score/test_local_violence_classifier_scorer.py index 3c0cef2cc5..49c5aa5fb9 100644 --- a/tests/unit/score/test_local_violence_classifier_scorer.py +++ b/tests/unit/score/test_local_violence_classifier_scorer.py @@ -110,6 +110,31 @@ def test_load_training_rows_rejects_changed_dataset(tmp_path: Path) -> None: assert texts == [] and labels == [] +def test_load_training_rows_skips_the_whole_leading_comment_block(tmp_path: Path) -> None: + """Every leading '#' line is a comment, not just the first one.""" + harm_path = tmp_path / "harm" + harm_path.mkdir() + header = "harm_category,objective,assistant_response,human_score_1,data_type\n" + bodies = { + "violence.csv": ( + "# dataset_version=1.0\n" + "# collected by the eval team\n" + f"{header}" + "violence,sample objective,a harmless reply,0,text\n" + ), + "violence_multi_score.csv": "# dataset_version=1.0\n" + header, + } + pins = {} + for file_name, body in bodies.items(): + data = body.encode() + (harm_path / file_name).write_bytes(data) + pins[file_name] = hashlib.sha256(data).hexdigest() + with patch.object(module, "_HARM_EVALS_PATH", harm_path), patch.object(module, "_TRAINING_DATASETS", pins): + texts, labels = _load_training_rows() + assert texts == ["PROMPT: sample objective\nRESPONSE: a harmless reply"] + assert labels == [0] + + @requires_torch def test_train_head_is_deterministic_and_separates() -> None: embeddings, labels = _separable_training_data() From 1ce138b9e5745365649632155fa702cf961ee147 Mon Sep 17 00:00:00 2001 From: Chen Yufeiyang Date: Tue, 20 Oct 2026 09:51:49 +0800 Subject: [PATCH 2/3] FIX: Preserve line endings when scanning classifier CSV comments --- pyrit/common/csv_helper.py | 9 ++-- .../local_violence_classifier_scorer.py | 2 +- .../local_refusal_classifier_scorer.py | 2 +- tests/unit/common/test_csv_helper.py | 18 +++++++- .../test_local_refusal_classifier_scorer.py | 42 +++++++++++++++++++ .../test_local_violence_classifier_scorer.py | 20 +++++++++ 6 files changed, 86 insertions(+), 7 deletions(-) diff --git a/pyrit/common/csv_helper.py b/pyrit/common/csv_helper.py index 167c9cf03f..ed30ea2683 100644 --- a/pyrit/common/csv_helper.py +++ b/pyrit/common/csv_helper.py @@ -15,9 +15,10 @@ def strip_leading_comment_block(text: str) -> str: is preserved as data. Callers that hand the result to ``csv.DictReader`` must use this instead of dropping every "#" line, which silently deletes data. - Lines are read the way ``open(..., newline=None)`` reads them; a ``StringIO`` - opened with the same newline setting keeps the block boundary identical to the - canonical scan instead of also splitting on separators such as "\\v". + Universal-newline mode still recognizes CR, LF and CRLF line boundaries, but + ``newline=""`` returns each ending untranslated. The returned text therefore keeps + the payload byte-for-byte, including CR and CRLF inside quoted cells, instead of + letting the scan rewrite what the CSV reader later parses. Args: text (str): The full CSV text. @@ -25,7 +26,7 @@ def strip_leading_comment_block(text: str) -> str: Returns: str: The text from the first non-comment, non-blank line onward. """ - stream = io.StringIO(text, newline=None) + stream = io.StringIO(text, newline="") remaining: list[str] = [] for line in stream: stripped = line.strip() diff --git a/pyrit/score/float_scale/local_violence_classifier_scorer.py b/pyrit/score/float_scale/local_violence_classifier_scorer.py index dbd2d790d8..b0bb721da6 100644 --- a/pyrit/score/float_scale/local_violence_classifier_scorer.py +++ b/pyrit/score/float_scale/local_violence_classifier_scorer.py @@ -309,7 +309,7 @@ def _load_training_rows() -> tuple[list[str], list[int]]: "LocalViolenceClassifierScorer's pinned hashes rather than scoring with an unchecked model." ) body = strip_leading_comment_block(raw.decode("utf-8")) - for row in csv.DictReader(io.StringIO(body)): + for row in csv.DictReader(io.StringIO(body, newline="")): raw_scores = [row.get(key) for key in ("human_score", "human_score_1", "human_score_2", "human_score_3")] scores = [float(value) for value in raw_scores if value not in (None, "")] response = (row.get("assistant_response") or "").strip() diff --git a/pyrit/score/true_false/local_refusal_classifier_scorer.py b/pyrit/score/true_false/local_refusal_classifier_scorer.py index 2a7879f27b..4d900b81dc 100644 --- a/pyrit/score/true_false/local_refusal_classifier_scorer.py +++ b/pyrit/score/true_false/local_refusal_classifier_scorer.py @@ -368,7 +368,7 @@ def _load_training_rows() -> tuple[list[tuple[str, str]], list[int]]: ) text = raw.decode("utf-8", errors="replace") body = strip_leading_comment_block(text) - for row in csv.DictReader(io.StringIO(body)): + for row in csv.DictReader(io.StringIO(body, newline="")): if row.get("data_type", "text") != "text" or row["human_score"] not in ("0", "1"): continue pairs.append((row["objective"], row["assistant_response"])) diff --git a/tests/unit/common/test_csv_helper.py b/tests/unit/common/test_csv_helper.py index b210297a7f..5717e0bf93 100644 --- a/tests/unit/common/test_csv_helper.py +++ b/tests/unit/common/test_csv_helper.py @@ -1,10 +1,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import csv import io from io import StringIO -from pyrit.common.csv_helper import read_csv, write_csv +from pyrit.common.csv_helper import read_csv, strip_leading_comment_block, write_csv def test_read_csv_returns_list_of_dicts(): @@ -57,3 +58,18 @@ def test_write_csv_writes_header_and_rows(): lines = file.getvalue().strip().splitlines() assert lines[0] == "name,role" assert lines[1] == "alice,admin" + + +def test_strip_leading_comment_block_preserves_crlf_and_cr_inside_quoted_cells(): + text = '# dataset_version=1.0\r\nobjective,assistant_response\r\nplain,"line one\r\n### heading\rline three"\r\n' + body = strip_leading_comment_block(text) + assert body == 'objective,assistant_response\r\nplain,"line one\r\n### heading\rline three"\r\n' + + +def test_strip_leading_comment_block_handles_cr_only_record_separators(): + text = "# dataset_version=1.0\robjective,assistant_response\rplain,refusal response\r" + body = strip_leading_comment_block(text) + assert body == "objective,assistant_response\rplain,refusal response\r" + assert list(csv.DictReader(StringIO(body, newline=""))) == [ + {"objective": "plain", "assistant_response": "refusal response"} + ] diff --git a/tests/unit/score/test_local_refusal_classifier_scorer.py b/tests/unit/score/test_local_refusal_classifier_scorer.py index 86e5d3294f..18f4e0da5f 100644 --- a/tests/unit/score/test_local_refusal_classifier_scorer.py +++ b/tests/unit/score/test_local_refusal_classifier_scorer.py @@ -94,6 +94,48 @@ def test_load_training_rows_keeps_hash_lines_outside_the_leading_comments(tmp_pa assert labels == [1, 0] +def test_load_training_rows_preserves_crlf_and_cr_inside_quoted_responses(tmp_path: Path) -> None: + """The leading-comment scan must not rewrite line endings inside quoted data.""" + response = "line one\r\n### heading\rline three" + bodies = { + "refusal.csv": ( + "# dataset_version=1.0\r\n" + "objective,assistant_response,human_score,data_type\r\n" + f'plain objective,"{response}",0,text\r\n' + ), + "refusal_extra.csv": "# dataset_version=1.0\r\nobjective,assistant_response,human_score,data_type\r\n", + } + pins = {} + for name, body in bodies.items(): + data = body.encode() + (tmp_path / name).write_bytes(data) + pins[name] = hashlib.sha256(data).hexdigest() + with patch.object(module, "_REFUSAL_EVALS_PATH", tmp_path), patch.object(module, "_TRAINING_DATASETS", pins): + pairs, labels = _load_training_rows() + assert pairs == [("plain objective", response)] + assert labels == [0] + + +def test_load_training_rows_accepts_cr_only_record_separators(tmp_path: Path) -> None: + """CR-only files still split into records and keep their payload intact.""" + bodies = { + "refusal.csv": ( + "# dataset_version=1.0\robjective,assistant_response,human_score,data_type\r" + "plain objective,refusal response,1,text\r" + ), + "refusal_extra.csv": "# dataset_version=1.0\robjective,assistant_response,human_score,data_type\r", + } + pins = {} + for name, body in bodies.items(): + data = body.encode() + (tmp_path / name).write_bytes(data) + pins[name] = hashlib.sha256(data).hexdigest() + with patch.object(module, "_REFUSAL_EVALS_PATH", tmp_path), patch.object(module, "_TRAINING_DATASETS", pins): + pairs, labels = _load_training_rows() + assert pairs == [("plain objective", "refusal response")] + assert labels == [1] + + @requires_torch def test_training_is_deterministic_and_preserves_rng() -> None: import torch diff --git a/tests/unit/score/test_local_violence_classifier_scorer.py b/tests/unit/score/test_local_violence_classifier_scorer.py index 49c5aa5fb9..25d696f1e6 100644 --- a/tests/unit/score/test_local_violence_classifier_scorer.py +++ b/tests/unit/score/test_local_violence_classifier_scorer.py @@ -135,6 +135,26 @@ def test_load_training_rows_skips_the_whole_leading_comment_block(tmp_path: Path assert labels == [0] +def test_load_training_rows_accepts_cr_only_record_separators(tmp_path: Path) -> None: + """CR-only files still split into records and keep their payload intact.""" + harm_path = tmp_path / "harm" + harm_path.mkdir() + header = "harm_category,objective,assistant_response,human_score_1,data_type\r" + bodies = { + "violence.csv": "# dataset_version=1.0\r" + header + "violence,sample objective,a harmless reply,0,text\r", + "violence_multi_score.csv": "# dataset_version=1.0\r" + header, + } + pins = {} + for file_name, body in bodies.items(): + data = body.encode() + (harm_path / file_name).write_bytes(data) + pins[file_name] = hashlib.sha256(data).hexdigest() + with patch.object(module, "_HARM_EVALS_PATH", harm_path), patch.object(module, "_TRAINING_DATASETS", pins): + texts, labels = _load_training_rows() + assert texts == ["PROMPT: sample objective\nRESPONSE: a harmless reply"] + assert labels == [0] + + @requires_torch def test_train_head_is_deterministic_and_separates() -> None: embeddings, labels = _separable_training_data() From 94bbec71d920ea7bbf8da1a5f84e5cc39f6a2c0d Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Mon, 19 Oct 2026 18:58:44 -0700 Subject: [PATCH 3/3] FIX: Preserve newline characters in classifier CSV cells Keep universal newline recognition without translating CR or CRLF in the leading-comment scan and both classifier CSV readers. Cover mixed quoted-cell and record separators, blank/comment prefixes, and non-newline separators. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c89266be-249b-4382-bfd3-0a3b3c449ae3 --- pyrit/common/csv_helper.py | 8 +++---- .../local_violence_classifier_scorer.py | 2 +- .../local_refusal_classifier_scorer.py | 2 +- tests/unit/common/test_csv_helper.py | 23 ++++++++++++++++++- .../test_local_refusal_classifier_scorer.py | 21 ++++++++++------- .../test_local_violence_classifier_scorer.py | 20 ++++++++++------ 6 files changed, 54 insertions(+), 22 deletions(-) diff --git a/pyrit/common/csv_helper.py b/pyrit/common/csv_helper.py index cd89758190..452e666a87 100644 --- a/pyrit/common/csv_helper.py +++ b/pyrit/common/csv_helper.py @@ -15,9 +15,9 @@ def strip_leading_comment_block(text: str) -> str: is preserved as data. Callers that hand the result to ``csv.DictReader`` must use this instead of dropping every "#" line, which silently deletes data. - Lines are read the way ``open(..., newline=None)`` reads them; a ``StringIO`` - opened with the same newline setting keeps the block boundary identical to the - canonical scan instead of also splitting on separators such as "\\v". + Universal newline recognition keeps the block boundary identical to the canonical + scan without splitting on separators such as "\\v". Newline characters are retained, + including CR and CRLF inside quoted cells. CSV readers must also preserve newlines. Args: text (str): The full CSV text. @@ -25,7 +25,7 @@ def strip_leading_comment_block(text: str) -> str: Returns: str: The text from the first non-comment, non-blank line onward. """ - stream = io.StringIO(text, newline=None) + stream = io.StringIO(text, newline="") remaining: list[str] = [] for line in stream: stripped = line.strip() diff --git a/pyrit/score/float_scale/local_violence_classifier_scorer.py b/pyrit/score/float_scale/local_violence_classifier_scorer.py index dbd2d790d8..b0bb721da6 100644 --- a/pyrit/score/float_scale/local_violence_classifier_scorer.py +++ b/pyrit/score/float_scale/local_violence_classifier_scorer.py @@ -309,7 +309,7 @@ def _load_training_rows() -> tuple[list[str], list[int]]: "LocalViolenceClassifierScorer's pinned hashes rather than scoring with an unchecked model." ) body = strip_leading_comment_block(raw.decode("utf-8")) - for row in csv.DictReader(io.StringIO(body)): + for row in csv.DictReader(io.StringIO(body, newline="")): raw_scores = [row.get(key) for key in ("human_score", "human_score_1", "human_score_2", "human_score_3")] scores = [float(value) for value in raw_scores if value not in (None, "")] response = (row.get("assistant_response") or "").strip() diff --git a/pyrit/score/true_false/local_refusal_classifier_scorer.py b/pyrit/score/true_false/local_refusal_classifier_scorer.py index 2a7879f27b..4d900b81dc 100644 --- a/pyrit/score/true_false/local_refusal_classifier_scorer.py +++ b/pyrit/score/true_false/local_refusal_classifier_scorer.py @@ -368,7 +368,7 @@ def _load_training_rows() -> tuple[list[tuple[str, str]], list[int]]: ) text = raw.decode("utf-8", errors="replace") body = strip_leading_comment_block(text) - for row in csv.DictReader(io.StringIO(body)): + for row in csv.DictReader(io.StringIO(body, newline="")): if row.get("data_type", "text") != "text" or row["human_score"] not in ("0", "1"): continue pairs.append((row["objective"], row["assistant_response"])) diff --git a/tests/unit/common/test_csv_helper.py b/tests/unit/common/test_csv_helper.py index 7424641d56..5dba0b7d9f 100644 --- a/tests/unit/common/test_csv_helper.py +++ b/tests/unit/common/test_csv_helper.py @@ -4,7 +4,28 @@ import io from io import StringIO -from pyrit.common.csv_helper import read_csv, write_csv +import pytest + +from pyrit.common.csv_helper import read_csv, strip_leading_comment_block, write_csv + + +@pytest.mark.parametrize("line_ending", ["\n", "\r\n", "\r"]) +@pytest.mark.parametrize("leading_block", ["", "\n", " \t\n", "# metadata\n", "\n # first\n\t\n# second\n"]) +def test_strip_leading_comment_block_preserves_body(*, line_ending: str, leading_block: str) -> None: + body = f'objective,assistant_response{line_ending}# objective,"line one\r\n### heading\rline three"{line_ending}' + text = leading_block.replace("\n", line_ending) + body + assert strip_leading_comment_block(text) == body + + +@pytest.mark.parametrize("separator", ["\v", "\f", "\x85", "\u2028", "\u2029"]) +def test_strip_leading_comment_block_preserves_non_newline_separators(separator: str) -> None: + body = f"value{separator}# header\none\n" + assert strip_leading_comment_block("# metadata\n" + body) == body + + +@pytest.mark.parametrize("text", ["", "\n\t\n", "# comment", "\r\n# comment\r\n \r"]) +def test_strip_leading_comment_block_without_content(text: str) -> None: + assert strip_leading_comment_block(text) == "" def test_read_csv_returns_list_of_dicts(): diff --git a/tests/unit/score/test_local_refusal_classifier_scorer.py b/tests/unit/score/test_local_refusal_classifier_scorer.py index 86e5d3294f..f5ab5f58ab 100644 --- a/tests/unit/score/test_local_refusal_classifier_scorer.py +++ b/tests/unit/score/test_local_refusal_classifier_scorer.py @@ -71,16 +71,22 @@ def test_training_rejects_changed_data(tmp_path: Path) -> None: assert _load_training_rows() == ([(objective, response)] * 2, [1, 1]) -def test_load_training_rows_keeps_hash_lines_outside_the_leading_comments(tmp_path: Path) -> None: +@pytest.mark.parametrize("record_separator", ["\n", "\r\n", "\r"]) +@pytest.mark.parametrize("cell_separator", ["\n", "\r\n", "\r"]) +def test_load_training_rows_keeps_hash_lines_outside_the_leading_comments( + *, tmp_path: Path, record_separator: str, cell_separator: str +) -> None: """Only leading '#' lines are comments; a '#' starting a row or inside a cell is data (#2974).""" + header = f"objective,assistant_response,human_score,data_type{record_separator}" + response = f"line one{cell_separator}### in-cell heading{cell_separator}line three" bodies = { "refusal.csv": ( - "# dataset_version=1.0\n" - "objective,assistant_response,human_score,data_type\n" - "# Hash-prefixed objective,refusal response,1,text\n" - 'plain objective,"line one\n### in-cell heading\nline three",0,text\n' + f"# dataset_version=1.0{record_separator}" + f"{header}" + f"# Hash-prefixed objective,refusal response,1,text{record_separator}" + f'plain objective,"{response}",0,text{record_separator}' ), - "refusal_extra.csv": "# dataset_version=1.0\nobjective,assistant_response,human_score,data_type\n", + "refusal_extra.csv": f"# dataset_version=1.0{record_separator}{header}", } pins = {} for name, body in bodies.items(): @@ -89,8 +95,7 @@ def test_load_training_rows_keeps_hash_lines_outside_the_leading_comments(tmp_pa pins[name] = hashlib.sha256(data).hexdigest() with patch.object(module, "_REFUSAL_EVALS_PATH", tmp_path), patch.object(module, "_TRAINING_DATASETS", pins): pairs, labels = _load_training_rows() - assert [objective for objective, _ in pairs] == ["# Hash-prefixed objective", "plain objective"] - assert "### in-cell heading" in pairs[1][1] + assert pairs == [("# Hash-prefixed objective", "refusal response"), ("plain objective", response)] assert labels == [1, 0] diff --git a/tests/unit/score/test_local_violence_classifier_scorer.py b/tests/unit/score/test_local_violence_classifier_scorer.py index 49c5aa5fb9..0721da303b 100644 --- a/tests/unit/score/test_local_violence_classifier_scorer.py +++ b/tests/unit/score/test_local_violence_classifier_scorer.py @@ -110,19 +110,25 @@ def test_load_training_rows_rejects_changed_dataset(tmp_path: Path) -> None: assert texts == [] and labels == [] -def test_load_training_rows_skips_the_whole_leading_comment_block(tmp_path: Path) -> None: +@pytest.mark.parametrize("record_separator", ["\n", "\r\n", "\r"]) +@pytest.mark.parametrize("cell_separator", ["\n", "\r\n", "\r"]) +def test_load_training_rows_skips_the_whole_leading_comment_block( + *, tmp_path: Path, record_separator: str, cell_separator: str +) -> None: """Every leading '#' line is a comment, not just the first one.""" harm_path = tmp_path / "harm" harm_path.mkdir() - header = "harm_category,objective,assistant_response,human_score_1,data_type\n" + header = f"harm_category,objective,assistant_response,human_score_1,data_type{record_separator}" + response = f"line one{cell_separator}### in-cell heading{cell_separator}line three" bodies = { "violence.csv": ( - "# dataset_version=1.0\n" - "# collected by the eval team\n" + f"{record_separator}" + f"# dataset_version=1.0{record_separator}" + f"# collected by the eval team{record_separator}" f"{header}" - "violence,sample objective,a harmless reply,0,text\n" + f'violence,# sample objective,"{response}",0,text{record_separator}' ), - "violence_multi_score.csv": "# dataset_version=1.0\n" + header, + "violence_multi_score.csv": f"# dataset_version=1.0{record_separator}{header}", } pins = {} for file_name, body in bodies.items(): @@ -131,7 +137,7 @@ def test_load_training_rows_skips_the_whole_leading_comment_block(tmp_path: Path pins[file_name] = hashlib.sha256(data).hexdigest() with patch.object(module, "_HARM_EVALS_PATH", harm_path), patch.object(module, "_TRAINING_DATASETS", pins): texts, labels = _load_training_rows() - assert texts == ["PROMPT: sample objective\nRESPONSE: a harmless reply"] + assert texts == [f"PROMPT: # sample objective\nRESPONSE: {response}"] assert labels == [0]