diff --git a/pyrit/common/csv_helper.py b/pyrit/common/csv_helper.py index 698718a453..452e666a87 100644 --- a/pyrit/common/csv_helper.py +++ b/pyrit/common/csv_helper.py @@ -2,9 +2,40 @@ # Licensed under the MIT license. import csv +import io from typing import IO, Any +def strip_leading_comment_block(text: str) -> str: + """ + Drop the leading "#" comment and blank lines of CSV text. + + Mirrors the scan in ``HumanLabeledDataset._read_leading_comment_block``: only the + leading block is a comment, so a "#" that starts a data row or sits inside a cell + is preserved as data. Callers that hand the result to ``csv.DictReader`` must use + this instead of dropping every "#" line, which silently deletes data. + + Universal newline recognition keeps the block boundary identical to the canonical + scan without splitting on separators such as "\\v". Newline characters are retained, + including CR and CRLF inside quoted cells. CSV readers must also preserve newlines. + + Args: + text (str): The full CSV text. + + Returns: + str: The text from the first non-comment, non-blank line onward. + """ + stream = io.StringIO(text, newline="") + remaining: list[str] = [] + for line in stream: + stripped = line.strip() + if stripped and not stripped.startswith("#"): + remaining.append(line) + break + remaining.append(stream.read()) + return "".join(remaining) + + def read_csv(file: IO[Any]) -> list[dict[str, str]]: """ Read a CSV file and return its rows as dictionaries. diff --git a/pyrit/score/float_scale/local_violence_classifier_scorer.py b/pyrit/score/float_scale/local_violence_classifier_scorer.py index f211eee2f5..b0bb721da6 100644 --- a/pyrit/score/float_scale/local_violence_classifier_scorer.py +++ b/pyrit/score/float_scale/local_violence_classifier_scorer.py @@ -16,6 +16,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar +from pyrit.common.csv_helper import strip_leading_comment_block from pyrit.common.path import SCORER_EVALS_HARM_PATH from pyrit.models import ComponentIdentifier, MessagePiece, Score, ScoreStatus from pyrit.score.float_scale.float_scale_scorer import MessageFloatScaleScorer @@ -307,9 +308,8 @@ def _load_training_rows() -> tuple[list[str], list[int]]: "If the dataset was updated deliberately, re-validate the scorer and update " "LocalViolenceClassifierScorer's pinned hashes rather than scoring with an unchecked model." ) - lines = raw.decode("utf-8").splitlines() - start = 1 if lines and lines[0].startswith("#") else 0 - for row in csv.DictReader(io.StringIO("\n".join(lines[start:]))): + body = strip_leading_comment_block(raw.decode("utf-8")) + for row in csv.DictReader(io.StringIO(body, newline="")): raw_scores = [row.get(key) for key in ("human_score", "human_score_1", "human_score_2", "human_score_3")] scores = [float(value) for value in raw_scores if value not in (None, "")] response = (row.get("assistant_response") or "").strip() diff --git a/pyrit/score/true_false/local_refusal_classifier_scorer.py b/pyrit/score/true_false/local_refusal_classifier_scorer.py index edc7a46d90..4d900b81dc 100644 --- a/pyrit/score/true_false/local_refusal_classifier_scorer.py +++ b/pyrit/score/true_false/local_refusal_classifier_scorer.py @@ -16,6 +16,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar +from pyrit.common.csv_helper import strip_leading_comment_block from pyrit.common.path import SCORER_EVALS_REFUSAL_SCORER_PATH from pyrit.models import ComponentIdentifier, Message, MessagePiece, Score, ScoreStatus from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -366,8 +367,8 @@ def _load_training_rows() -> tuple[list[tuple[str, str]], list[int]]: f"({expected_hash}, found {actual_hash}). Re-validate the scorer and update the pin." ) text = raw.decode("utf-8", errors="replace") - body = "".join(line for line in text.splitlines(keepends=True) if not line.startswith("#")) - for row in csv.DictReader(io.StringIO(body)): + body = strip_leading_comment_block(text) + for row in csv.DictReader(io.StringIO(body, newline="")): if row.get("data_type", "text") != "text" or row["human_score"] not in ("0", "1"): continue pairs.append((row["objective"], row["assistant_response"])) diff --git a/tests/unit/common/test_csv_helper.py b/tests/unit/common/test_csv_helper.py index 7424641d56..be10110ca0 100644 --- a/tests/unit/common/test_csv_helper.py +++ b/tests/unit/common/test_csv_helper.py @@ -1,10 +1,32 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import csv import io from io import StringIO -from pyrit.common.csv_helper import read_csv, write_csv +import pytest + +from pyrit.common.csv_helper import read_csv, strip_leading_comment_block, write_csv + + +@pytest.mark.parametrize("line_ending", ["\n", "\r\n", "\r"]) +@pytest.mark.parametrize("leading_block", ["", "\n", " \t\n", "# metadata\n", "\n # first\n\t\n# second\n"]) +def test_strip_leading_comment_block_preserves_body(*, line_ending: str, leading_block: str) -> None: + body = f'objective,assistant_response{line_ending}# objective,"line one\r\n### heading\rline three"{line_ending}' + text = leading_block.replace("\n", line_ending) + body + assert strip_leading_comment_block(text) == body + + +@pytest.mark.parametrize("separator", ["\v", "\f", "\x85", "\u2028", "\u2029"]) +def test_strip_leading_comment_block_preserves_non_newline_separators(separator: str) -> None: + body = f"value{separator}# header\none\n" + assert strip_leading_comment_block("# metadata\n" + body) == body + + +@pytest.mark.parametrize("text", ["", "\n\t\n", "# comment", "\r\n# comment\r\n \r"]) +def test_strip_leading_comment_block_without_content(text: str) -> None: + assert strip_leading_comment_block(text) == "" def test_read_csv_returns_list_of_dicts(): @@ -81,3 +103,18 @@ def test_write_csv_keeps_first_seen_column_order(): output.seek(0) lines = output.read().strip().splitlines() assert lines[0] == "a,b,c" + + +def test_strip_leading_comment_block_preserves_crlf_and_cr_inside_quoted_cells(): + text = '# dataset_version=1.0\r\nobjective,assistant_response\r\nplain,"line one\r\n### heading\rline three"\r\n' + body = strip_leading_comment_block(text) + assert body == 'objective,assistant_response\r\nplain,"line one\r\n### heading\rline three"\r\n' + + +def test_strip_leading_comment_block_handles_cr_only_record_separators(): + text = "# dataset_version=1.0\robjective,assistant_response\rplain,refusal response\r" + body = strip_leading_comment_block(text) + assert body == "objective,assistant_response\rplain,refusal response\r" + assert list(csv.DictReader(StringIO(body, newline=""))) == [ + {"objective": "plain", "assistant_response": "refusal response"} + ] diff --git a/tests/unit/score/test_local_refusal_classifier_scorer.py b/tests/unit/score/test_local_refusal_classifier_scorer.py index 900bae4f9c..913580d19b 100644 --- a/tests/unit/score/test_local_refusal_classifier_scorer.py +++ b/tests/unit/score/test_local_refusal_classifier_scorer.py @@ -71,6 +71,76 @@ def test_training_rejects_changed_data(tmp_path: Path) -> None: assert _load_training_rows() == ([(objective, response)] * 2, [1, 1]) +@pytest.mark.parametrize("record_separator", ["\n", "\r\n", "\r"]) +@pytest.mark.parametrize("cell_separator", ["\n", "\r\n", "\r"]) +def test_load_training_rows_keeps_hash_lines_outside_the_leading_comments( + *, tmp_path: Path, record_separator: str, cell_separator: str +) -> None: + """Only leading '#' lines are comments; a '#' starting a row or inside a cell is data (#2974).""" + header = f"objective,assistant_response,human_score,data_type{record_separator}" + response = f"line one{cell_separator}### in-cell heading{cell_separator}line three" + bodies = { + "refusal.csv": ( + f"# dataset_version=1.0{record_separator}" + f"{header}" + f"# Hash-prefixed objective,refusal response,1,text{record_separator}" + f'plain objective,"{response}",0,text{record_separator}' + ), + "refusal_extra.csv": f"# dataset_version=1.0{record_separator}{header}", + } + pins = {} + for name, body in bodies.items(): + data = body.encode() + (tmp_path / name).write_bytes(data) + pins[name] = hashlib.sha256(data).hexdigest() + with patch.object(module, "_REFUSAL_EVALS_PATH", tmp_path), patch.object(module, "_TRAINING_DATASETS", pins): + pairs, labels = _load_training_rows() + assert pairs == [("# Hash-prefixed objective", "refusal response"), ("plain objective", response)] + assert labels == [1, 0] + + +def test_load_training_rows_preserves_crlf_and_cr_inside_quoted_responses(tmp_path: Path) -> None: + """The leading-comment scan must not rewrite line endings inside quoted data.""" + response = "line one\r\n### heading\rline three" + bodies = { + "refusal.csv": ( + "# dataset_version=1.0\r\n" + "objective,assistant_response,human_score,data_type\r\n" + f'plain objective,"{response}",0,text\r\n' + ), + "refusal_extra.csv": "# dataset_version=1.0\r\nobjective,assistant_response,human_score,data_type\r\n", + } + pins = {} + for name, body in bodies.items(): + data = body.encode() + (tmp_path / name).write_bytes(data) + pins[name] = hashlib.sha256(data).hexdigest() + with patch.object(module, "_REFUSAL_EVALS_PATH", tmp_path), patch.object(module, "_TRAINING_DATASETS", pins): + pairs, labels = _load_training_rows() + assert pairs == [("plain objective", response)] + assert labels == [0] + + +def test_load_training_rows_accepts_cr_only_record_separators(tmp_path: Path) -> None: + """CR-only files still split into records and keep their payload intact.""" + bodies = { + "refusal.csv": ( + "# dataset_version=1.0\robjective,assistant_response,human_score,data_type\r" + "plain objective,refusal response,1,text\r" + ), + "refusal_extra.csv": "# dataset_version=1.0\robjective,assistant_response,human_score,data_type\r", + } + pins = {} + for name, body in bodies.items(): + data = body.encode() + (tmp_path / name).write_bytes(data) + pins[name] = hashlib.sha256(data).hexdigest() + with patch.object(module, "_REFUSAL_EVALS_PATH", tmp_path), patch.object(module, "_TRAINING_DATASETS", pins): + pairs, labels = _load_training_rows() + assert pairs == [("plain objective", "refusal response")] + assert labels == [1] + + @requires_torch def test_training_is_deterministic_and_preserves_rng() -> None: import torch diff --git a/tests/unit/score/test_local_violence_classifier_scorer.py b/tests/unit/score/test_local_violence_classifier_scorer.py index 3c0cef2cc5..a02715fcc3 100644 --- a/tests/unit/score/test_local_violence_classifier_scorer.py +++ b/tests/unit/score/test_local_violence_classifier_scorer.py @@ -110,6 +110,57 @@ def test_load_training_rows_rejects_changed_dataset(tmp_path: Path) -> None: assert texts == [] and labels == [] +@pytest.mark.parametrize("record_separator", ["\n", "\r\n", "\r"]) +@pytest.mark.parametrize("cell_separator", ["\n", "\r\n", "\r"]) +def test_load_training_rows_skips_the_whole_leading_comment_block( + *, tmp_path: Path, record_separator: str, cell_separator: str +) -> None: + """Every leading '#' line is a comment, not just the first one.""" + harm_path = tmp_path / "harm" + harm_path.mkdir() + header = f"harm_category,objective,assistant_response,human_score_1,data_type{record_separator}" + response = f"line one{cell_separator}### in-cell heading{cell_separator}line three" + bodies = { + "violence.csv": ( + f"{record_separator}" + f"# dataset_version=1.0{record_separator}" + f"# collected by the eval team{record_separator}" + f"{header}" + f'violence,# sample objective,"{response}",0,text{record_separator}' + ), + "violence_multi_score.csv": f"# dataset_version=1.0{record_separator}{header}", + } + pins = {} + for file_name, body in bodies.items(): + data = body.encode() + (harm_path / file_name).write_bytes(data) + pins[file_name] = hashlib.sha256(data).hexdigest() + with patch.object(module, "_HARM_EVALS_PATH", harm_path), patch.object(module, "_TRAINING_DATASETS", pins): + texts, labels = _load_training_rows() + assert texts == [f"PROMPT: # sample objective\nRESPONSE: {response}"] + assert labels == [0] + + +def test_load_training_rows_accepts_cr_only_record_separators(tmp_path: Path) -> None: + """CR-only files still split into records and keep their payload intact.""" + harm_path = tmp_path / "harm" + harm_path.mkdir() + header = "harm_category,objective,assistant_response,human_score_1,data_type\r" + bodies = { + "violence.csv": "# dataset_version=1.0\r" + header + "violence,sample objective,a harmless reply,0,text\r", + "violence_multi_score.csv": "# dataset_version=1.0\r" + header, + } + pins = {} + for file_name, body in bodies.items(): + data = body.encode() + (harm_path / file_name).write_bytes(data) + pins[file_name] = hashlib.sha256(data).hexdigest() + with patch.object(module, "_HARM_EVALS_PATH", harm_path), patch.object(module, "_TRAINING_DATASETS", pins): + texts, labels = _load_training_rows() + assert texts == ["PROMPT: sample objective\nRESPONSE: a harmless reply"] + assert labels == [0] + + @requires_torch def test_train_head_is_deterministic_and_separates() -> None: embeddings, labels = _separable_training_data()