FIX: Use the canonical leading-comment rule when loading classifier CSVs

pyrit/common/csv_helper.py

@@ -2,9 +2,40 @@
 # Licensed under the MIT license.
 
 import csv
+import io
 from typing import IO, Any
 
 
+def strip_leading_comment_block(text: str) -> str:
+    """
+    Drop the leading "#" comment and blank lines of CSV text.
+
+    Mirrors the scan in ``HumanLabeledDataset._read_leading_comment_block``: only the
+    leading block is a comment, so a "#" that starts a data row or sits inside a cell
+    is preserved as data. Callers that hand the result to ``csv.DictReader`` must use
+    this instead of dropping every "#" line, which silently deletes data.
+
+    Universal newline recognition keeps the block boundary identical to the canonical
+    scan without splitting on separators such as "\\v". Newline characters are retained,
+    including CR and CRLF inside quoted cells. CSV readers must also preserve newlines.
+
+    Args:
+        text (str): The full CSV text.
+
+    Returns:
+        str: The text from the first non-comment, non-blank line onward.
+    """
+    stream = io.StringIO(text, newline="")
+    remaining: list[str] = []
+    for line in stream:
+        stripped = line.strip()
+        if stripped and not stripped.startswith("#"):
+            remaining.append(line)
+            break
+    remaining.append(stream.read())
+    return "".join(remaining)
+
+
 def read_csv(file: IO[Any]) -> list[dict[str, str]]:
     """
     Read a CSV file and return its rows as dictionaries.

pyrit/score/float_scale/local_violence_classifier_scorer.py

@@ -16,6 +16,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar
 
+from pyrit.common.csv_helper import strip_leading_comment_block
 from pyrit.common.path import SCORER_EVALS_HARM_PATH
 from pyrit.models import ComponentIdentifier, MessagePiece, Score, ScoreStatus
 from pyrit.score.float_scale.float_scale_scorer import MessageFloatScaleScorer
@@ -307,9 +308,8 @@ def _load_training_rows() -> tuple[list[str], list[int]]:
                 "If the dataset was updated deliberately, re-validate the scorer and update "
                 "LocalViolenceClassifierScorer's pinned hashes rather than scoring with an unchecked model."
             )
-        lines = raw.decode("utf-8").splitlines()
-        start = 1 if lines and lines[0].startswith("#") else 0
-        for row in csv.DictReader(io.StringIO("\n".join(lines[start:]))):
+        body = strip_leading_comment_block(raw.decode("utf-8"))
+        for row in csv.DictReader(io.StringIO(body, newline="")):
             raw_scores = [row.get(key) for key in ("human_score", "human_score_1", "human_score_2", "human_score_3")]
             scores = [float(value) for value in raw_scores if value not in (None, "")]
             response = (row.get("assistant_response") or "").strip()

pyrit/score/true_false/local_refusal_classifier_scorer.py

@@ -16,6 +16,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar
 
+from pyrit.common.csv_helper import strip_leading_comment_block
 from pyrit.common.path import SCORER_EVALS_REFUSAL_SCORER_PATH
 from pyrit.models import ComponentIdentifier, Message, MessagePiece, Score, ScoreStatus
 from pyrit.score.scorer_prompt_validator import ScorerPromptValidator
@@ -366,8 +367,8 @@ def _load_training_rows() -> tuple[list[tuple[str, str]], list[int]]:
                 f"({expected_hash}, found {actual_hash}). Re-validate the scorer and update the pin."
             )
         text = raw.decode("utf-8", errors="replace")
-        body = "".join(line for line in text.splitlines(keepends=True) if not line.startswith("#"))
-        for row in csv.DictReader(io.StringIO(body)):
+        body = strip_leading_comment_block(text)
+        for row in csv.DictReader(io.StringIO(body, newline="")):
             if row.get("data_type", "text") != "text" or row["human_score"] not in ("0", "1"):
                 continue
             pairs.append((row["objective"], row["assistant_response"]))

tests/unit/common/test_csv_helper.py

@@ -1,10 +1,32 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
+import csv
 import io
 from io import StringIO
 
-from pyrit.common.csv_helper import read_csv, write_csv
+import pytest
+
+from pyrit.common.csv_helper import read_csv, strip_leading_comment_block, write_csv
+
+
+@pytest.mark.parametrize("line_ending", ["\n", "\r\n", "\r"])
+@pytest.mark.parametrize("leading_block", ["", "\n", " \t\n", "# metadata\n", "\n  # first\n\t\n# second\n"])
+def test_strip_leading_comment_block_preserves_body(*, line_ending: str, leading_block: str) -> None:
+    body = f'objective,assistant_response{line_ending}# objective,"line one\r\n### heading\rline three"{line_ending}'
+    text = leading_block.replace("\n", line_ending) + body
+    assert strip_leading_comment_block(text) == body
+
+
+@pytest.mark.parametrize("separator", ["\v", "\f", "\x85", "\u2028", "\u2029"])
+def test_strip_leading_comment_block_preserves_non_newline_separators(separator: str) -> None:
+    body = f"value{separator}# header\none\n"
+    assert strip_leading_comment_block("# metadata\n" + body) == body
+
+
+@pytest.mark.parametrize("text", ["", "\n\t\n", "# comment", "\r\n# comment\r\n \r"])
+def test_strip_leading_comment_block_without_content(text: str) -> None:
+    assert strip_leading_comment_block(text) == ""
 
 
 def test_read_csv_returns_list_of_dicts():
@@ -81,3 +103,18 @@ def test_write_csv_keeps_first_seen_column_order():
     output.seek(0)
     lines = output.read().strip().splitlines()
     assert lines[0] == "a,b,c"
+
+
+def test_strip_leading_comment_block_preserves_crlf_and_cr_inside_quoted_cells():
+    text = '# dataset_version=1.0\r\nobjective,assistant_response\r\nplain,"line one\r\n### heading\rline three"\r\n'
+    body = strip_leading_comment_block(text)
+    assert body == 'objective,assistant_response\r\nplain,"line one\r\n### heading\rline three"\r\n'
+
+
+def test_strip_leading_comment_block_handles_cr_only_record_separators():
+    text = "# dataset_version=1.0\robjective,assistant_response\rplain,refusal response\r"
+    body = strip_leading_comment_block(text)
+    assert body == "objective,assistant_response\rplain,refusal response\r"
+    assert list(csv.DictReader(StringIO(body, newline=""))) == [
+        {"objective": "plain", "assistant_response": "refusal response"}
+    ]

tests/unit/score/test_local_refusal_classifier_scorer.py

@@ -71,6 +71,76 @@ def test_training_rejects_changed_data(tmp_path: Path) -> None:
             assert _load_training_rows() == ([(objective, response)] * 2, [1, 1])
 
 
+@pytest.mark.parametrize("record_separator", ["\n", "\r\n", "\r"])
+@pytest.mark.parametrize("cell_separator", ["\n", "\r\n", "\r"])
+def test_load_training_rows_keeps_hash_lines_outside_the_leading_comments(
+    *, tmp_path: Path, record_separator: str, cell_separator: str
+) -> None:
+    """Only leading '#' lines are comments; a '#' starting a row or inside a cell is data (#2974)."""
+    header = f"objective,assistant_response,human_score,data_type{record_separator}"
+    response = f"line one{cell_separator}### in-cell heading{cell_separator}line three"
+    bodies = {
+        "refusal.csv": (
+            f"# dataset_version=1.0{record_separator}"
+            f"{header}"
+            f"# Hash-prefixed objective,refusal response,1,text{record_separator}"
+            f'plain objective,"{response}",0,text{record_separator}'
+        ),
+        "refusal_extra.csv": f"# dataset_version=1.0{record_separator}{header}",
+    }
+    pins = {}
+    for name, body in bodies.items():
+        data = body.encode()
+        (tmp_path / name).write_bytes(data)
+        pins[name] = hashlib.sha256(data).hexdigest()
+    with patch.object(module, "_REFUSAL_EVALS_PATH", tmp_path), patch.object(module, "_TRAINING_DATASETS", pins):
+        pairs, labels = _load_training_rows()
+    assert pairs == [("# Hash-prefixed objective", "refusal response"), ("plain objective", response)]
+    assert labels == [1, 0]
+
+
+def test_load_training_rows_preserves_crlf_and_cr_inside_quoted_responses(tmp_path: Path) -> None:
+    """The leading-comment scan must not rewrite line endings inside quoted data."""
+    response = "line one\r\n### heading\rline three"
+    bodies = {
+        "refusal.csv": (
+            "# dataset_version=1.0\r\n"
+            "objective,assistant_response,human_score,data_type\r\n"
+            f'plain objective,"{response}",0,text\r\n'
+        ),
+        "refusal_extra.csv": "# dataset_version=1.0\r\nobjective,assistant_response,human_score,data_type\r\n",
+    }
+    pins = {}
+    for name, body in bodies.items():
+        data = body.encode()
+        (tmp_path / name).write_bytes(data)
+        pins[name] = hashlib.sha256(data).hexdigest()
+    with patch.object(module, "_REFUSAL_EVALS_PATH", tmp_path), patch.object(module, "_TRAINING_DATASETS", pins):
+        pairs, labels = _load_training_rows()
+    assert pairs == [("plain objective", response)]
+    assert labels == [0]
+
+
+def test_load_training_rows_accepts_cr_only_record_separators(tmp_path: Path) -> None:
+    """CR-only files still split into records and keep their payload intact."""
+    bodies = {
+        "refusal.csv": (
+            "# dataset_version=1.0\robjective,assistant_response,human_score,data_type\r"
+            "plain objective,refusal response,1,text\r"
+        ),
+        "refusal_extra.csv": "# dataset_version=1.0\robjective,assistant_response,human_score,data_type\r",
+    }
+    pins = {}
+    for name, body in bodies.items():
+        data = body.encode()
+        (tmp_path / name).write_bytes(data)
+        pins[name] = hashlib.sha256(data).hexdigest()
+    with patch.object(module, "_REFUSAL_EVALS_PATH", tmp_path), patch.object(module, "_TRAINING_DATASETS", pins):
+        pairs, labels = _load_training_rows()
+    assert pairs == [("plain objective", "refusal response")]
+    assert labels == [1]
+
+
 @requires_torch
 def test_training_is_deterministic_and_preserves_rng() -> None:
     import torch

tests/unit/score/test_local_violence_classifier_scorer.py

@@ -110,6 +110,57 @@ def test_load_training_rows_rejects_changed_dataset(tmp_path: Path) -> None:
     assert texts == [] and labels == []
 
 
+@pytest.mark.parametrize("record_separator", ["\n", "\r\n", "\r"])
+@pytest.mark.parametrize("cell_separator", ["\n", "\r\n", "\r"])
+def test_load_training_rows_skips_the_whole_leading_comment_block(
+    *, tmp_path: Path, record_separator: str, cell_separator: str
+) -> None:
+    """Every leading '#' line is a comment, not just the first one."""
+    harm_path = tmp_path / "harm"
+    harm_path.mkdir()
+    header = f"harm_category,objective,assistant_response,human_score_1,data_type{record_separator}"
+    response = f"line one{cell_separator}### in-cell heading{cell_separator}line three"
+    bodies = {
+        "violence.csv": (
+            f"{record_separator}"
+            f"# dataset_version=1.0{record_separator}"
+            f"# collected by the eval team{record_separator}"
+            f"{header}"
+            f'violence,# sample objective,"{response}",0,text{record_separator}'
+        ),
+        "violence_multi_score.csv": f"# dataset_version=1.0{record_separator}{header}",
+    }
+    pins = {}
+    for file_name, body in bodies.items():
+        data = body.encode()
+        (harm_path / file_name).write_bytes(data)
+        pins[file_name] = hashlib.sha256(data).hexdigest()
+    with patch.object(module, "_HARM_EVALS_PATH", harm_path), patch.object(module, "_TRAINING_DATASETS", pins):
+        texts, labels = _load_training_rows()
+    assert texts == [f"PROMPT: # sample objective\nRESPONSE: {response}"]
+    assert labels == [0]
+
+
+def test_load_training_rows_accepts_cr_only_record_separators(tmp_path: Path) -> None:
+    """CR-only files still split into records and keep their payload intact."""
+    harm_path = tmp_path / "harm"
+    harm_path.mkdir()
+    header = "harm_category,objective,assistant_response,human_score_1,data_type\r"
+    bodies = {
+        "violence.csv": "# dataset_version=1.0\r" + header + "violence,sample objective,a harmless reply,0,text\r",
+        "violence_multi_score.csv": "# dataset_version=1.0\r" + header,
+    }
+    pins = {}
+    for file_name, body in bodies.items():
+        data = body.encode()
+        (harm_path / file_name).write_bytes(data)
+        pins[file_name] = hashlib.sha256(data).hexdigest()
+    with patch.object(module, "_HARM_EVALS_PATH", harm_path), patch.object(module, "_TRAINING_DATASETS", pins):
+        texts, labels = _load_training_rows()
+    assert texts == ["PROMPT: sample objective\nRESPONSE: a harmless reply"]
+    assert labels == [0]
+
+
 @requires_torch
 def test_train_head_is_deterministic_and_separates() -> None:
     embeddings, labels = _separable_training_data()