From 56515061a68bc4988f94c8a72393e448ddca3669 Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Fri, 11 Sep 2026 23:52:26 -0700 Subject: [PATCH] audit(token_audit): one submission carrying two slash commands is one message `/reflect /cat-mode` on a single line injects the same user text twice, tens of milliseconds apart. The verbatim-repeat rule read that as a re-send, the strongest single frustration signal, and escalated a pass to FAIL on it. A repeat now needs at least five seconds between the two rows. A genuine re-send minutes later still counts, asserted both ways. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KU2pPKob4MJ1NqjsfTNyYJ Change-Id: I0f516b8f1e8856df6ae98bf63c2389cdae97e2f9 --- .../reflect/scripts/tests/test_token_audit.py | 34 +++++++++++++++++++ engine/skills/reflect/scripts/token_audit.py | 5 ++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/engine/skills/reflect/scripts/tests/test_token_audit.py b/engine/skills/reflect/scripts/tests/test_token_audit.py index f7757cbb..c36e70b5 100644 --- a/engine/skills/reflect/scripts/tests/test_token_audit.py +++ b/engine/skills/reflect/scripts/tests/test_token_audit.py @@ -670,6 +670,40 @@ def test_product_blame_and_plain_questions_do_not_flag_intervention(self): finally: os.unlink(path) + def test_two_slash_commands_in_one_submission_are_not_a_repeat(self): + """`/reflect /cat-mode` on one line injects the same text twice, tens + of milliseconds apart. That is one message, not a re-send.""" + text = "please prevent this shit from happening again" + lines = [ + codex_response_item("user", text, ts="2026-09-11T05:40:00.000Z"), + codex_response_item("user", text, ts="2026-09-11T05:40:00.041Z"), + codex_response_item("assistant", "Working on it.", ts="2026-09-11T05:40:05.000Z"), + ] + path = write_jsonl(lines) + try: + with redirect_stdout(io.StringIO()): + result = token_audit.audit_codex(path) + kinds = {k for f in result["frustration"]["flagged"] for k in f["kinds"]} + self.assertNotIn("verbatim-repeat", kinds) + finally: + os.unlink(path) + + def test_a_real_resend_minutes_later_still_counts_as_a_repeat(self): + text = "please prevent this shit from happening again" + lines = [ + codex_response_item("user", text, ts="2026-09-11T05:40:00.000Z"), + codex_response_item("assistant", "Working on it.", ts="2026-09-11T05:40:20.000Z"), + codex_response_item("user", text, ts="2026-09-11T05:44:00.000Z"), + ] + path = write_jsonl(lines) + try: + with redirect_stdout(io.StringIO()): + result = token_audit.audit_codex(path) + kinds = {k for f in result["frustration"]["flagged"] for k in f["kinds"]} + self.assertIn("verbatim-repeat", kinds) + finally: + os.unlink(path) + def test_handed_over_something_broken_flags_intervention(self): """The wording that scored zero: no profanity, no told-you. The user says the artifact was bogus, then that it did not run.""" diff --git a/engine/skills/reflect/scripts/token_audit.py b/engine/skills/reflect/scripts/token_audit.py index b27865be..96424118 100644 --- a/engine/skills/reflect/scripts/token_audit.py +++ b/engine/skills/reflect/scripts/token_audit.py @@ -150,6 +150,8 @@ def _direct_run_targets(command): # class (told-you / accusation / agent-blame twice, two of those kinds in # one session, or a verbatim re-send) is the automate-me trigger. Product # blame ("the ui is messed up") does not match agent-blame. +MIN_RESEND_GAP_SECS = 5 + INTERVENTION_KINDS = frozenset({ "told-you", "accusation", "agent-blame", "restated-ask", "proof-challenge", "cheap-way-out", "explicit-invocation", @@ -299,7 +301,8 @@ def frustration_signals(user_msgs, interruptions=0, failed_turn_indices=None): norm = re.sub(r"\s+", " ", t).casefold() if len(norm) >= 12: for prev_secs, prev_norm, prev_idx in seen: - if prev_norm == norm and (secs is None or prev_secs is None or 0 <= secs - prev_secs <= 600): + gap = None if (secs is None or prev_secs is None) else secs - prev_secs + if prev_norm == norm and (gap is None or MIN_RESEND_GAP_SECS <= gap <= 600): if _failed is not None and _has_index_between(_failed, prev_idx, idx): continue kinds.append("verbatim-repeat")