From 4894e276efaacbb4837b890e6b764ce6de50236b Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Sat, 12 Sep 2026 00:06:39 -0700 Subject: [PATCH] audit: use the transcript's own markers for what the human actually sent Two shapes were each counted as a second human message. A line carrying two slash commands writes the same command-args twice; the transcript marks the second row stackedExpansion, and both skill bodies are already marked isMeta. Typing while the agent is busy writes an enqueue row and, minutes later, a delivery row with identical text; the gap between them read as the user re-sending in frustration. Both are now dropped by those markers rather than by a timing guess. On the session that motivated this: 8 of 23 flagged with a verbatim-repeat becomes 7 of 22 with none. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KU2pPKob4MJ1NqjsfTNyYJ Change-Id: Ia2a6527cc6efe8311affd0d7e85c43f9884d8dd6 --- .../provenance/queued_message/claude.jsonl | 3 ++ .../queued_undelivered/claude.jsonl | 2 ++ .../provenance/stacked_commands/claude.jsonl | 4 +++ .../tests/test_transcript_provenance.py | 31 +++++++++++++++++++ .../reflect/scripts/transcript_provenance.py | 2 ++ 5 files changed, 42 insertions(+) create mode 100644 engine/skills/reflect/scripts/tests/fixtures/provenance/queued_message/claude.jsonl create mode 100644 engine/skills/reflect/scripts/tests/fixtures/provenance/queued_undelivered/claude.jsonl create mode 100644 engine/skills/reflect/scripts/tests/fixtures/provenance/stacked_commands/claude.jsonl diff --git a/engine/skills/reflect/scripts/tests/fixtures/provenance/queued_message/claude.jsonl b/engine/skills/reflect/scripts/tests/fixtures/provenance/queued_message/claude.jsonl new file mode 100644 index 00000000..ad526d57 --- /dev/null +++ b/engine/skills/reflect/scripts/tests/fixtures/provenance/queued_message/claude.jsonl @@ -0,0 +1,3 @@ +{"type": "queue-operation", "operation": "enqueue", "timestamp": "2026-09-12T06:45:37.097Z", "sessionId": "s1", "content": "ok if there is a better fix, then jsut do it"} +{"type": "assistant", "timestamp": "2026-09-12T06:46:00.000Z", "sessionId": "s1", "message": {"role": "assistant", "content": [{"type": "text", "text": "still working"}]}} +{"type": "user", "timestamp": "2026-09-12T06:50:28.468Z", "sessionId": "s1", "promptId": "p9", "origin": {"kind": "human"}, "message": {"role": "user", "content": "ok if there is a better fix, then jsut do it"}} diff --git a/engine/skills/reflect/scripts/tests/fixtures/provenance/queued_undelivered/claude.jsonl b/engine/skills/reflect/scripts/tests/fixtures/provenance/queued_undelivered/claude.jsonl new file mode 100644 index 00000000..9727cd5a --- /dev/null +++ b/engine/skills/reflect/scripts/tests/fixtures/provenance/queued_undelivered/claude.jsonl @@ -0,0 +1,2 @@ +{"type": "queue-operation", "operation": "enqueue", "timestamp": "2026-09-12T06:45:37.097Z", "sessionId": "s1", "content": "ok if there is a better fix, then jsut do it"} +{"type": "assistant", "timestamp": "2026-09-12T06:46:00.000Z", "sessionId": "s1", "message": {"role": "assistant", "content": [{"type": "text", "text": "still working"}]}} diff --git a/engine/skills/reflect/scripts/tests/fixtures/provenance/stacked_commands/claude.jsonl b/engine/skills/reflect/scripts/tests/fixtures/provenance/stacked_commands/claude.jsonl new file mode 100644 index 00000000..387c82c2 --- /dev/null +++ b/engine/skills/reflect/scripts/tests/fixtures/provenance/stacked_commands/claude.jsonl @@ -0,0 +1,4 @@ +{"type": "user", "timestamp": "2026-09-12T06:10:43.745Z", "sessionId": "s1", "promptId": "p1", "origin": {"kind": "human"}, "stackedOriginalInput": "/reflect /cat-mode please prevent this shit from happening again", "message": {"role": "user", "content": "reflect\n/reflect\nplease prevent this shit from happening again"}} +{"type": "user", "timestamp": "2026-09-12T06:10:43.745Z", "sessionId": "s1", "promptId": "p1", "isMeta": true, "turnCompanion": true, "message": {"role": "user", "content": "Base directory for this skill: /skills/reflect\n\n# Reflect\n\nbody"}} +{"type": "user", "timestamp": "2026-09-12T06:10:43.786Z", "sessionId": "s1", "promptId": "p1", "origin": {"kind": "human"}, "stackedExpansion": true, "message": {"role": "user", "content": "cat-mode\n/cat-mode\nplease prevent this shit from happening again"}} +{"type": "user", "timestamp": "2026-09-12T06:10:43.786Z", "sessionId": "s1", "promptId": "p1", "isMeta": true, "turnCompanion": true, "message": {"role": "user", "content": "Base directory for this skill: /skills/cat-mode\n\n# cat-mode\n\nbody"}} diff --git a/engine/skills/reflect/scripts/tests/test_transcript_provenance.py b/engine/skills/reflect/scripts/tests/test_transcript_provenance.py index faf0826f..a4a41a12 100644 --- a/engine/skills/reflect/scripts/tests/test_transcript_provenance.py +++ b/engine/skills/reflect/scripts/tests/test_transcript_provenance.py @@ -72,6 +72,37 @@ def test_human_complaint_beside_teammate_relay_still_fires(self): self.assertEqual(len(direct), 1) self.assertTrue(direct[0].text.startswith("I told you to reproduce my screenshot")) + def test_one_submission_with_two_slash_commands_is_one_human_message(self): + """`/reflect /cat-mode ` writes the same args twice, tens of + milliseconds apart. The second row is marked stackedExpansion, and the + two skill bodies are isMeta. One human message, not three.""" + path = os.path.join(FIXTURES, "stacked_commands", "claude.jsonl") + direct = provenance.direct_human_utterances(path, "claude") + self.assertEqual( + [row.text for row in direct], + ["please prevent this shit from happening again"], + ) + rows = provenance.extract_utterances(path, "claude") + self.assertEqual([row.provenance for row in rows], ["direct_human", "hook", "hook"]) + + def test_a_queued_message_and_its_delivery_are_one_human_message(self): + """Typing while the agent is busy writes an enqueue row, and the + delivery writes a user row minutes later. One message, counted once, + or the gap between them reads as the user re-sending in frustration.""" + path = os.path.join(FIXTURES, "queued_message", "claude.jsonl") + direct = provenance.direct_human_utterances( + path, "claude", include_queue_operations=True, + ) + self.assertEqual(len(direct), 1, [row.text for row in direct]) + self.assertEqual(direct[0].text, "ok if there is a better fix, then jsut do it") + + def test_an_undelivered_queued_message_still_counts_once(self): + path = os.path.join(FIXTURES, "queued_undelivered", "claude.jsonl") + direct = provenance.direct_human_utterances( + path, "claude", include_queue_operations=True, + ) + self.assertEqual(len(direct), 1, [row.text for row in direct]) + def test_negative_subagent_copies_share_lineage_but_are_not_direct_human(self): cases = { "claude": ("claude-root.jsonl", "agent-claude.jsonl"), diff --git a/engine/skills/reflect/scripts/transcript_provenance.py b/engine/skills/reflect/scripts/transcript_provenance.py index d4c5bc38..58f61b86 100644 --- a/engine/skills/reflect/scripts/transcript_provenance.py +++ b/engine/skills/reflect/scripts/transcript_provenance.py @@ -155,6 +155,8 @@ def _claude_utterances( }} if row.get("type") != "user": continue + if row.get("stackedExpansion"): + continue message = row.get("message") content = message.get("content") if isinstance(message, dict) else None if isinstance(content, list) and any(