diff --git a/engine/skills/reflect/scripts/tests/test_token_audit.py b/engine/skills/reflect/scripts/tests/test_token_audit.py index fdf09046..f7757cbb 100644 --- a/engine/skills/reflect/scripts/tests/test_token_audit.py +++ b/engine/skills/reflect/scripts/tests/test_token_audit.py @@ -658,7 +658,7 @@ def test_product_blame_and_plain_questions_do_not_flag_intervention(self): lines = [ codex_response_item("user", "the UI is messed up on the workers page", ts="2026-09-09T04:40:47.000Z"), codex_response_item("assistant", "Looking at the panel now.", ts="2026-09-09T04:41:00.000Z"), - codex_response_item("user", "what would you recommend for the deploy window?", ts="2026-09-09T05:46:08.000Z"), + codex_response_item("user", "why did you choose Saturday for the deploy window?", ts="2026-09-09T05:46:08.000Z"), codex_response_item("assistant", "Three options below.", ts="2026-09-09T05:47:00.000Z"), ] path = write_jsonl(lines) @@ -670,6 +670,26 @@ def test_product_blame_and_plain_questions_do_not_flag_intervention(self): finally: os.unlink(path) + def test_handed_over_something_broken_flags_intervention(self): + """The wording that scored zero: no profanity, no told-you. The user + says the artifact was bogus, then that it did not run.""" + lines = [ + codex_response_item("user", "huh? why did you give me a bogus script?", ts="2026-09-11T05:40:00.000Z"), + codex_response_item("assistant", "Fixing the quoting now.", ts="2026-09-11T05:41:00.000Z"), + codex_response_item("user", "it still didn't run", ts="2026-09-11T05:50:00.000Z"), + codex_response_item("assistant", "Smoke-testing the transport first.", ts="2026-09-11T05:51:00.000Z"), + ] + path = write_jsonl(lines) + try: + with redirect_stdout(io.StringIO()): + result = token_audit.audit_codex(path) + kinds = {k for f in result["frustration"]["flagged"] for k in f["kinds"]} + self.assertIn("cheap-way-out", kinds) + flags = {fl["name"]: fl for fl in result["flags"]} + self.assertEqual(flags["intervention-must-automate"]["value"], "yes") + finally: + os.unlink(path) + def test_committed_codex_fixture_hits_frustration_and_intervention(self): result = token_audit.audit_codex(fixture("codex_thrash_session.jsonl")) flags = {fl["name"]: fl for fl in result["flags"]} diff --git a/engine/skills/reflect/scripts/token_audit.py b/engine/skills/reflect/scripts/token_audit.py index 2de08754..b27865be 100644 --- a/engine/skills/reflect/scripts/token_audit.py +++ b/engine/skills/reflect/scripts/token_audit.py @@ -142,7 +142,7 @@ def _direct_run_targets(command): r"|\bis the proof att?ach?ed\b" r"|\bwhere'?s the proof\b" r"|\bprove (to me )?that (it|this|that|the|#?\d)", re.I)), - ("cheap-way-out", re.compile(r"\bcheap way out\b|\bwhy would you\b|\bthat'?s (weird|wierd)\b|\bstraight up\b", re.I)), + ("cheap-way-out", re.compile(r"\bcheap way out\b|\bwhy would you\b|\bbogus\b|\bdidn'?t (even )?(work|run)\b|\bthat'?s (weird|wierd)\b|\bstraight up\b", re.I)), ("explicit-invocation", re.compile(r"(?:^|\s)/(?:automate-me|reflect|thrash)\b", re.I)), ]