From bf32badd194b1c7659a2ed89233fb3656b33a79c Mon Sep 17 00:00:00 2001 From: chen Date: Thu, 23 Jul 2026 20:01:31 +0800 Subject: [PATCH 1/4] test: parity and input oracles --- .../dual_path_parity_oracle.test.js | 81 +++++ tests/test_dual_path_parity_oracle.py | 325 ++++++++++++++++++ 2 files changed, 406 insertions(+) create mode 100644 static/js/render/tool_result/dual_path_parity_oracle.test.js create mode 100644 tests/test_dual_path_parity_oracle.py diff --git a/static/js/render/tool_result/dual_path_parity_oracle.test.js b/static/js/render/tool_result/dual_path_parity_oracle.test.js new file mode 100644 index 0000000..2f1bd1f --- /dev/null +++ b/static/js/render/tool_result/dual_path_parity_oracle.test.js @@ -0,0 +1,81 @@ +/** + * Part A — Golden-parity: JS renderer path for the adversarial bash tool_result. + * + * The Python md-exporter path is covered by + * ``tests/test_dual_path_parity_oracle.py``. Both suites use the same + * adversarial payload (````). + * + * These tests drive the **real** JS dispatch path via ``renderToolResult`` + * (registry.js) and the direct renderer ``renderBashResult`` (bash.js) — no + * reimplementations. + * + * Oracle discipline (test-review C3/C8/C9): + * Every assertion checks the real observable HTML output, never merely that + * the function does not throw. + */ + +import { describe, it, expect } from 'vitest'; +import { renderToolResult } from '../registry.js'; +import { renderBashResult } from './bash.js'; + +// Shared adversarial payload — must match the Python constant in +// tests/test_dual_path_parity_oracle.py. +const PAYLOAD = ''; + +// Parsed bash dict that mirrors what ``_parse_tool_result`` / ``_tool_result_build_bash`` +// would produce for the adversarial blob. +const ADVERSARIAL_PARSED = { + result_type: 'bash', + slug: null, + stdout: PAYLOAD, + stderr: '', + exit_code: 0, + interrupted: false, + is_error: false, + return_code_interpretation: 'success', +}; + +describe('Part A: golden-parity — adversarial bash result (JS renderer path)', () => { + it('renders a non-empty HTML string for the adversarial payload', () => { + // Oracle: output must not be empty — the payload is not silently dropped. + const html = renderToolResult(ADVERSARIAL_PARSED); + expect(typeof html).toBe('string'); + expect(html.length).toBeGreaterThan(0); + }); + + it('HTML-escapes the adversarial payload — raw XSS tag must not appear', () => { + // Oracle: the raw `` { + // Driving both the registry dispatch (renderToolResult) and the direct + // renderer (renderBashResult) confirms they produce the same output. + // If registry.js ever wired a different renderer for result_type 'bash', + // this equality assertion would fail. + const viaRegistry = renderToolResult(ADVERSARIAL_PARSED); + const viaDirect = renderBashResult(ADVERSARIAL_PARSED); + expect(viaRegistry).toBe(viaDirect); + }); + + it('negative control: the raw payload is a dangerous string without escaping', () => { + // Documents why esc() is load-bearing: PAYLOAD itself contains the + // executable XSS vector. If the renderer passed it through unchanged, + // the browser could execute it. + expect(PAYLOAD).toContain(' str: + """Write JSONL lines to *path*, return str path for parse_session.""" + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return str(path) + + +def _user_entry(uuid: str, text: str = "hello", parent_uuid: str | None = None) -> str: + entry: dict[str, object] = { + "type": "user", + "uuid": uuid, + "timestamp": "2026-01-01T00:00:00Z", + "message": {"content": [{"type": "text", "text": text}]}, + } + if parent_uuid is not None: + entry["parentUuid"] = parent_uuid + return json.dumps(entry) + + +def _assistant_entry(uuid: str, parent_uuid: str, text: str = "hi") -> str: + return json.dumps( + { + "type": "assistant", + "uuid": uuid, + "parentUuid": parent_uuid, + "timestamp": "2026-01-01T00:00:01Z", + "message": { + "model": "claude-test", + "content": [{"type": "text", "text": text}], + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + } + ) + + +# =========================================================================== +# Part A — Golden-parity test (Python md-exporter path) +# =========================================================================== + + +def test_adversarial_bash_blob_dispatches_to_bash() -> None: + """The adversarial bash blob classifies as result_type 'bash'. + + This drives the real ``_parse_tool_result`` dispatch path; a registry miss + would produce result_type='unknown' and the assertion would fail. + """ + tr: dict[str, object] = { + "stdout": ADVERSARIAL_PAYLOAD, + "exitCode": 0, + "stderr": "", + "interrupted": False, + "is_error": False, + } + parsed = _parse_tool_result(tr) + + assert parsed is not None + assert parsed["result_type"] == "bash", ( + f"expected result_type='bash', got {parsed['result_type']!r}; " + "dispatch registry path must classify the blob correctly" + ) + # stdout preserved through dispatch (not dropped) + assert parsed["stdout"] == ADVERSARIAL_PAYLOAD + + +def test_adversarial_bash_result_python_path_wraps_in_code_fence() -> None: + """Part A oracle: Python md-exporter wraps adversarial stdout in a Markdown + code fence, neutralising the XSS vector as inert literal text. + + Negative control: if the code fence were removed from ``_render_tool_result`` + (renderer regressed to raw prose), ``opening_fence`` below would be None and + the assertion would fail — a diverging renderer cannot pass this test. + """ + tr: dict[str, object] = { + "stdout": ADVERSARIAL_PAYLOAD, + "exitCode": 0, + "stderr": "", + "interrupted": False, + "is_error": False, + } + parsed = _parse_tool_result(tr) + assert parsed is not None + + md = _render_tool_result(parsed) + + # Content must not be silently dropped + assert ADVERSARIAL_PAYLOAD in md, ( + "adversarial payload must appear in Markdown output; it was silently dropped" + ) + + # The payload must sit inside a code fence, not in bare Markdown prose. + # (Code fences render as 
…
, which browsers display + # as literal text and cannot execute as HTML.) + lines = md.splitlines() + fence_indices = [i for i, ln in enumerate(lines) if ln.strip().startswith("```")] + payload_indices = [i for i, ln in enumerate(lines) if ADVERSARIAL_PAYLOAD in ln] + + assert fence_indices, "at least one Markdown code fence marker must be present" + assert payload_indices, "adversarial payload must appear in at least one output line" + + payload_idx = payload_indices[0] + opening_fence = next((i for i in fence_indices if i < payload_idx), None) + closing_fence = next((i for i in fence_indices if i > payload_idx), None) + + assert opening_fence is not None, ( + f"no opening code fence before payload at line {payload_idx}; " + "payload is exposed in bare Markdown prose — Python renderer diverged" + ) + assert closing_fence is not None, ( + f"no closing code fence after payload at line {payload_idx}; " + "code fence is unclosed — Python renderer diverged" + ) + + +# =========================================================================== +# Part B — Malformed-input oracle tests +# =========================================================================== + + +def test_truncated_final_line_returns_partial_parse(tmp_path: Path) -> None: + """Truncated final line → earlier valid messages are returned, not a silent empty. + + Oracle: exactly 2 messages are parsed (the truncated line produces a + JSONDecodeError and is silently skipped). The result is distinguishable + from both empty (0 messages, which would indicate catastrophic loss) and + full success (3 messages, which would require parsing the truncated line). + """ + line1 = _user_entry("u1") + line2 = _assistant_entry("a1", parent_uuid="u1") + # Simulate a truncated write: JSON object cut mid-serialisation + full_line3 = _user_entry("u2", parent_uuid="a1") + truncated = full_line3[: max(1, len(full_line3) // 2)] + + path = _write_jsonl(tmp_path / "truncated.jsonl", [line1, line2, truncated]) + session = parse_session(path) + + # Oracle: exactly 2 messages (truncated line dropped, prior messages intact) + assert len(session["messages"]) == 2, ( + f"expected 2 messages (valid lines preserved, truncated line skipped), " + f"got {len(session['messages'])}; " + "a silent-empty regression would produce 0, a mis-parse would produce 1 or 3" + ) + assert session["messages"][0]["role"] == "user" + assert session["messages"][1]["role"] == "assistant" + + +def test_broken_parent_uuid_chain_stored_verbatim(tmp_path: Path) -> None: + """Broken parentUuid → all messages parsed; broken pointer stored verbatim. + + The parser does not cross-reference parentUuid against known UUIDs; it stores + the raw value and lets downstream tree traversal detect the break. + + Oracle: the assistant message's ``parent_uuid`` equals the non-existent UUID + (not None, not the correct predecessor "u1"). Distinguishable from a + well-formed chain where the value would be "u1". + """ + DEAD_UUID = "00000000-dead-dead-dead-000000000000" + + line1 = _user_entry("u1") + # assistant claims to descend from a UUID that never appeared in this file + line2 = _assistant_entry("a1", parent_uuid=DEAD_UUID) + + path = _write_jsonl(tmp_path / "broken_chain.jsonl", [line1, line2]) + session = parse_session(path) + + # Both messages are present (no crash, no silent drop) + assert len(session["messages"]) == 2, ( + f"expected 2 messages, got {len(session['messages'])}; " + "broken parentUuid must not cause messages to be dropped" + ) + + assistant_msg = session["messages"][1] + assert assistant_msg["role"] == "assistant" + + # Oracle: the broken pointer is stored as-is — not silenced to None or corrected + assert assistant_msg["parent_uuid"] == DEAD_UUID, ( + f"expected broken UUID {DEAD_UUID!r}, got {assistant_msg['parent_uuid']!r}; " + "parser must store the verbatim parentUuid value, not correct or silence it" + ) + # Explicitly distinguishable from a well-formed chain + assert assistant_msg["parent_uuid"] != "u1", ( + "parent_uuid must NOT be the correct predecessor UUID — this is the malformed case" + ) + + +def test_nul_bytes_in_line_dropped_valid_lines_recovered(tmp_path: Path) -> None: + """NUL bytes embedded at the start and end of a JSONL line → that line is + dropped; the subsequent valid line is parsed and returned. + + The parser opens with ``errors='replace'``, so NUL bytes survive to + ``json.loads``, which rejects the leading NUL and raises JSONDecodeError. + The next valid line is still processed. + + Oracle: exactly 1 message is recovered and its ``text`` equals the string + from the valid line. Not a silent empty (which would be 0 messages). + """ + nul_line = b'\x00{"type":"user","uuid":"bad","message":{"content":[{"type":"text","text":"gone"}]}}\x00' + valid_line = _user_entry("u-recovered", text="recovered-after-nul") + + path = tmp_path / "nuls.jsonl" + path.write_bytes(nul_line + b"\n" + valid_line.encode("utf-8") + b"\n") + + session = parse_session(str(path)) + + # Oracle: exactly 1 message (NUL-byte line dropped, valid line recovered) + assert len(session["messages"]) == 1, ( + f"expected 1 message (NUL-line dropped, valid line recovered), " + f"got {len(session['messages'])}; " + "a silent-empty regression would produce 0" + ) + assert session["messages"][0]["role"] == "user" + # Specific oracle: text is from the valid line, not garbled NUL content + assert session["messages"][0].get("text") == "recovered-after-nul", ( + f"expected text 'recovered-after-nul', got {session['messages'][0].get('text')!r}; " + "the recovered message must carry the valid line's content" + ) + + +def test_tool_result_with_no_matching_tool_use_yields_specific_result_type( + tmp_path: Path, +) -> None: + """A user entry with a bash toolUseResult but no matching assistant tool_use + → the message is parsed and ``tool_result_parsed`` carries result_type='bash'. + + This is the malformed case: in a well-formed session the user tool_result + entry is preceded by an assistant entry with a matching tool_use id. Here + there is no such assistant entry at all. The parser must still dispatch the + blob and store the result — it must not silently set tool_result_parsed=None. + + Oracle: ``tool_result_parsed["result_type"] == "bash"`` and ``stdout`` + matches the input. The message's ``slug`` is stored verbatim. + Distinguishable from a missing tool_result (where tool_result_parsed + would be None) and from an unknown dispatch (result_type='unknown'). + """ + bash_tr: dict[str, object] = { + "stdout": "orphaned output", + "exitCode": 0, + "stderr": "", + "interrupted": False, + "is_error": False, + } + entry: dict[str, object] = { + "type": "user", + "uuid": "u-orphan", + "parentUuid": "a-nonexistent", + "timestamp": "2026-01-01T00:00:00Z", + "slug": "orphaned-slug", + "toolUseResult": bash_tr, + "message": {"content": []}, + } + + path = _write_jsonl(tmp_path / "orphan_tr.jsonl", [json.dumps(entry)]) + session = parse_session(path) + + # Not a silent empty — exactly 1 message returned + assert len(session["messages"]) == 1, ( + f"expected 1 message, got {len(session['messages'])}; " + "orphaned tool_result must not cause the message to be silently dropped" + ) + msg = session["messages"][0] + assert msg["role"] == "user" + + # tool_result_parsed must be populated with the correct specific type + trp = msg.get("tool_result_parsed") + assert trp is not None, ( + "tool_result_parsed must not be None when toolUseResult is present; " + "orphaned blob must still be dispatched through the real registry path" + ) + assert trp["result_type"] == "bash", ( + f"expected result_type='bash' for bash blob, got {trp['result_type']!r}; " + "distinguishable from result_type='unknown' (dispatch miss)" + ) + assert trp.get("stdout") == "orphaned output", ( + f"expected stdout='orphaned output', got {trp.get('stdout')!r}; " + "stdout must be preserved even without a matching tool_use" + ) + # slug stored on the message (not on tool_result_parsed) + assert msg.get("slug") == "orphaned-slug", ( + "slug must be stored verbatim on the user message" + ) From 53689dced7dcbe0f80a0018a02aca40f7179030f Mon Sep 17 00:00:00 2001 From: chen Date: Thu, 23 Jul 2026 21:16:37 +0800 Subject: [PATCH 2/4] style: fix ruff failures in parity oracle tests --- tests/test_dual_path_parity_oracle.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_dual_path_parity_oracle.py b/tests/test_dual_path_parity_oracle.py index 4e1b47a..e636511 100644 --- a/tests/test_dual_path_parity_oracle.py +++ b/tests/test_dual_path_parity_oracle.py @@ -20,7 +20,6 @@ from __future__ import annotations import json -import os import sys from pathlib import Path @@ -239,7 +238,8 @@ def test_nul_bytes_in_line_dropped_valid_lines_recovered(tmp_path: Path) -> None Oracle: exactly 1 message is recovered and its ``text`` equals the string from the valid line. Not a silent empty (which would be 0 messages). """ - nul_line = b'\x00{"type":"user","uuid":"bad","message":{"content":[{"type":"text","text":"gone"}]}}\x00' + bad_user = b'{"type":"user","uuid":"bad","message":{"content":[{"type":"text","text":"gone"}]}}' + nul_line = b"\x00" + bad_user + b"\x00" valid_line = _user_entry("u-recovered", text="recovered-after-nul") path = tmp_path / "nuls.jsonl" @@ -320,6 +320,4 @@ def test_tool_result_with_no_matching_tool_use_yields_specific_result_type( "stdout must be preserved even without a matching tool_use" ) # slug stored on the message (not on tool_result_parsed) - assert msg.get("slug") == "orphaned-slug", ( - "slug must be stored verbatim on the user message" - ) + assert msg.get("slug") == "orphaned-slug", "slug must be stored verbatim on the user message" From 68f41f446d6c7232570211891911db02ca08e5af Mon Sep 17 00:00:00 2001 From: chen Date: Thu, 23 Jul 2026 21:21:58 +0800 Subject: [PATCH 3/4] clean up: shared component wired --- tests/test_dual_path_parity_oracle.py | 29 ++++++++++++--------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/tests/test_dual_path_parity_oracle.py b/tests/test_dual_path_parity_oracle.py index e636511..e602901 100644 --- a/tests/test_dual_path_parity_oracle.py +++ b/tests/test_dual_path_parity_oracle.py @@ -39,6 +39,17 @@ ADVERSARIAL_PAYLOAD: str = "" +def _adversarial_bash_tr() -> dict[str, object]: + """toolUseResult-shaped blob used by Part A parity tests.""" + return { + "stdout": ADVERSARIAL_PAYLOAD, + "exitCode": 0, + "stderr": "", + "interrupted": False, + "is_error": False, + } + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -89,14 +100,7 @@ def test_adversarial_bash_blob_dispatches_to_bash() -> None: This drives the real ``_parse_tool_result`` dispatch path; a registry miss would produce result_type='unknown' and the assertion would fail. """ - tr: dict[str, object] = { - "stdout": ADVERSARIAL_PAYLOAD, - "exitCode": 0, - "stderr": "", - "interrupted": False, - "is_error": False, - } - parsed = _parse_tool_result(tr) + parsed = _parse_tool_result(_adversarial_bash_tr()) assert parsed is not None assert parsed["result_type"] == "bash", ( @@ -115,14 +119,7 @@ def test_adversarial_bash_result_python_path_wraps_in_code_fence() -> None: (renderer regressed to raw prose), ``opening_fence`` below would be None and the assertion would fail — a diverging renderer cannot pass this test. """ - tr: dict[str, object] = { - "stdout": ADVERSARIAL_PAYLOAD, - "exitCode": 0, - "stderr": "", - "interrupted": False, - "is_error": False, - } - parsed = _parse_tool_result(tr) + parsed = _parse_tool_result(_adversarial_bash_tr()) assert parsed is not None md = _render_tool_result(parsed) From c15588b036a8567343234a23757de1cdaad8c806 Mon Sep 17 00:00:00 2001 From: chen Date: Fri, 24 Jul 2026 00:42:05 +0800 Subject: [PATCH 4/4] address timon's feedback --- .../tool_result/dual_path_parity_oracle.test.js | 5 +++-- tests/test_dual_path_parity_oracle.py | 15 +++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/static/js/render/tool_result/dual_path_parity_oracle.test.js b/static/js/render/tool_result/dual_path_parity_oracle.test.js index 2f1bd1f..7de67fa 100644 --- a/static/js/render/tool_result/dual_path_parity_oracle.test.js +++ b/static/js/render/tool_result/dual_path_parity_oracle.test.js @@ -22,8 +22,9 @@ import { renderBashResult } from './bash.js'; // tests/test_dual_path_parity_oracle.py. const PAYLOAD = ''; -// Parsed bash dict that mirrors what ``_parse_tool_result`` / ``_tool_result_build_bash`` -// would produce for the adversarial blob. +// Hand-built parsed shape for the JS renderer only. Classification and fields from +// ``_parse_tool_result`` are asserted in tests/test_dual_path_parity_oracle.py; +// this file will not catch drift if the Python bash builder changes key names. const ADVERSARIAL_PARSED = { result_type: 'bash', slug: null, diff --git a/tests/test_dual_path_parity_oracle.py b/tests/test_dual_path_parity_oracle.py index e602901..9e961ed 100644 --- a/tests/test_dual_path_parity_oracle.py +++ b/tests/test_dual_path_parity_oracle.py @@ -116,8 +116,7 @@ def test_adversarial_bash_result_python_path_wraps_in_code_fence() -> None: code fence, neutralising the XSS vector as inert literal text. Negative control: if the code fence were removed from ``_render_tool_result`` - (renderer regressed to raw prose), ``opening_fence`` below would be None and - the assertion would fail — a diverging renderer cannot pass this test. + (renderer regressed to raw prose), the odd-fence check below would fail. """ parsed = _parse_tool_result(_adversarial_bash_tr()) assert parsed is not None @@ -140,13 +139,13 @@ def test_adversarial_bash_result_python_path_wraps_in_code_fence() -> None: assert payload_indices, "adversarial payload must appear in at least one output line" payload_idx = payload_indices[0] - opening_fence = next((i for i in fence_indices if i < payload_idx), None) - closing_fence = next((i for i in fence_indices if i > payload_idx), None) - - assert opening_fence is not None, ( - f"no opening code fence before payload at line {payload_idx}; " - "payload is exposed in bare Markdown prose — Python renderer diverged" + fences_before = [i for i in fence_indices if i < payload_idx] + assert len(fences_before) % 2 == 1, ( + f"payload at line {payload_idx} must sit inside an open code fence " + f"({len(fences_before)} fence marker(s) before it); " + "bare prose between fenced blocks would still pass a naive before/after check" ) + closing_fence = next((i for i in fence_indices if i > payload_idx), None) assert closing_fence is not None, ( f"no closing code fence after payload at line {payload_idx}; " "code fence is unclosed — Python renderer diverged"